jroIO_base.py
1825 lines
| 53.3 KiB
| text/x-python
|
PythonLexer
Daniel Valdez
|
r487 | ''' | |
Miguel Valdez
|
r568 | Created on Jul 2, 2014 | |
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r568 | @author: roj-idl71 | |
|
Daniel Valdez
|
r487 | ''' | |
| import os | |||
| import sys | |||
| import glob | |||
| import time | |||
| import numpy | |||
| import fnmatch | |||
José Chávez
|
r944 | import inspect | |
|
José Chávez
|
r1074 | import time | |
| import datetime | |||
|
Daniel Valdez
|
r487 | import traceback | |
|
José Chávez
|
r963 | import zmq | |
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r568 | try: | |
| from gevent import sleep | |||
| except: | |||
| from time import sleep | |||
| r892 | |||
| r1130 | import schainpy.admin | ||
|
Miguel Valdez
|
r568 | from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader | |
|
Miguel Valdez
|
r616 | from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width | |
Juan C. Espinoza
|
r1112 | from schainpy.utils import log | |
| r1150 | import schainpy.admin | ||
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r568 | LOCALTIME = True | |
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r568 | def isNumber(cad): | |
|
Daniel Valdez
|
r487 | """ | |
| Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. | |||
| r892 | Excepciones: | ||
|
Daniel Valdez
|
r487 | Si un determinado string no puede ser convertido a numero | |
| Input: | |||
| str, string al cual se le analiza para determinar si convertible a un numero o no | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Return: | |
| True : si el string es uno numerico | |||
| False : no es un string numerico | |||
| """ | |||
| try: | |||
|
José Chávez
|
r1074 | float(cad) | |
|
Daniel Valdez
|
r487 | return True | |
| except: | |||
| return False | |||
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r640 | def isFileInEpoch(filename, startUTSeconds, endUTSeconds): | |
|
Daniel Valdez
|
r487 | """ | |
| Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Inputs: | |
| filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en | |
| segundos contados desde 01/01/1970. | |||
| endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en | |||
| segundos contados desde 01/01/1970. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Return: | |
| Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |||
| fecha especificado, de lo contrario retorna False. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Excepciones: | |
| Si el archivo no existe o no puede ser abierto | |||
| Si la cabecera no puede ser leida. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | """ | |
| basicHeaderObj = BasicHeader(LOCALTIME) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | try: | |
|
José Chávez
|
r1074 | fp = open(filename, 'rb') | |
|
Daniel Valdez
|
r487 | except IOError: | |
George Yong
|
r1167 | print("The file %s can't be opened" % (filename)) | |
|
Miguel Valdez
|
r684 | return 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | sts = basicHeaderObj.read(fp) | |
| fp.close() | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if not(sts): | |
|
George Yong
|
r1167 | print("Skipping the file %s because it has not a valid header" % (filename)) | |
|
Daniel Valdez
|
r487 | return 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)): | |
| return 0 | |||
| r892 | |||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r759 | def isTimeInRange(thisTime, startTime, endTime): | |
| if endTime >= startTime: | |||
| if (thisTime < startTime) or (thisTime > endTime): | |||
| return 0 | |||
| return 1 | |||
| else: | |||
| if (thisTime < startTime) and (thisTime > endTime): | |||
| return 0 | |||
| return 1 | |||
| r892 | |||
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r652 | def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): | |
|
Daniel Valdez
|
r487 | """ | |
| Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Inputs: | |
| filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |||
| r892 | |||
|
Miguel Valdez
|
r652 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| r892 | |||
|
Miguel Valdez
|
r652 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| r892 | |||
|
Daniel Valdez
|
r487 | startTime : tiempo inicial del rango seleccionado en formato datetime.time | |
| r892 | |||
|
Daniel Valdez
|
r487 | endTime : tiempo final del rango seleccionado en formato datetime.time | |
| r892 | |||
|
Daniel Valdez
|
r487 | Return: | |
| Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |||
| fecha especificado, de lo contrario retorna False. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Excepciones: | |
| Si el archivo no existe o no puede ser abierto | |||
| Si la cabecera no puede ser leida. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | """ | |
| r892 | |||
|
Daniel Valdez
|
r487 | try: | |
|
José Chávez
|
r1074 | fp = open(filename, 'rb') | |
|
Daniel Valdez
|
r487 | except IOError: | |
|
George Yong
|
r1167 | print("The file %s can't be opened" % (filename)) | |
|
Miguel Valdez
|
r684 | return None | |
| r892 | |||
|
Miguel Valdez
|
r759 | firstBasicHeaderObj = BasicHeader(LOCALTIME) | |
| systemHeaderObj = SystemHeader() | |||
| radarControllerHeaderObj = RadarControllerHeader() | |||
| processingHeaderObj = ProcessingHeader() | |||
| r892 | |||
|
Miguel Valdez
|
r759 | lastBasicHeaderObj = BasicHeader(LOCALTIME) | |
| r892 | |||
|
Miguel Valdez
|
r759 | sts = firstBasicHeaderObj.read(fp) | |
| r892 | |||
|
Miguel Valdez
|
r780 | if not(sts): | |
|
George Yong
|
r1167 | print("[Reading] Skipping the file %s because it has not a valid header" % (filename)) | |
|
Miguel Valdez
|
r780 | return None | |
| r892 | |||
|
Miguel Valdez
|
r803 | if not systemHeaderObj.read(fp): | |
| return None | |||
| r892 | |||
|
Miguel Valdez
|
r803 | if not radarControllerHeaderObj.read(fp): | |
| return None | |||
| r892 | |||
|
Miguel Valdez
|
r803 | if not processingHeaderObj.read(fp): | |
| return None | |||
| r892 | |||
|
Miguel Valdez
|
r780 | filesize = os.path.getsize(filename) | |
| r892 | |||
|
José Chávez
|
r1074 | offset = processingHeaderObj.blockSize + 24 # header size | |
| r892 | |||
|
Miguel Valdez
|
r780 | if filesize <= offset: | |
|
George Yong
|
r1167 | print("[Reading] %s: This file has not enough data" % filename) | |
|
Miguel Valdez
|
r780 | return None | |
| r892 | |||
|
Miguel Valdez
|
r759 | fp.seek(-offset, 2) | |
| r892 | |||
|
Miguel Valdez
|
r759 | sts = lastBasicHeaderObj.read(fp) | |
| r892 | |||
|
Miguel Valdez
|
r759 | fp.close() | |
| r892 | |||
|
Miguel Valdez
|
r780 | thisDatetime = lastBasicHeaderObj.datatime | |
| thisTime_last_block = thisDatetime.time() | |||
| r892 | |||
|
Miguel Valdez
|
r759 | thisDatetime = firstBasicHeaderObj.datatime | |
| thisDate = thisDatetime.date() | |||
| thisTime_first_block = thisDatetime.time() | |||
| r892 | |||
|
José Chávez
|
r1074 | # General case | |
|
Miguel Valdez
|
r648 | # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o | |
| #-----------o----------------------------o----------- | |||
| # startTime endTime | |||
| r892 | |||
|
Miguel Valdez
|
r648 | if endTime >= startTime: | |
|
Miguel Valdez
|
r759 | if (thisTime_last_block < startTime) or (thisTime_first_block > endTime): | |
|
Miguel Valdez
|
r648 | return None | |
| r892 | |||
|
Miguel Valdez
|
r648 | return thisDatetime | |
| r892 | |||
|
José Chávez
|
r1074 | # If endTime < startTime then endTime belongs to the next day | |
| r892 | |||
|
Miguel Valdez
|
r648 | #<<<<<<<<<<<o o>>>>>>>>>>> | |
| #-----------o----------------------------o----------- | |||
| # endTime startTime | |||
| r892 | |||
|
Miguel Valdez
|
r759 | if (thisDate == startDate) and (thisTime_last_block < startTime): | |
|
Miguel Valdez
|
r652 | return None | |
| r892 | |||
|
Miguel Valdez
|
r759 | if (thisDate == endDate) and (thisTime_first_block > endTime): | |
|
Miguel Valdez
|
r652 | return None | |
| r892 | |||
|
Miguel Valdez
|
r759 | if (thisTime_last_block < startTime) and (thisTime_first_block > endTime): | |
|
Daniel Valdez
|
r487 | return None | |
| r892 | |||
|
Daniel Valdez
|
r487 | return thisDatetime | |
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r640 | def isFolderInDateRange(folder, startDate=None, endDate=None): | |
| """ | |||
| Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |||
| r892 | |||
|
Miguel Valdez
|
r640 | Inputs: | |
| folder : nombre completo del directorio. | |||
| Su formato deberia ser "/path_root/?YYYYDDD" | |||
| r892 | |||
|
Miguel Valdez
|
r640 | siendo: | |
| YYYY : Anio (ejemplo 2015) | |||
| DDD : Dia del anio (ejemplo 305) | |||
| r892 | |||
|
Miguel Valdez
|
r640 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| r892 | |||
|
Miguel Valdez
|
r640 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| r892 | |||
|
Miguel Valdez
|
r640 | Return: | |
| Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |||
| fecha especificado, de lo contrario retorna False. | |||
| Excepciones: | |||
| Si el directorio no tiene el formato adecuado | |||
| """ | |||
| r892 | |||
|
Miguel Valdez
|
r640 | basename = os.path.basename(folder) | |
| r892 | |||
|
Miguel Valdez
|
r640 | if not isRadarFolder(basename): | |
|
George Yong
|
r1167 | print("The folder %s has not the rigth format" % folder) | |
|
Miguel Valdez
|
r684 | return 0 | |
| r892 | |||
|
Miguel Valdez
|
r640 | if startDate and endDate: | |
| thisDate = getDateFromRadarFolder(basename) | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if thisDate < startDate: | |
| return 0 | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if thisDate > endDate: | |
| return 0 | |||
| r892 | |||
|
Miguel Valdez
|
r640 | return 1 | |
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r640 | def isFileInDateRange(filename, startDate=None, endDate=None): | |
| """ | |||
| Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |||
| r892 | |||
|
Miguel Valdez
|
r640 | Inputs: | |
| filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |||
| r892 | |||
|
Miguel Valdez
|
r640 | Su formato deberia ser "?YYYYDDDsss" | |
| r892 | |||
|
Miguel Valdez
|
r640 | siendo: | |
| YYYY : Anio (ejemplo 2015) | |||
| DDD : Dia del anio (ejemplo 305) | |||
| sss : set | |||
| r892 | |||
|
Miguel Valdez
|
r640 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| r892 | |||
|
Miguel Valdez
|
r640 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| r892 | |||
|
Miguel Valdez
|
r640 | Return: | |
| Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |||
| fecha especificado, de lo contrario retorna False. | |||
| Excepciones: | |||
| Si el archivo no tiene el formato adecuado | |||
| """ | |||
| r892 | |||
|
Miguel Valdez
|
r640 | basename = os.path.basename(filename) | |
| r892 | |||
|
Miguel Valdez
|
r640 | if not isRadarFile(basename): | |
|
George Yong
|
r1167 | print("The filename %s has not the rigth format" % filename) | |
|
Miguel Valdez
|
r684 | return 0 | |
| r892 | |||
|
Miguel Valdez
|
r640 | if startDate and endDate: | |
| thisDate = getDateFromRadarFile(basename) | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if thisDate < startDate: | |
| return 0 | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if thisDate > endDate: | |
| return 0 | |||
| r892 | |||
|
Miguel Valdez
|
r640 | return 1 | |
|
José Chávez
|
r1074 | ||
|
Daniel Valdez
|
r487 | def getFileFromSet(path, ext, set): | |
| validFilelist = [] | |||
| fileList = os.listdir(path) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | # 0 1234 567 89A BCDE | |
| # H YYYY DDD SSS .ext | |||
| r892 | |||
|
Daniel Valdez
|
r487 | for thisFile in fileList: | |
| try: | |||
| year = int(thisFile[1:5]) | |||
|
José Chávez
|
r1074 | doy = int(thisFile[5:8]) | |
|
Daniel Valdez
|
r487 | except: | |
| continue | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | |
| continue | |||
| r892 | |||
|
Daniel Valdez
|
r487 | validFilelist.append(thisFile) | |
|
José Chávez
|
r1074 | myfile = fnmatch.filter( | |
| validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set)) | |||
| r892 | |||
|
José Chávez
|
r1074 | if len(myfile) != 0: | |
|
Daniel Valdez
|
r487 | return myfile[0] | |
| else: | |||
|
José Chávez
|
r1074 | filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower()) | |
|
George Yong
|
r1167 | print('the filename %s does not exist' % filename) | |
| print('...going to the last file: ') | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if validFilelist: | |
|
José Chávez
|
r1074 | validFilelist = sorted(validFilelist, key=str.lower) | |
|
Daniel Valdez
|
r487 | return validFilelist[-1] | |
| return None | |||
|
José Chávez
|
r1074 | ||
|
Daniel Valdez
|
r487 | def getlastFileFromPath(path, ext): | |
| """ | |||
| Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext" | |||
| r892 | al final de la depuracion devuelve el ultimo file de la lista que quedo. | ||
| Input: | |||
|
Daniel Valdez
|
r487 | fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta | |
| r892 | ext : extension de los files contenidos en una carpeta | ||
|
Daniel Valdez
|
r487 | Return: | |
| El ultimo file de una determinada carpeta, no se considera el path. | |||
| """ | |||
| validFilelist = [] | |||
| fileList = os.listdir(path) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | # 0 1234 567 89A BCDE | |
| # H YYYY DDD SSS .ext | |||
| r892 | |||
|
Daniel Valdez
|
r487 | for thisFile in fileList: | |
| r892 | |||
|
Daniel Valdez
|
r487 | year = thisFile[1:5] | |
| if not isNumber(year): | |||
| continue | |||
| r892 | |||
|
Daniel Valdez
|
r487 | doy = thisFile[5:8] | |
| if not isNumber(doy): | |||
| continue | |||
| r892 | |||
|
Daniel Valdez
|
r487 | year = int(year) | |
| doy = int(doy) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | |
| continue | |||
| r892 | |||
|
Daniel Valdez
|
r487 | validFilelist.append(thisFile) | |
| if validFilelist: | |||
|
José Chávez
|
r1074 | validFilelist = sorted(validFilelist, key=str.lower) | |
|
Daniel Valdez
|
r487 | return validFilelist[-1] | |
| return None | |||
|
José Chávez
|
r1074 | ||
|
Daniel Valdez
|
r487 | def checkForRealPath(path, foldercounter, year, doy, set, ext): | |
| """ | |||
| Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path, | |||
| Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar | |||
| el path exacto de un determinado file. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Example : | |
| nombre correcto del file es .../.../D2009307/P2009307367.ext | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Entonces la funcion prueba con las siguientes combinaciones | |
| .../.../y2009307367.ext | |||
| .../.../Y2009307367.ext | |||
| .../.../x2009307/y2009307367.ext | |||
| .../.../x2009307/Y2009307367.ext | |||
| .../.../X2009307/y2009307367.ext | |||
| .../.../X2009307/Y2009307367.ext | |||
| r892 | siendo para este caso, la ultima combinacion de letras, identica al file buscado | ||
|
Daniel Valdez
|
r487 | Return: | |
| r892 | Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file | ||
| caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas | |||
| para el filename | |||
|
Daniel Valdez
|
r487 | """ | |
| fullfilename = None | |||
| find_flag = False | |||
| filename = None | |||
| r892 | |||
|
José Chávez
|
r1074 | prefixDirList = [None, 'd', 'D'] | |
| if ext.lower() == ".r": # voltage | |||
| prefixFileList = ['d', 'D'] | |||
| elif ext.lower() == ".pdata": # spectra | |||
| prefixFileList = ['p', 'P'] | |||
|
Daniel Valdez
|
r487 | else: | |
| return None, filename | |||
| r892 | |||
|
José Chávez
|
r1074 | # barrido por las combinaciones posibles | |
|
Daniel Valdez
|
r487 | for prefixDir in prefixDirList: | |
| thispath = path | |||
| if prefixDir != None: | |||
|
José Chávez
|
r1074 | # formo el nombre del directorio xYYYYDDD (x=d o x=D) | |
|
Daniel Valdez
|
r487 | if foldercounter == 0: | |
|
José Chávez
|
r1074 | thispath = os.path.join(path, "%s%04d%03d" % | |
| (prefixDir, year, doy)) | |||
|
Daniel Valdez
|
r487 | else: | |
|
José Chávez
|
r1074 | thispath = os.path.join(path, "%s%04d%03d_%02d" % ( | |
| prefixDir, year, doy, foldercounter)) | |||
| for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D" | |||
| # formo el nombre del file xYYYYDDDSSS.ext | |||
| filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext) | |||
| fullfilename = os.path.join( | |||
| thispath, filename) # formo el path completo | |||
| if os.path.exists(fullfilename): # verifico que exista | |||
|
Daniel Valdez
|
r487 | find_flag = True | |
| break | |||
| if find_flag: | |||
| break | |||
| if not(find_flag): | |||
| return None, filename | |||
| return fullfilename, filename | |||
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r589 | def isRadarFolder(folder): | |
|
Daniel Valdez
|
r487 | try: | |
| year = int(folder[1:5]) | |||
| doy = int(folder[5:8]) | |||
| except: | |||
| return 0 | |||
| r892 | |||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r589 | def isRadarFile(file): | |
|
Juan C. Espinoza
|
r1112 | try: | |
|
José Chávez
|
r1074 | year = int(file[1:5]) | |
| doy = int(file[5:8]) | |||
| set = int(file[8:11]) | |||
| except: | |||
| return 0 | |||
| return 1 | |||
| r892 | |||
|
Miguel Valdez
|
r589 | ||
| def getDateFromRadarFile(file): | |||
|
Juan C. Espinoza
|
r1112 | try: | |
|
Miguel Valdez
|
r640 | year = int(file[1:5]) | |
| doy = int(file[5:8]) | |||
|
Juan C. Espinoza
|
r1112 | set = int(file[8:11]) | |
|
Miguel Valdez
|
r640 | except: | |
| return None | |||
|
José Chávez
|
r1074 | thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1) | |
|
Miguel Valdez
|
r640 | return thisDate | |
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r640 | def getDateFromRadarFolder(folder): | |
| try: | |||
| year = int(folder[1:5]) | |||
| doy = int(folder[5:8]) | |||
| except: | |||
| return None | |||
|
José Chávez
|
r1074 | thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1) | |
|
Miguel Valdez
|
r640 | return thisDate | |
| r892 | |||
|
José Chávez
|
r1074 | ||
|
Daniel Valdez
|
r487 | class JRODataIO: | |
| r892 | |||
|
Daniel Valdez
|
r487 | c = 3E8 | |
| r892 | |||
|
Daniel Valdez
|
r487 | isConfig = False | |
| r892 | |||
|
Daniel Valdez
|
r487 | basicHeaderObj = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | systemHeaderObj = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | radarControllerHeaderObj = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | processingHeaderObj = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | dtype = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | pathList = [] | |
| r892 | |||
|
Daniel Valdez
|
r487 | filenameList = [] | |
| r892 | |||
|
Daniel Valdez
|
r487 | filename = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | ext = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | flagIsNewFile = 1 | |
| r892 | |||
|
Miguel Valdez
|
r568 | flagDiscontinuousBlock = 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | flagIsNewBlock = 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | fp = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | firstHeaderSize = 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | basicHeaderSize = 24 | |
| r892 | |||
|
Daniel Valdez
|
r487 | versionFile = 1103 | |
| r892 | |||
|
Daniel Valdez
|
r487 | fileSize = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | # ippSeconds = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | fileSizeByHeader = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | fileIndex = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | profileIndex = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | blockIndex = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | nTotalBlocks = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | maxTimeStep = 30 | |
| r892 | |||
|
Daniel Valdez
|
r487 | lastUTTime = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | datablock = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | dataOut = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | blocksize = None | |
| r892 | |||
|
Miguel Valdez
|
r527 | getByBlock = False | |
| r892 | |||
|
Daniel Valdez
|
r487 | def __init__(self): | |
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
| r892 | |||
|
Daniel Valdez
|
r487 | def run(self): | |
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r616 | def getDtypeWidth(self): | |
| r892 | |||
|
Miguel Valdez
|
r616 | dtype_index = get_dtype_index(self.dtype) | |
| dtype_width = get_dtype_width(dtype_index) | |||
| r892 | |||
|
Miguel Valdez
|
r616 | return dtype_width | |
| r892 | |||
|
José Chávez
|
r944 | def getAllowedArgs(self): | |
|
Juan C. Espinoza
|
r1097 | if hasattr(self, '__attrs__'): | |
| return self.__attrs__ | |||
| else: | |||
| return inspect.getargspec(self.run).args | |||
|
José Chávez
|
r944 | ||
|
José Chávez
|
r1074 | ||
|
Daniel Valdez
|
r487 | class JRODataReader(JRODataIO): | |
|
Juan C. Espinoza
|
r1052 | ||
|
Miguel Valdez
|
r659 | online = 0 | |
| r892 | |||
|
Miguel Valdez
|
r659 | realtime = 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | nReadBlocks = 0 | |
| r892 | |||
|
José Chávez
|
r1074 | delay = 10 # number of seconds waiting a new file | |
| r892 | |||
|
José Chávez
|
r1074 | nTries = 3 # quantity tries | |
| r892 | |||
|
José Chávez
|
r1074 | nFiles = 3 # number of files for searching | |
| r892 | |||
|
Daniel Valdez
|
r487 | path = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | foldercounter = 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | flagNoMoreFiles = 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | datetimeList = [] | |
| r892 | |||
|
Daniel Valdez
|
r487 | __isFirstTimeOnline = 1 | |
| r892 | |||
|
Daniel Valdez
|
r487 | __printInfo = True | |
| r892 | |||
|
Daniel Valdez
|
r487 | profileIndex = None | |
| r892 | |||
|
Miguel Valdez
|
r534 | nTxs = 1 | |
| r892 | |||
|
Miguel Valdez
|
r534 | txIndex = None | |
| r892 | |||
|
José Chávez
|
r1074 | # Added-------------------- | |
| r892 | |||
Julio Valdez
|
r833 | selBlocksize = None | |
| r892 | |||
|
Julio Valdez
|
r833 | selBlocktime = None | |
|
Juan C. Espinoza
|
r1052 | ||
|
Daniel Valdez
|
r487 | def __init__(self): | |
| """ | |||
|
Miguel Valdez
|
r684 | This class is used to find data files | |
| r892 | |||
|
Miguel Valdez
|
r684 | Example: | |
| reader = JRODataReader() | |||
| fileList = reader.findDataFiles() | |||
| r892 | |||
|
Miguel Valdez
|
r684 | """ | |
| pass | |||
| r892 | |||
|
Daniel Valdez
|
r487 | def createObjByDefault(self): | |
| """ | |||
| """ | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def getBlockDimension(self): | |||
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
|
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | |
| path, | |||
| startDate=None, | |||
| endDate=None, | |||
|
José Chávez
|
r1074 | startTime=datetime.time(0, 0, 0), | |
| endTime=datetime.time(23, 59, 59), | |||
|
Juan C. Espinoza
|
r1052 | set=None, | |
| expLabel='', | |||
| ext='.r', | |||
| cursor=None, | |||
| skip=None, | |||
| walk=True): | |||
|
Miguel Valdez
|
r589 | self.filenameList = [] | |
| self.datetimeList = [] | |||
| r892 | |||
|
Daniel Valdez
|
r487 | pathList = [] | |
| r892 | |||
|
José Chávez
|
r1074 | dateList, pathList = self.findDatafiles( | |
| path, startDate, endDate, expLabel, ext, walk, include_path=True) | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if dateList == []: | |
|
Juan C. Espinoza
|
r1052 | return [], [] | |
| r892 | |||
|
Miguel Valdez
|
r640 | if len(dateList) > 1: | |
|
George Yong
|
r1167 | print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))) | |
|
Miguel Valdez
|
r640 | else: | |
|
George Yong
|
r1167 | print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])) | |
| r892 | |||
|
Daniel Valdez
|
r487 | filenameList = [] | |
| datetimeList = [] | |||
|
Juan C. Espinoza
|
r1112 | ||
|
Miguel Valdez
|
r640 | for thisPath in pathList: | |
|
Juan C. Espinoza
|
r1052 | ||
|
José Chávez
|
r1074 | fileList = glob.glob1(thisPath, "*%s" % ext) | |
|
Daniel Valdez
|
r487 | fileList.sort() | |
| r892 | |||
|
Juan C. Espinoza
|
r1112 | for file in fileList: | |
| r892 | |||
|
José Chávez
|
r1074 | filename = os.path.join(thisPath, file) | |
| r892 | |||
|
Miguel Valdez
|
r640 | if not isFileInDateRange(filename, startDate, endDate): | |
| continue | |||
| r892 | |||
|
José Chávez
|
r1074 | thisDatetime = isFileInTimeRange( | |
| filename, startDate, endDate, startTime, endTime) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if not(thisDatetime): | |
| continue | |||
| r892 | |||
|
Daniel Valdez
|
r487 | filenameList.append(filename) | |
| datetimeList.append(thisDatetime) | |||
| r892 | |||
|
Juan C. Espinoza
|
r1112 | if cursor is not None and skip is not None: | |
| filenameList = filenameList[cursor * skip:cursor * skip + skip] | |||
| datetimeList = datetimeList[cursor * skip:cursor * skip + skip] | |||
|
Daniel Valdez
|
r487 | if not(filenameList): | |
|
George Yong
|
r1167 | print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)) | |
|
Juan C. Espinoza
|
r1052 | return [], [] | |
| r892 | |||
|
George Yong
|
r1167 | print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)) | |
| r892 | |||
|
Juan C. Espinoza
|
r1052 | # for i in range(len(filenameList)): | |
| # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) | |||
|
Daniel Valdez
|
r487 | ||
| self.filenameList = filenameList | |||
|
Juan C. Espinoza
|
r1052 | self.datetimeList = datetimeList | |
|
Daniel Valdez
|
r487 | return pathList, filenameList | |
|
José Chávez
|
r1074 | def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None): | |
|
Daniel Valdez
|
r487 | """ | |
|
Juan C. Espinoza
|
r1052 | Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y | |
| devuelve el archivo encontrado ademas de otros datos. | |||
| Input: | |||
| path : carpeta donde estan contenidos los files que contiene data | |||
| expLabel : Nombre del subexperimento (subfolder) | |||
| ext : extension de los files | |||
| walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath) | |||
| Return: | |||
| directory : eL directorio donde esta el file encontrado | |||
| filename : el ultimo file de una determinada carpeta | |||
| year : el anho | |||
| doy : el numero de dia del anho | |||
| set : el set del archivo | |||
|
Daniel Valdez
|
r487 | """ | |
|
Miguel Valdez
|
r700 | if not os.path.isdir(path): | |
| return None, None, None, None, None, None | |||
| r892 | |||
|
Miguel Valdez
|
r700 | dirList = [] | |
| r892 | |||
|
Daniel Valdez
|
r487 | if not walk: | |
| fullpath = path | |||
| foldercounter = 0 | |||
| else: | |||
|
José Chávez
|
r1074 | # Filtra solo los directorios | |
|
Daniel Valdez
|
r487 | for thisPath in os.listdir(path): | |
|
José Chávez
|
r1074 | if not os.path.isdir(os.path.join(path, thisPath)): | |
|
Daniel Valdez
|
r487 | continue | |
|
Miguel Valdez
|
r589 | if not isRadarFolder(thisPath): | |
|
Daniel Valdez
|
r487 | continue | |
| r892 | |||
|
Daniel Valdez
|
r487 | dirList.append(thisPath) | |
| r892 | |||
|
Daniel Valdez
|
r487 | if not(dirList): | |
| return None, None, None, None, None, None | |||
| r892 | |||
|
José Chávez
|
r1074 | dirList = sorted(dirList, key=str.lower) | |
| r892 | |||
|
Daniel Valdez
|
r487 | doypath = dirList[-1] | |
|
José Chávez
|
r1074 | foldercounter = int(doypath.split('_')[1]) if len( | |
| doypath.split('_')) > 1 else 0 | |||
|
Daniel Valdez
|
r487 | fullpath = os.path.join(path, doypath, expLabel) | |
| r892 | |||
|
George Yong
|
r1167 | print("[Reading] %s folder was found: " % (fullpath)) | |
|
Daniel Valdez
|
r487 | ||
| if set == None: | |||
| filename = getlastFileFromPath(fullpath, ext) | |||
| else: | |||
| filename = getFileFromSet(fullpath, ext, set) | |||
| if not(filename): | |||
| return None, None, None, None, None, None | |||
| r892 | |||
|
George Yong
|
r1167 | print("[Reading] %s file was found" % (filename)) | |
| r892 | |||
|
Daniel Valdez
|
r487 | if not(self.__verifyFile(os.path.join(fullpath, filename))): | |
| return None, None, None, None, None, None | |||
|
José Chávez
|
r1074 | year = int(filename[1:5]) | |
| doy = int(filename[5:8]) | |||
| set = int(filename[8:11]) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | return fullpath, foldercounter, filename, year, doy, set | |
| r892 | |||
|
Daniel Valdez
|
r487 | def __setNextFileOffline(self): | |
| r892 | |||
|
Daniel Valdez
|
r487 | idFile = self.fileIndex | |
| while (True): | |||
| idFile += 1 | |||
| if not(idFile < len(self.filenameList)): | |||
| self.flagNoMoreFiles = 1 | |||
|
Juan C. Espinoza
|
r1052 | # print "[Reading] No more Files" | |
|
Daniel Valdez
|
r487 | return 0 | |
| filename = self.filenameList[idFile] | |||
| if not(self.__verifyFile(filename)): | |||
| continue | |||
| fileSize = os.path.getsize(filename) | |||
|
José Chávez
|
r1074 | fp = open(filename, 'rb') | |
|
Daniel Valdez
|
r487 | break | |
| self.flagIsNewFile = 1 | |||
| self.fileIndex = idFile | |||
| self.filename = filename | |||
| self.fileSize = fileSize | |||
| self.fp = fp | |||
|
Juan C. Espinoza
|
r1052 | # print "[Reading] Setting the file: %s"%self.filename | |
|
Daniel Valdez
|
r487 | ||
| return 1 | |||
| def __setNextFileOnline(self): | |||
| """ | |||
|
Juan C. Espinoza
|
r1052 | Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si | |
| no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files | |||
| siguientes. | |||
| Affected: | |||
| self.flagIsNewFile | |||
| self.filename | |||
| self.fileSize | |||
| self.fp | |||
| self.set | |||
| self.flagNoMoreFiles | |||
| Return: | |||
| 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado | |||
| 1 : si el file fue abierto con exito y esta listo a ser leido | |||
| Excepciones: | |||
| Si un determinado file no puede ser abierto | |||
| """ | |||
|
Daniel Valdez
|
r487 | nFiles = 0 | |
| r892 | fileOk_flag = False | ||
|
Daniel Valdez
|
r487 | firstTime_flag = True | |
| self.set += 1 | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if self.set > 999: | |
| self.set = 0 | |||
| r892 | self.foldercounter += 1 | ||
|
José Chávez
|
r1074 | # busca el 1er file disponible | |
| fullfilename, filename = checkForRealPath( | |||
| self.path, self.foldercounter, self.year, self.doy, self.set, self.ext) | |||
|
Daniel Valdez
|
r487 | if fullfilename: | |
| if self.__verifyFile(fullfilename, False): | |||
| fileOk_flag = True | |||
|
José Chávez
|
r1074 | # si no encuentra un file entonces espera y vuelve a buscar | |
| r892 | if not(fileOk_flag): | ||
|
José Chávez
|
r1074 | # busco en los siguientes self.nFiles+1 files posibles | |
| for nFiles in range(self.nFiles + 1): | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces | |
|
Daniel Valdez
|
r487 | tries = self.nTries | |
| else: | |||
|
José Chávez
|
r1074 | tries = 1 # si no es la 1era vez entonces solo lo hace una vez | |
| r892 | |||
|
José Chávez
|
r1074 | for nTries in range(tries): | |
|
Daniel Valdez
|
r487 | if firstTime_flag: | |
|
George Yong
|
r1167 | print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)) | |
|
José Chávez
|
r1074 | sleep(self.delay) | |
|
Daniel Valdez
|
r487 | else: | |
|
George Yong
|
r1167 | print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)) | |
| r892 | |||
|
José Chávez
|
r1074 | fullfilename, filename = checkForRealPath( | |
| self.path, self.foldercounter, self.year, self.doy, self.set, self.ext) | |||
|
Daniel Valdez
|
r487 | if fullfilename: | |
| if self.__verifyFile(fullfilename): | |||
| fileOk_flag = True | |||
| break | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if fileOk_flag: | |
| break | |||
| firstTime_flag = False | |||
| r1130 | log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename)) | ||
|
Daniel Valdez
|
r487 | self.set += 1 | |
| r892 | |||
|
José Chávez
|
r1074 | # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta | |
| if nFiles == (self.nFiles - 1): | |||
|
Daniel Valdez
|
r487 | self.set = 0 | |
| self.doy += 1 | |||
| self.foldercounter = 0 | |||
| if fileOk_flag: | |||
|
José Chávez
|
r1074 | self.fileSize = os.path.getsize(fullfilename) | |
|
Daniel Valdez
|
r487 | self.filename = fullfilename | |
| self.flagIsNewFile = 1 | |||
|
José Chávez
|
r1074 | if self.fp != None: | |
| self.fp.close() | |||
|
Daniel Valdez
|
r487 | self.fp = open(fullfilename, 'rb') | |
| self.flagNoMoreFiles = 0 | |||
|
Juan C. Espinoza
|
r1052 | # print '[Reading] Setting the file: %s' % fullfilename | |
|
Daniel Valdez
|
r487 | else: | |
| self.fileSize = 0 | |||
| self.filename = None | |||
| self.flagIsNewFile = 0 | |||
| self.fp = None | |||
| self.flagNoMoreFiles = 1 | |||
| return fileOk_flag | |||
| r892 | |||
|
Daniel Valdez
|
r487 | def setNextFile(self): | |
| if self.fp != None: | |||
| self.fp.close() | |||
|
Juan C. Espinoza
|
r1052 | ||
|
Daniel Valdez
|
r487 | if self.online: | |
| newFile = self.__setNextFileOnline() | |||
| else: | |||
| newFile = self.__setNextFileOffline() | |||
|
Juan C. Espinoza
|
r1052 | ||
| r1130 | if not(newFile): | ||
| raise schainpy.admin.SchainWarning('No more files to read') | |||
|
Daniel Valdez
|
r487 | return 0 | |
|
José Chávez
|
r931 | if self.verbose: | |
|
George Yong
|
r1167 | print('[Reading] Setting the file: %s' % self.filename) | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.__readFirstHeader() | |
| self.nReadBlocks = 0 | |||
| return 1 | |||
| def __waitNewBlock(self): | |||
| """ | |||
| r892 | Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. | ||
|
Daniel Valdez
|
r487 | Si el modo de lectura es OffLine siempre retorn 0 | |
| """ | |||
| if not self.online: | |||
| return 0 | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile): | |
| return 0 | |||
| r892 | |||
|
Daniel Valdez
|
r487 | currentPointer = self.fp.tell() | |
| r892 | |||
|
Daniel Valdez
|
r487 | neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |
| r892 | |||
|
José Chávez
|
r1074 | for nTries in range(self.nTries): | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.fp.close() | |
|
José Chávez
|
r1074 | self.fp = open(self.filename, 'rb') | |
| self.fp.seek(currentPointer) | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | self.fileSize = os.path.getsize(self.filename) | |
|
Daniel Valdez
|
r487 | currentSize = self.fileSize - currentPointer | |
|
José Chávez
|
r1074 | if (currentSize >= neededSize): | |
|
Daniel Valdez
|
r487 | self.basicHeaderObj.read(self.fp) | |
| return 1 | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if self.fileSize == self.fileSizeByHeader: | |
|
José Chávez
|
r1074 | # self.flagEoF = True | |
|
Daniel Valdez
|
r487 | return 0 | |
| r892 | |||
|
George Yong
|
r1167 | print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)) | |
|
José Chávez
|
r1074 | sleep(self.delay) | |
| r892 | |||
| return 0 | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | def waitDataBlock(self, pointer_location): | |
| r892 | |||
|
Daniel Valdez
|
r487 | currentPointer = pointer_location | |
| r892 | |||
|
José Chávez
|
r1074 | neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize | |
| r892 | |||
|
José Chávez
|
r1074 | for nTries in range(self.nTries): | |
|
Daniel Valdez
|
r487 | self.fp.close() | |
|
José Chávez
|
r1074 | self.fp = open(self.filename, 'rb') | |
| self.fp.seek(currentPointer) | |||
| r892 | |||
|
José Chávez
|
r1074 | self.fileSize = os.path.getsize(self.filename) | |
|
Daniel Valdez
|
r487 | currentSize = self.fileSize - currentPointer | |
| r892 | |||
|
José Chávez
|
r1074 | if (currentSize >= neededSize): | |
|
Daniel Valdez
|
r487 | return 1 | |
| r892 | |||
|
George Yong
|
r1167 | print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)) | |
|
José Chávez
|
r1074 | sleep(self.delay) | |
| r892 | |||
|
Daniel Valdez
|
r487 | return 0 | |
| def __jumpToLastBlock(self): | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if not(self.__isFirstTimeOnline): | |
| return | |||
| r892 | |||
|
Daniel Valdez
|
r487 | csize = self.fileSize - self.fp.tell() | |
| blocksize = self.processingHeaderObj.blockSize | |||
| r892 | |||
|
José Chávez
|
r1074 | # salta el primer bloque de datos | |
|
Daniel Valdez
|
r487 | if csize > self.processingHeaderObj.blockSize: | |
| self.fp.seek(self.fp.tell() + blocksize) | |||
| else: | |||
| return | |||
| r892 | |||
|
Daniel Valdez
|
r487 | csize = self.fileSize - self.fp.tell() | |
| neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |||
| while True: | |||
| r892 | |||
|
José Chávez
|
r1074 | if self.fp.tell() < self.fileSize: | |
|
Daniel Valdez
|
r487 | self.fp.seek(self.fp.tell() + neededsize) | |
| else: | |||
| self.fp.seek(self.fp.tell() - neededsize) | |||
| break | |||
|
Juan C. Espinoza
|
r1052 | ||
| # csize = self.fileSize - self.fp.tell() | |||
| # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |||
| # factor = int(csize/neededsize) | |||
| # if factor > 0: | |||
| # self.fp.seek(self.fp.tell() + factor*neededsize) | |||
|
Daniel Valdez
|
r487 | self.flagIsNewFile = 0 | |
| self.__isFirstTimeOnline = 0 | |||
| def __setNewBlock(self): | |||
|
José Chávez
|
r1074 | # if self.server is None: | |
|
José Chávez
|
r975 | if self.fp == None: | |
| return 0 | |||
|
Juan C. Espinoza
|
r1052 | ||
| # if self.online: | |||
|
José Chávez
|
r1074 | # self.__jumpToLastBlock() | |
|
Juan C. Espinoza
|
r1052 | ||
|
Daniel Valdez
|
r487 | if self.flagIsNewFile: | |
|
Miguel Valdez
|
r659 | self.lastUTTime = self.basicHeaderObj.utc | |
|
Daniel Valdez
|
r487 | return 1 | |
| r892 | |||
|
Miguel Valdez
|
r659 | if self.realtime: | |
| self.flagDiscontinuousBlock = 1 | |||
| if not(self.setNextFile()): | |||
| return 0 | |||
| else: | |||
|
José Chávez
|
r1074 | return 1 | |
| # if self.server is None: | |||
|
José Chávez
|
r975 | currentSize = self.fileSize - self.fp.tell() | |
| neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |||
| if (currentSize >= neededSize): | |||
| self.basicHeaderObj.read(self.fp) | |||
|
Miguel Valdez
|
r659 | self.lastUTTime = self.basicHeaderObj.utc | |
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r975 | # else: | |
| # self.basicHeaderObj.read(self.zHeader) | |||
| # self.lastUTTime = self.basicHeaderObj.utc | |||
| # return 1 | |||
|
Daniel Valdez
|
r487 | if self.__waitNewBlock(): | |
|
Miguel Valdez
|
r659 | self.lastUTTime = self.basicHeaderObj.utc | |
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r1074 | # if self.server is None: | |
|
José Chávez
|
r975 | if not(self.setNextFile()): | |
| return 0 | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | deltaTime = self.basicHeaderObj.utc - self.lastUTTime | |
|
Miguel Valdez
|
r659 | self.lastUTTime = self.basicHeaderObj.utc | |
| r892 | |||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 0 | |
|
Daniel Valdez
|
r487 | ||
| if deltaTime > self.maxTimeStep: | |||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 1 | |
|
Daniel Valdez
|
r487 | ||
| return 1 | |||
| def readNextBlock(self): | |||
| r892 | |||
|
José Chávez
|
r1074 | # Skip block out of startTime and endTime | |
| while True: | |||
| if not(self.__setNewBlock()): | |||
|
George Yong
|
r1167 | raise schainpy | |
|
Miguel Valdez
|
r759 | return 0 | |
|
José Chávez
|
r1074 | ||
|
Miguel Valdez
|
r759 | if not(self.readBlock()): | |
| return 0 | |||
|
Juan C. Espinoza
|
r1052 | ||
|
Miguel Valdez
|
r759 | self.getBasicHeader() | |
|
José Chávez
|
r1079 | if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)): | |
|
George Yong
|
r1167 | print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks, | |
|
José Chávez
|
r1074 | self.processingHeaderObj.dataBlocksPerFile, | |
|
George Yong
|
r1167 | self.dataOut.datatime.ctime())) | |
|
Miguel Valdez
|
r759 | continue | |
| r892 | |||
|
Miguel Valdez
|
r759 | break | |
| r892 | |||
|
José Chávez
|
r931 | if self.verbose: | |
|
George Yong
|
r1167 | print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks, | |
|
José Chávez
|
r1074 | self.processingHeaderObj.dataBlocksPerFile, | |
|
George Yong
|
r1167 | self.dataOut.datatime.ctime())) | |
|
Daniel Valdez
|
r487 | return 1 | |
| def __readFirstHeader(self): | |||
| r892 | |||
|
Daniel Valdez
|
r487 | self.basicHeaderObj.read(self.fp) | |
| self.systemHeaderObj.read(self.fp) | |||
| self.radarControllerHeaderObj.read(self.fp) | |||
| self.processingHeaderObj.read(self.fp) | |||
| self.firstHeaderSize = self.basicHeaderObj.size | |||
|
José Chávez
|
r1074 | datatype = int(numpy.log2((self.processingHeaderObj.processFlags & | |
| PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR)) | |||
|
Daniel Valdez
|
r487 | if datatype == 0: | |
|
José Chávez
|
r1074 | datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')]) | |
|
Daniel Valdez
|
r487 | elif datatype == 1: | |
|
José Chávez
|
r1074 | datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')]) | |
|
Daniel Valdez
|
r487 | elif datatype == 2: | |
|
José Chávez
|
r1074 | datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')]) | |
|
Daniel Valdez
|
r487 | elif datatype == 3: | |
|
José Chávez
|
r1074 | datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')]) | |
|
Daniel Valdez
|
r487 | elif datatype == 4: | |
|
José Chávez
|
r1074 | datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')]) | |
|
Daniel Valdez
|
r487 | elif datatype == 5: | |
|
José Chávez
|
r1074 | datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')]) | |
|
Daniel Valdez
|
r487 | else: | |
|
George Yong
|
r1167 | raise ValueError('Data type was not defined') | |
|
Daniel Valdez
|
r487 | ||
| self.dtype = datatype_str | |||
| #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c | |||
|
José Chávez
|
r1074 | self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \ | |
| self.firstHeaderSize + self.basicHeaderSize * \ | |||
| (self.processingHeaderObj.dataBlocksPerFile - 1) | |||
|
José Chávez
|
r1079 | # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels) | |
| # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels) | |||
|
Daniel Valdez
|
r487 | self.getBlockDimension() | |
| r892 | |||
|
Daniel Valdez
|
r487 | def __verifyFile(self, filename, msgFlag=True): | |
| r892 | |||
|
Daniel Valdez
|
r487 | msg = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | try: | |
| fp = open(filename, 'rb') | |||
| except IOError: | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if msgFlag: | |
|
George Yong
|
r1167 | print("[Reading] File %s can't be opened" % (filename)) | |
| r892 | |||
|
Daniel Valdez
|
r487 | return False | |
| r892 | |||
|
Miguel Valdez
|
r684 | currentPosition = fp.tell() | |
|
Daniel Valdez
|
r487 | neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize | |
| if neededSize == 0: | |||
| basicHeaderObj = BasicHeader(LOCALTIME) | |||
| systemHeaderObj = SystemHeader() | |||
| radarControllerHeaderObj = RadarControllerHeader() | |||
| processingHeaderObj = ProcessingHeader() | |||
| r892 | |||
|
José Chávez
|
r1074 | if not(basicHeaderObj.read(fp)): | |
|
Miguel Valdez
|
r684 | fp.close() | |
| return False | |||
| r892 | |||
|
José Chávez
|
r1074 | if not(systemHeaderObj.read(fp)): | |
|
Daniel Valdez
|
r487 | fp.close() | |
| return False | |||
| r892 | |||
|
José Chávez
|
r1074 | if not(radarControllerHeaderObj.read(fp)): | |
|
Miguel Valdez
|
r684 | fp.close() | |
| return False | |||
| r892 | |||
|
José Chávez
|
r1074 | if not(processingHeaderObj.read(fp)): | |
|
Miguel Valdez
|
r684 | fp.close() | |
| return False | |||
| r892 | |||
|
Miguel Valdez
|
r684 | neededSize = processingHeaderObj.blockSize + basicHeaderObj.size | |
|
Daniel Valdez
|
r487 | else: | |
|
José Chávez
|
r1074 | msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename | |
|
Daniel Valdez
|
r487 | ||
| fp.close() | |||
| r892 | |||
|
Daniel Valdez
|
r487 | fileSize = os.path.getsize(filename) | |
| currentSize = fileSize - currentPosition | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if currentSize < neededSize: | |
| if msgFlag and (msg != None): | |||
|
George Yong
|
r1167 | print(msg) | |
|
Daniel Valdez
|
r487 | return False | |
| return True | |||
|
Miguel Valdez
|
r640 | def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False): | |
| r892 | |||
|
Miguel Valdez
|
r759 | path_empty = True | |
| r892 | |||
|
Miguel Valdez
|
r589 | dateList = [] | |
| pathList = [] | |||
| r892 | |||
|
Miguel Valdez
|
r640 | multi_path = path.split(',') | |
| r892 | |||
|
Miguel Valdez
|
r589 | if not walk: | |
| r892 | |||
|
Miguel Valdez
|
r589 | for single_path in multi_path: | |
| r892 | |||
|
Miguel Valdez
|
r607 | if not os.path.isdir(single_path): | |
| continue | |||
| r892 | |||
|
José Chávez
|
r1074 | fileList = glob.glob1(single_path, "*" + ext) | |
| r892 | |||
|
Miguel Valdez
|
r726 | if not fileList: | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r759 | path_empty = False | |
| r892 | |||
|
Miguel Valdez
|
r726 | fileList.sort() | |
| r892 | |||
|
Miguel Valdez
|
r589 | for thisFile in fileList: | |
| r892 | |||
|
Miguel Valdez
|
r589 | if not os.path.isfile(os.path.join(single_path, thisFile)): | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r589 | if not isRadarFile(thisFile): | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if not isFileInDateRange(thisFile, startDate, endDate): | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r589 | thisDate = getDateFromRadarFile(thisFile) | |
| r892 | |||
|
Miguel Valdez
|
r640 | if thisDate in dateList: | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r640 | dateList.append(thisDate) | |
|
Miguel Valdez
|
r589 | pathList.append(single_path) | |
| r892 | |||
|
Miguel Valdez
|
r640 | else: | |
| for single_path in multi_path: | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if not os.path.isdir(single_path): | |
|
Miguel Valdez
|
r589 | continue | |
| r892 | |||
|
Miguel Valdez
|
r640 | dirList = [] | |
| r892 | |||
|
Miguel Valdez
|
r640 | for thisPath in os.listdir(single_path): | |
| r892 | |||
|
José Chávez
|
r1074 | if not os.path.isdir(os.path.join(single_path, thisPath)): | |
|
Miguel Valdez
|
r589 | continue | |
| r892 | |||
|
Miguel Valdez
|
r640 | if not isRadarFolder(thisPath): | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r640 | if not isFolderInDateRange(thisPath, startDate, endDate): | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r640 | dirList.append(thisPath) | |
| r892 | |||
|
Miguel Valdez
|
r640 | if not dirList: | |
| continue | |||
| r892 | |||
|
Miguel Valdez
|
r726 | dirList.sort() | |
| r892 | |||
|
Miguel Valdez
|
r601 | for thisDir in dirList: | |
| r892 | |||
|
Miguel Valdez
|
r632 | datapath = os.path.join(single_path, thisDir, expLabel) | |
|
José Chávez
|
r1074 | fileList = glob.glob1(datapath, "*" + ext) | |
| r892 | |||
|
Miguel Valdez
|
r759 | if not fileList: | |
|
Miguel Valdez
|
r632 | continue | |
| r892 | |||
|
Miguel Valdez
|
r759 | path_empty = False | |
| r892 | |||
|
Miguel Valdez
|
r640 | thisDate = getDateFromRadarFolder(thisDir) | |
| r892 | |||
|
Miguel Valdez
|
r632 | pathList.append(datapath) | |
|
Miguel Valdez
|
r589 | dateList.append(thisDate) | |
| r892 | |||
|
Miguel Valdez
|
r640 | dateList.sort() | |
| r892 | |||
|
Miguel Valdez
|
r684 | if walk: | |
| pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel) | |||
| else: | |||
| pattern_path = multi_path[0] | |||
| r892 | |||
|
Miguel Valdez
|
r759 | if path_empty: | |
|
George Yong
|
r1167 | print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)) | |
|
Miguel Valdez
|
r759 | else: | |
| if not dateList: | |||
|
George Yong
|
r1167 | print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)) | |
|
Miguel Valdez
|
r759 | ||
|
Miguel Valdez
|
r640 | if include_path: | |
| return dateList, pathList | |||
| r892 | |||
|
Miguel Valdez
|
r640 | return dateList | |
| r892 | |||
|
Daniel Valdez
|
r487 | def setup(self, | |
|
José Chávez
|
r1074 | path=None, | |
| startDate=None, | |||
| endDate=None, | |||
| startTime=datetime.time(0, 0, 0), | |||
| endTime=datetime.time(23, 59, 59), | |||
| set=None, | |||
| expLabel="", | |||
| ext=None, | |||
| online=False, | |||
| delay=60, | |||
| walk=True, | |||
| getblock=False, | |||
| nTxs=1, | |||
| realtime=False, | |||
| blocksize=None, | |||
| blocktime=None, | |||
| skip=None, | |||
| cursor=None, | |||
| warnings=True, | |||
| verbose=True, | |||
| server=None, | |||
| format=None, | |||
| oneDDict=None, | |||
| twoDDict=None, | |||
| ind2DList=None): | |||
|
Juan C. Espinoza
|
r1052 | if server is not None: | |
| if 'tcp://' in server: | |||
| address = server | |||
| else: | |||
| address = 'ipc:///tmp/%s' % server | |||
| self.server = address | |||
| self.context = zmq.Context() | |||
| self.receiver = self.context.socket(zmq.PULL) | |||
| self.receiver.connect(self.server) | |||
| time.sleep(0.5) | |||
|
George Yong
|
r1167 | print('[Starting] ReceiverData from {}'.format(self.server)) | |
|
José Chávez
|
r1074 | else: | |
|
Juan C. Espinoza
|
r1052 | self.server = None | |
| if path == None: | |||
|
George Yong
|
r1167 | raise ValueError("[Reading] The path is not valid") | |
|
Daniel Valdez
|
r487 | ||
|
Juan C. Espinoza
|
r1052 | if ext == None: | |
| ext = self.ext | |||
|
Daniel Valdez
|
r487 | ||
|
Juan C. Espinoza
|
r1052 | if online: | |
|
George Yong
|
r1167 | print("[Reading] Searching files in online mode...") | |
|
Juan C. Espinoza
|
r1052 | ||
|
José Chávez
|
r1074 | for nTries in range(self.nTries): | |
| fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine( | |||
| path=path, expLabel=expLabel, ext=ext, walk=walk, set=set) | |||
|
Juan C. Espinoza
|
r1052 | ||
| if fullpath: | |||
| break | |||
|
George Yong
|
r1167 | print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (delay, path, nTries + 1)) | |
| r1130 | sleep(delay) | ||
|
Juan C. Espinoza
|
r1052 | ||
| r1130 | if not(fullpath): | ||
| raise schainpy.admin.SchainWarning('There isn\'t any valid file in {}'.format(path)) | |||
|
Juan C. Espinoza
|
r1052 | return | |
| self.year = year | |||
|
José Chávez
|
r1074 | self.doy = doy | |
| self.set = set - 1 | |||
|
Juan C. Espinoza
|
r1052 | self.path = path | |
| self.foldercounter = foldercounter | |||
| last_set = None | |||
| else: | |||
|
George Yong
|
r1167 | print("[Reading] Searching files in offline mode ...") | |
|
Juan C. Espinoza
|
r1052 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, | |
|
José Chávez
|
r1074 | startTime=startTime, endTime=endTime, | |
| set=set, expLabel=expLabel, ext=ext, | |||
| walk=walk, cursor=cursor, | |||
| skip=skip) | |||
|
José Chávez
|
r1019 | ||
|
Juan C. Espinoza
|
r1052 | if not(pathList): | |
|
José Chávez
|
r1019 | self.fileIndex = -1 | |
|
Juan C. Espinoza
|
r1052 | self.pathList = [] | |
| self.filenameList = [] | |||
| return | |||
|
Miguel Valdez
|
r676 | self.fileIndex = -1 | |
|
Juan C. Espinoza
|
r1052 | self.pathList = pathList | |
| self.filenameList = filenameList | |||
| file_name = os.path.basename(filenameList[-1]) | |||
| basename, ext = os.path.splitext(file_name) | |||
| last_set = int(basename[-3:]) | |||
| self.online = online | |||
| self.realtime = realtime | |||
| self.delay = delay | |||
| ext = ext.lower() | |||
| self.ext = ext | |||
| self.getByBlock = getblock | |||
| self.nTxs = nTxs | |||
| self.startTime = startTime | |||
| self.endTime = endTime | |||
|
José Chávez
|
r1079 | self.endDate = endDate | |
| self.startDate = startDate | |||
|
José Chávez
|
r1074 | # Added----------------- | |
|
Juan C. Espinoza
|
r1052 | self.selBlocksize = blocksize | |
| self.selBlocktime = blocktime | |||
| # Verbose----------- | |||
| self.verbose = verbose | |||
| self.warnings = warnings | |||
| r892 | |||
|
Juan C. Espinoza
|
r1052 | if not(self.setNextFile()): | |
|
José Chávez
|
r1074 | if (startDate != None) and (endDate != None): | |
|
George Yong
|
r1167 | print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())) | |
|
Juan C. Espinoza
|
r1052 | elif startDate != None: | |
|
George Yong
|
r1167 | print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())) | |
|
Juan C. Espinoza
|
r1052 | else: | |
|
George Yong
|
r1167 | print("[Reading] No files") | |
|
José Chávez
|
r1019 | ||
|
Juan C. Espinoza
|
r1052 | self.fileIndex = -1 | |
| self.pathList = [] | |||
| self.filenameList = [] | |||
| return | |||
| # self.getBasicHeader() | |||
|
José Chávez
|
r1019 | ||
|
Juan C. Espinoza
|
r1052 | if last_set != None: | |
|
José Chávez
|
r1074 | self.dataOut.last_block = last_set * \ | |
| self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock | |||
|
Daniel Valdez
|
r487 | return | |
| def getBasicHeader(self): | |||
| r892 | |||
|
José Chávez
|
r1074 | self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \ | |
| 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds | |||
| r892 | |||
|
Miguel Valdez
|
r568 | self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.dataOut.timeZone = self.basicHeaderObj.timeZone | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.dataOut.dstFlag = self.basicHeaderObj.dstFlag | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.dataOut.errorCount = self.basicHeaderObj.errorCount | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime | |
| r892 | |||
|
José Chávez
|
r1074 | self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs | |
|
Juan C. Espinoza
|
r1052 | ||
| # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs | |||
|
Daniel Valdez
|
r487 | def getFirstHeader(self): | |
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
| r892 | |||
|
Daniel Valdez
|
r487 | def getData(self): | |
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def hasNotDataInBuffer(self): | |||
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def readBlock(self): | |||
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
| r892 | |||
|
Daniel Valdez
|
r487 | def isEndProcess(self): | |
| r892 | |||
|
Daniel Valdez
|
r487 | return self.flagNoMoreFiles | |
| r892 | |||
|
Daniel Valdez
|
r487 | def printReadBlocks(self): | |
| r892 | |||
|
George Yong
|
r1167 | print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks) | |
| r892 | |||
|
Daniel Valdez
|
r487 | def printTotalBlocks(self): | |
| r892 | |||
|
George Yong
|
r1167 | print("[Reading] Number of read blocks %04d" % self.nTotalBlocks) | |
|
Daniel Valdez
|
r487 | ||
| def printNumberOfBlock(self): | |||
ebocanegra
|
r1001 | 'SPAM!' | |
| r892 | |||
|
ebocanegra
|
r1001 | # if self.flagIsNewBlock: | |
| # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, | |||
| # self.processingHeaderObj.dataBlocksPerFile, | |||
| # self.dataOut.datatime.ctime()) | |||
|
Daniel Valdez
|
r487 | ||
| def printInfo(self): | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if self.__printInfo == False: | |
| return | |||
| r892 | |||
|
Daniel Valdez
|
r487 | self.basicHeaderObj.printInfo() | |
| self.systemHeaderObj.printInfo() | |||
| self.radarControllerHeaderObj.printInfo() | |||
| self.processingHeaderObj.printInfo() | |||
| r892 | |||
|
Daniel Valdez
|
r487 | self.__printInfo = False | |
| r892 | |||
|
José Chávez
|
r944 | def run(self, | |
|
José Chávez
|
r1039 | path=None, | |
| startDate=None, | |||
| endDate=None, | |||
|
José Chávez
|
r1074 | startTime=datetime.time(0, 0, 0), | |
| endTime=datetime.time(23, 59, 59), | |||
|
José Chávez
|
r1039 | set=None, | |
|
José Chávez
|
r1074 | expLabel="", | |
| ext=None, | |||
| online=False, | |||
| delay=60, | |||
| walk=True, | |||
| getblock=False, | |||
| nTxs=1, | |||
|
José Chávez
|
r1039 | realtime=False, | |
| blocksize=None, | |||
| blocktime=None, | |||
| skip=None, | |||
| cursor=None, | |||
| warnings=True, | |||
| server=None, | |||
|
José Chávez
|
r1074 | verbose=True, | |
| format=None, | |||
| oneDDict=None, | |||
| twoDDict=None, | |||
| ind2DList=None, **kwargs): | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if not(self.isConfig): | |
|
Juan C. Espinoza
|
r1052 | self.setup(path=path, | |
| startDate=startDate, | |||
| endDate=endDate, | |||
| startTime=startTime, | |||
| endTime=endTime, | |||
| set=set, | |||
| expLabel=expLabel, | |||
| ext=ext, | |||
| online=online, | |||
| delay=delay, | |||
| walk=walk, | |||
| getblock=getblock, | |||
| nTxs=nTxs, | |||
| realtime=realtime, | |||
| blocksize=blocksize, | |||
| blocktime=blocktime, | |||
| skip=skip, | |||
| cursor=cursor, | |||
| warnings=warnings, | |||
| server=server, | |||
| r1065 | verbose=verbose, | ||
| format=format, | |||
| oneDDict=oneDDict, | |||
| twoDDict=twoDDict, | |||
| ind2DList=ind2DList) | |||
|
Daniel Valdez
|
r487 | self.isConfig = True | |
|
José Chávez
|
r975 | if server is None: | |
| self.getData() | |||
|
José Chávez
|
r1074 | else: | |
|
José Chávez
|
r975 | self.getFromServer() | |
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | ||
|
Daniel Valdez
|
r487 | class JRODataWriter(JRODataIO): | |
| r892 | """ | ||
|
Daniel Valdez
|
r487 | Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura | |
| r892 | de los datos siempre se realiza por bloques. | ||
|
Daniel Valdez
|
r487 | """ | |
| r892 | |||
|
Daniel Valdez
|
r487 | blockIndex = 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | path = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | setFile = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | profilesPerBlock = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | blocksPerFile = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | nWriteBlocks = 0 | |
| r892 | |||
|
Miguel Valdez
|
r632 | fileDate = None | |
| r892 | |||
|
Daniel Valdez
|
r487 | def __init__(self, dataOut=None): | |
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def hasAllDataInBuffer(self): | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def setBlockDimension(self): | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def writeBlock(self): | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def putData(self): | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r616 | def getProcessFlags(self): | |
| r892 | |||
|
Miguel Valdez
|
r616 | processFlags = 0 | |
| r892 | |||
|
Miguel Valdez
|
r616 | dtype_index = get_dtype_index(self.dtype) | |
| procflag_dtype = get_procflag_dtype(dtype_index) | |||
| r892 | |||
|
Miguel Valdez
|
r616 | processFlags += procflag_dtype | |
| r892 | |||
|
Miguel Valdez
|
r616 | if self.dataOut.flagDecodeData: | |
| processFlags += PROCFLAG.DECODE_DATA | |||
| r892 | |||
|
Miguel Valdez
|
r616 | if self.dataOut.flagDeflipData: | |
| processFlags += PROCFLAG.DEFLIP_DATA | |||
| r892 | |||
|
Miguel Valdez
|
r616 | if self.dataOut.code is not None: | |
| processFlags += PROCFLAG.DEFINE_PROCESS_CODE | |||
| r892 | |||
|
Miguel Valdez
|
r616 | if self.dataOut.nCohInt > 1: | |
| processFlags += PROCFLAG.COHERENT_INTEGRATION | |||
| r892 | |||
|
Miguel Valdez
|
r616 | if self.dataOut.type == "Spectra": | |
| if self.dataOut.nIncohInt > 1: | |||
| processFlags += PROCFLAG.INCOHERENT_INTEGRATION | |||
| r892 | |||
|
Miguel Valdez
|
r616 | if self.dataOut.data_dc is not None: | |
| processFlags += PROCFLAG.SAVE_CHANNELS_DC | |||
| r892 | |||
|
Miguel Valdez
|
r624 | if self.dataOut.flagShiftFFT: | |
| processFlags += PROCFLAG.SHIFT_FFT_DATA | |||
| r892 | |||
|
Miguel Valdez
|
r616 | return processFlags | |
| r892 | |||
|
Daniel Valdez
|
r487 | def setBasicHeader(self): | |
| r892 | |||
|
José Chávez
|
r1074 | self.basicHeaderObj.size = self.basicHeaderSize # bytes | |
|
Daniel Valdez
|
r487 | self.basicHeaderObj.version = self.versionFile | |
|
Miguel Valdez
|
r632 | self.basicHeaderObj.dataBlock = self.nTotalBlocks | |
| r892 | |||
|
Daniel Valdez
|
r487 | utc = numpy.floor(self.dataOut.utctime) | |
|
José Chávez
|
r1074 | milisecond = (self.dataOut.utctime - utc) * 1000.0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.basicHeaderObj.utc = utc | |
| self.basicHeaderObj.miliSecond = milisecond | |||
| self.basicHeaderObj.timeZone = self.dataOut.timeZone | |||
| self.basicHeaderObj.dstFlag = self.dataOut.dstFlag | |||
| self.basicHeaderObj.errorCount = self.dataOut.errorCount | |||
| r892 | |||
|
Daniel Valdez
|
r487 | def setFirstHeader(self): | |
| """ | |||
| Obtiene una copia del First Header | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Affected: | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.basicHeaderObj | |
| self.systemHeaderObj | |||
| self.radarControllerHeaderObj | |||
| self.processingHeaderObj self. | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Return: | |
| None | |||
| """ | |||
| r892 | |||
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
| r892 | |||
|
Daniel Valdez
|
r487 | def __writeFirstHeader(self): | |
| """ | |||
| Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Affected: | |
| __dataType | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Return: | |
| None | |||
| """ | |||
|
Juan C. Espinoza
|
r1052 | ||
| # CALCULAR PARAMETROS | |||
|
José Chávez
|
r1074 | sizeLongHeader = self.systemHeaderObj.size + \ | |
| self.radarControllerHeaderObj.size + self.processingHeaderObj.size | |||
|
Daniel Valdez
|
r487 | self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.basicHeaderObj.write(self.fp) | |
| self.systemHeaderObj.write(self.fp) | |||
| self.radarControllerHeaderObj.write(self.fp) | |||
| self.processingHeaderObj.write(self.fp) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | def __setNewBlock(self): | |
| """ | |||
| Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Return: | |
| 0 : si no pudo escribir nada | |||
| 1 : Si escribio el Basic el First Header | |||
| r892 | """ | ||
|
Daniel Valdez
|
r487 | if self.fp == None: | |
| self.setNextFile() | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if self.flagIsNewFile: | |
| return 1 | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile: | |
| self.basicHeaderObj.write(self.fp) | |||
| return 1 | |||
| r892 | |||
|
José Chávez
|
r1074 | if not(self.setNextFile()): | |
|
Daniel Valdez
|
r487 | return 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | return 1 | |
| def writeNextBlock(self): | |||
| """ | |||
| Selecciona el bloque siguiente de datos y los escribe en un file | |||
| r892 | |||
| Return: | |||
| 0 : Si no hizo pudo escribir el bloque de datos | |||
|
Daniel Valdez
|
r487 | 1 : Si no pudo escribir el bloque de datos | |
| """ | |||
|
José Chávez
|
r1074 | if not(self.__setNewBlock()): | |
|
Daniel Valdez
|
r487 | return 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.writeBlock() | |
| r892 | |||
|
George Yong
|
r1167 | print("[Writing] Block No. %d/%d" % (self.blockIndex, | |
| self.processingHeaderObj.dataBlocksPerFile)) | |||
| r892 | |||
| return 1 | |||
|
Daniel Valdez
|
r487 | ||
| def setNextFile(self): | |||
| r892 | """ | ||
|
Daniel Valdez
|
r487 | Determina el siguiente file que sera escrito | |
| r892 | Affected: | ||
|
Daniel Valdez
|
r487 | self.filename | |
| self.subfolder | |||
| self.fp | |||
| self.setFile | |||
| self.flagIsNewFile | |||
| Return: | |||
| 0 : Si el archivo no puede ser escrito | |||
| 1 : Si el archivo esta listo para ser escrito | |||
| """ | |||
| ext = self.ext | |||
| path = self.path | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if self.fp != None: | |
| self.fp.close() | |||
| r892 | |||
|
José Chávez
|
r1074 | timeTuple = time.localtime(self.dataOut.utctime) | |
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday) | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | fullpath = os.path.join(path, subfolder) | |
|
Miguel Valdez
|
r632 | setFile = self.setFile | |
| r892 | |||
|
José Chávez
|
r1074 | if not(os.path.exists(fullpath)): | |
|
Daniel Valdez
|
r487 | os.mkdir(fullpath) | |
|
José Chávez
|
r1074 | setFile = -1 # inicializo mi contador de seteo | |
|
Daniel Valdez
|
r487 | else: | |
|
José Chávez
|
r1074 | filesList = os.listdir(fullpath) | |
| if len(filesList) > 0: | |||
| filesList = sorted(filesList, key=str.lower) | |||
|
Daniel Valdez
|
r487 | filen = filesList[-1] | |
| # el filename debera tener el siguiente formato | |||
| # 0 1234 567 89A BCDE (hex) | |||
| # x YYYY DDD SSS .ext | |||
|
José Chávez
|
r1074 | if isNumber(filen[8:11]): | |
| # inicializo mi contador de seteo al seteo del ultimo file | |||
| setFile = int(filen[8:11]) | |||
| r892 | else: | ||
|
Miguel Valdez
|
r632 | setFile = -1 | |
|
Daniel Valdez
|
r487 | else: | |
|
José Chávez
|
r1074 | setFile = -1 # inicializo mi contador de seteo | |
| r892 | |||
|
Daniel Valdez
|
r487 | setFile += 1 | |
| r892 | |||
|
José Chávez
|
r1074 | # If this is a new day it resets some values | |
|
Miguel Valdez
|
r632 | if self.dataOut.datatime.date() > self.fileDate: | |
| setFile = 0 | |||
| self.nTotalBlocks = 0 | |||
|
Juan C. Espinoza
|
r1112 | ||
| filen = '{}{:04d}{:03d}{:03d}{}'.format( | |||
| self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) | |||
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | filename = os.path.join(path, subfolder, filen) | |
|
Daniel Valdez
|
r487 | ||
|
José Chávez
|
r1074 | fp = open(filename, 'wb') | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.blockIndex = 0 | |
| r892 | |||
|
José Chávez
|
r1074 | # guardando atributos | |
|
Daniel Valdez
|
r487 | self.filename = filename | |
| self.subfolder = subfolder | |||
| self.fp = fp | |||
| self.setFile = setFile | |||
| self.flagIsNewFile = 1 | |||
|
Miguel Valdez
|
r632 | self.fileDate = self.dataOut.datatime.date() | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.setFirstHeader() | |
| r892 | |||
|
George Yong
|
r1167 | print('[Writing] Opening file: %s' % self.filename) | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.__writeFirstHeader() | |
| r892 | |||
|
Daniel Valdez
|
r487 | return 1 | |
| r892 | |||
|
Juan C. Espinoza
|
r1052 | def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4): | |
|
Daniel Valdez
|
r487 | """ | |
| r892 | Setea el tipo de formato en la cual sera guardada la data y escribe el First Header | ||
|
Daniel Valdez
|
r487 | Inputs: | |
|
Miguel Valdez
|
r616 | path : directory where data will be saved | |
| r892 | profilesPerBlock : number of profiles per block | ||
|
Miguel Valdez
|
r626 | set : initial file set | |
|
Miguel Valdez
|
r616 | datatype : An integer number that defines data type: | |
| 0 : int8 (1 byte) | |||
| 1 : int16 (2 bytes) | |||
| 2 : int32 (4 bytes) | |||
| 3 : int64 (8 bytes) | |||
|
Miguel Valdez
|
r626 | 4 : float32 (4 bytes) | |
| 5 : double64 (8 bytes) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | Return: | |
| 0 : Si no realizo un buen seteo | |||
| r892 | 1 : Si realizo un buen seteo | ||
|
Daniel Valdez
|
r487 | """ | |
| r892 | |||
|
Daniel Valdez
|
r487 | if ext == None: | |
| ext = self.ext | |||
| r892 | |||
|
Miguel Valdez
|
r632 | self.ext = ext.lower() | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.path = path | |
|
Juan C. Espinoza
|
r1112 | ||
|
Miguel Valdez
|
r632 | if set is None: | |
| self.setFile = -1 | |||
| else: | |||
|
Juan C. Espinoza
|
r1112 | self.setFile = set - 1 | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.blocksPerFile = blocksPerFile | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.profilesPerBlock = profilesPerBlock | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.dataOut = dataOut | |
|
Miguel Valdez
|
r632 | self.fileDate = self.dataOut.datatime.date() | |
|
José Chávez
|
r1074 | # By default | |
|
Miguel Valdez
|
r616 | self.dtype = self.dataOut.dtype | |
| r892 | |||
|
Miguel Valdez
|
r616 | if datatype is not None: | |
| self.dtype = get_numpy_dtype(datatype) | |||
| r892 | |||
|
Daniel Valdez
|
r487 | if not(self.setNextFile()): | |
|
George Yong
|
r1167 | print("[Writing] There isn't a next file") | |
|
Daniel Valdez
|
r487 | return 0 | |
| r892 | |||
|
Daniel Valdez
|
r487 | self.setBlockDimension() | |
| r892 | |||
|
Daniel Valdez
|
r487 | return 1 | |
| r892 | |||
|
José Chávez
|
r944 | def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs): | |
| r892 | |||
|
Daniel Valdez
|
r487 | if not(self.isConfig): | |
| r892 | |||
|
José Chávez
|
r1074 | self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, | |
| set=set, ext=ext, datatype=datatype, **kwargs) | |||
|
Daniel Valdez
|
r487 | self.isConfig = True | |
|
George Yong
|
r1167 | self.putData() |
