##// END OF EJS Templates
schain-jc
RSS
Signal Chain GUI...
Signal Chain GUI -Se agregaron nuevos filtros a los graficos de Spectr ay Cross Spectra (y se corrigieron los Bugs) -Se corrigio la seleccion de perfiles en los datos de la USRP -Se validaron la mayoria de las funcionalidades del GUI v2.1
Miguel Valdez - - Load All Authors

File last commit:

r589:341989823444
r600:19c1076f2d8a
Show More
jroIO_base.py
1451 lines | 44.8 KiB | text/x-python | PythonLexer
/ schainpy / model / io / jroIO_base.py
History | Source | Raw |Copy content |Copy permalink
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 '''
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 Created on Jul 2, 2014
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 @author: roj-idl71
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 '''
import os
import sys
import glob
import time
import numpy
import fnmatch
import time, datetime
Daniel Valdez
AMISR modules to read hdf5 files and link to SignalChain Objects...
 r491 #import h5py
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 import traceback
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 try:
from gevent import sleep
except:
from time import sleep
from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 LOCALTIME = True
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 def isNumber(cad):
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 """
Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
Excepciones:
Si un determinado string no puede ser convertido a numero
Input:
str, string al cual se le analiza para determinar si convertible a un numero o no
Return:
True : si el string es uno numerico
False : no es un string numerico
"""
try:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 float( cad )
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 return True
except:
return False
def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
"""
Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
Inputs:
filename : nombre completo del archivo de datos en formato Jicamarca (.r)
startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
segundos contados desde 01/01/1970.
endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
segundos contados desde 01/01/1970.
Return:
Boolean : Retorna True si el archivo de datos contiene datos en el rango de
fecha especificado, de lo contrario retorna False.
Excepciones:
Si el archivo no existe o no puede ser abierto
Si la cabecera no puede ser leida.
"""
basicHeaderObj = BasicHeader(LOCALTIME)
try:
fp = open(filename,'rb')
except IOError:
traceback.print_exc()
raise IOError, "The file %s can't be opened" %(filename)
sts = basicHeaderObj.read(fp)
fp.close()
if not(sts):
print "Skipping the file %s because it has not a valid header" %(filename)
return 0
if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
return 0
return 1
def isFileinThisTime(filename, startTime, endTime):
"""
Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
Inputs:
filename : nombre completo del archivo de datos en formato Jicamarca (.r)
startTime : tiempo inicial del rango seleccionado en formato datetime.time
endTime : tiempo final del rango seleccionado en formato datetime.time
Return:
Boolean : Retorna True si el archivo de datos contiene datos en el rango de
fecha especificado, de lo contrario retorna False.
Excepciones:
Si el archivo no existe o no puede ser abierto
Si la cabecera no puede ser leida.
"""
try:
fp = open(filename,'rb')
except IOError:
traceback.print_exc()
raise IOError, "The file %s can't be opened" %(filename)
basicHeaderObj = BasicHeader(LOCALTIME)
sts = basicHeaderObj.read(fp)
fp.close()
thisDatetime = basicHeaderObj.datatime
thisTime = thisDatetime.time()
if not(sts):
print "Skipping the file %s because it has not a valid header" %(filename)
return None
if not ((startTime <= thisTime) and (endTime > thisTime)):
return None
return thisDatetime
def getFileFromSet(path, ext, set):
validFilelist = []
fileList = os.listdir(path)
# 0 1234 567 89A BCDE
# H YYYY DDD SSS .ext
for thisFile in fileList:
try:
year = int(thisFile[1:5])
doy = int(thisFile[5:8])
except:
continue
if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
continue
validFilelist.append(thisFile)
myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
if len(myfile)!= 0:
return myfile[0]
else:
filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
print 'the filename %s does not exist'%filename
print '...going to the last file: '
if validFilelist:
validFilelist = sorted( validFilelist, key=str.lower )
return validFilelist[-1]
return None
def getlastFileFromPath(path, ext):
"""
Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
al final de la depuracion devuelve el ultimo file de la lista que quedo.
Input:
fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
ext : extension de los files contenidos en una carpeta
Return:
El ultimo file de una determinada carpeta, no se considera el path.
"""
validFilelist = []
fileList = os.listdir(path)
# 0 1234 567 89A BCDE
# H YYYY DDD SSS .ext
for thisFile in fileList:
year = thisFile[1:5]
if not isNumber(year):
continue
doy = thisFile[5:8]
if not isNumber(doy):
continue
year = int(year)
doy = int(doy)
if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
continue
validFilelist.append(thisFile)
if validFilelist:
validFilelist = sorted( validFilelist, key=str.lower )
return validFilelist[-1]
return None
def checkForRealPath(path, foldercounter, year, doy, set, ext):
"""
Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
el path exacto de un determinado file.
Example :
nombre correcto del file es .../.../D2009307/P2009307367.ext
Entonces la funcion prueba con las siguientes combinaciones
.../.../y2009307367.ext
.../.../Y2009307367.ext
.../.../x2009307/y2009307367.ext
.../.../x2009307/Y2009307367.ext
.../.../X2009307/y2009307367.ext
.../.../X2009307/Y2009307367.ext
siendo para este caso, la ultima combinacion de letras, identica al file buscado
Return:
Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
para el filename
"""
fullfilename = None
find_flag = False
filename = None
prefixDirList = [None,'d','D']
if ext.lower() == ".r": #voltage
prefixFileList = ['d','D']
elif ext.lower() == ".pdata": #spectra
prefixFileList = ['p','P']
else:
return None, filename
#barrido por las combinaciones posibles
for prefixDir in prefixDirList:
thispath = path
if prefixDir != None:
#formo el nombre del directorio xYYYYDDD (x=d o x=D)
if foldercounter == 0:
thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
else:
thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
fullfilename = os.path.join( thispath, filename ) #formo el path completo
if os.path.exists( fullfilename ): #verifico que exista
find_flag = True
break
if find_flag:
break
if not(find_flag):
return None, filename
return fullfilename, filename
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 def isRadarFolder(folder):
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 try:
year = int(folder[1:5])
doy = int(folder[5:8])
except:
return 0
return 1
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 def isRadarFile(file):
try:
year = int(file[1:5])
doy = int(file[5:8])
set = int(file[8:11])
except:
return 0
return 1
def getDateFromRadarFile(file):
try:
year = int(file[1:5])
doy = int(file[5:8])
set = int(file[8:11])
except:
return None
thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
return thisDate
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 class JRODataIO:
c = 3E8
isConfig = False
basicHeaderObj = None
systemHeaderObj = None
radarControllerHeaderObj = None
processingHeaderObj = None
online = 0
dtype = None
pathList = []
filenameList = []
filename = None
ext = None
flagIsNewFile = 1
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 flagDiscontinuousBlock = 0
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
flagIsNewBlock = 0
fp = None
firstHeaderSize = 0
basicHeaderSize = 24
versionFile = 1103
fileSize = None
# ippSeconds = None
fileSizeByHeader = None
fileIndex = None
profileIndex = None
blockIndex = None
nTotalBlocks = None
maxTimeStep = 30
lastUTTime = None
datablock = None
dataOut = None
blocksize = None
Miguel Valdez
JRODATA: timeInterval is a property now...
 r527 getByBlock = False
Daniel Valdez
Reading and doing Operation in Blocks to processing radar data from MST_ISR_EEJ Experiment
 r495
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 def __init__(self):
raise ValueError, "Not implemented"
def run(self):
raise ValueError, "Not implemented"
class JRODataReader(JRODataIO):
nReadBlocks = 0
delay = 10 #number of seconds waiting a new file
nTries = 3 #quantity tries
nFiles = 3 #number of files for searching
path = None
foldercounter = 0
flagNoMoreFiles = 0
datetimeList = []
__isFirstTimeOnline = 1
__printInfo = True
profileIndex = None
Miguel Valdez
Affected:...
 r534 nTxs = 1
txIndex = None
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 def __init__(self):
"""
"""
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 # raise NotImplementedError, "This method has not been implemented"
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
def createObjByDefault(self):
"""
"""
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 raise NotImplementedError, "This method has not been implemented"
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
def getBlockDimension(self):
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 raise NotImplementedError, "No implemented"
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
def __searchFilesOffLine(self,
path,
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 startDate=None,
endDate=None,
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 startTime=datetime.time(0,0,0),
endTime=datetime.time(23,59,59),
set=None,
expLabel='',
ext='.r',
walk=True):
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 self.filenameList = []
self.datetimeList = []
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 pathList = []
if not walk:
#pathList.append(path)
multi_path = path.split(',')
for single_path in multi_path:
pathList.append(single_path)
else:
#dirList = []
multi_path = path.split(',')
for single_path in multi_path:
dirList = []
for thisPath in os.listdir(single_path):
if not os.path.isdir(os.path.join(single_path,thisPath)):
continue
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 if not isRadarFolder(thisPath):
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 continue
dirList.append(thisPath)
if not(dirList):
return None, None
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589
if startDate and endDate:
thisDate = startDate
while(thisDate <= endDate):
year = thisDate.timetuple().tm_year
doy = thisDate.timetuple().tm_yday
matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
if len(matchlist) == 0:
thisDate += datetime.timedelta(1)
continue
for match in matchlist:
pathList.append(os.path.join(single_path,match,expLabel))
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 thisDate += datetime.timedelta(1)
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 else:
for thiDir in dirList:
pathList.append(os.path.join(single_path,thiDir,expLabel))
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
if pathList == []:
print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
return None, None
print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
filenameList = []
datetimeList = []
pathDict = {}
filenameList_to_sort = []
for i in range(len(pathList)):
thisPath = pathList[i]
fileList = glob.glob1(thisPath, "*%s" %ext)
Miguel Valdez
Bug plotting RTI fixed: timezone was removed from getLimit function
 r566 if len(fileList) < 1:
continue
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 fileList.sort()
pathDict.setdefault(fileList[0])
pathDict[fileList[0]] = i
filenameList_to_sort.append(fileList[0])
filenameList_to_sort.sort()
for file in filenameList_to_sort:
thisPath = pathList[pathDict[file]]
fileList = glob.glob1(thisPath, "*%s" %ext)
fileList.sort()
for file in fileList:
filename = os.path.join(thisPath,file)
thisDatetime = isFileinThisTime(filename, startTime, endTime)
if not(thisDatetime):
continue
filenameList.append(filename)
datetimeList.append(thisDatetime)
if not(filenameList):
print "Any file was found for the time range %s - %s" %(startTime, endTime)
return None, None
print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
print
for i in range(len(filenameList)):
print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
self.filenameList = filenameList
self.datetimeList = datetimeList
return pathList, filenameList
def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
"""
Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
devuelve el archivo encontrado ademas de otros datos.
Input:
path : carpeta donde estan contenidos los files que contiene data
expLabel : Nombre del subexperimento (subfolder)
ext : extension de los files
walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
Return:
directory : eL directorio donde esta el file encontrado
filename : el ultimo file de una determinada carpeta
year : el anho
doy : el numero de dia del anho
set : el set del archivo
"""
dirList = []
if not walk:
fullpath = path
foldercounter = 0
else:
#Filtra solo los directorios
for thisPath in os.listdir(path):
if not os.path.isdir(os.path.join(path,thisPath)):
continue
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 if not isRadarFolder(thisPath):
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 continue
dirList.append(thisPath)
if not(dirList):
return None, None, None, None, None, None
dirList = sorted( dirList, key=str.lower )
doypath = dirList[-1]
foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
fullpath = os.path.join(path, doypath, expLabel)
print "%s folder was found: " %(fullpath )
if set == None:
filename = getlastFileFromPath(fullpath, ext)
else:
filename = getFileFromSet(fullpath, ext, set)
if not(filename):
return None, None, None, None, None, None
print "%s file was found" %(filename)
if not(self.__verifyFile(os.path.join(fullpath, filename))):
return None, None, None, None, None, None
year = int( filename[1:5] )
doy = int( filename[5:8] )
set = int( filename[8:11] )
return fullpath, foldercounter, filename, year, doy, set
def __setNextFileOffline(self):
idFile = self.fileIndex
while (True):
idFile += 1
if not(idFile < len(self.filenameList)):
self.flagNoMoreFiles = 1
Miguel Valdez
Bug fixed writing pdata files
 r584 # print "[Reading] No more Files"
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 return 0
filename = self.filenameList[idFile]
if not(self.__verifyFile(filename)):
continue
fileSize = os.path.getsize(filename)
fp = open(filename,'rb')
break
self.flagIsNewFile = 1
self.fileIndex = idFile
self.filename = filename
self.fileSize = fileSize
self.fp = fp
Miguel Valdez
Bug fixed writing pdata files
 r584 # print "[Reading] Setting the file: %s"%self.filename
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
return 1
def __setNextFileOnline(self):
"""
Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
siguientes.
Affected:
self.flagIsNewFile
self.filename
self.fileSize
self.fp
self.set
self.flagNoMoreFiles
Return:
0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
1 : si el file fue abierto con exito y esta listo a ser leido
Excepciones:
Si un determinado file no puede ser abierto
"""
nFiles = 0
fileOk_flag = False
firstTime_flag = True
self.set += 1
if self.set > 999:
self.set = 0
self.foldercounter += 1
#busca el 1er file disponible
fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
if fullfilename:
if self.__verifyFile(fullfilename, False):
fileOk_flag = True
#si no encuentra un file entonces espera y vuelve a buscar
if not(fileOk_flag):
for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
tries = self.nTries
else:
tries = 1 #si no es la 1era vez entonces solo lo hace una vez
for nTries in range( tries ):
if firstTime_flag:
Miguel Valdez
Bug fixed writing pdata files
 r584 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 sleep( self.delay )
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 else:
Miguel Valdez
Bug fixed writing pdata files
 r584 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
if fullfilename:
if self.__verifyFile(fullfilename):
fileOk_flag = True
break
if fileOk_flag:
break
firstTime_flag = False
Miguel Valdez
Bug fixed writing pdata files
 r584 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 self.set += 1
if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
self.set = 0
self.doy += 1
self.foldercounter = 0
if fileOk_flag:
self.fileSize = os.path.getsize( fullfilename )
self.filename = fullfilename
self.flagIsNewFile = 1
if self.fp != None: self.fp.close()
self.fp = open(fullfilename, 'rb')
self.flagNoMoreFiles = 0
Miguel Valdez
Bug fixed writing pdata files
 r584 # print '[Reading] Setting the file: %s' % fullfilename
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 else:
self.fileSize = 0
self.filename = None
self.flagIsNewFile = 0
self.fp = None
self.flagNoMoreFiles = 1
Miguel Valdez
Bug fixed writing pdata files
 r584 # print '[Reading] No more files to read'
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
return fileOk_flag
def setNextFile(self):
if self.fp != None:
self.fp.close()
if self.online:
newFile = self.__setNextFileOnline()
else:
newFile = self.__setNextFileOffline()
if not(newFile):
Miguel Valdez
Bug fixed writing pdata files
 r584 print '[Reading] No more files to read'
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 return 0
Miguel Valdez
Bug fixed writing pdata files
 r584 print '[Reading] Setting the file: %s' % self.filename
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 self.__readFirstHeader()
self.nReadBlocks = 0
return 1
def __waitNewBlock(self):
"""
Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
Si el modo de lectura es OffLine siempre retorn 0
"""
if not self.online:
return 0
if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
return 0
currentPointer = self.fp.tell()
neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
for nTries in range( self.nTries ):
self.fp.close()
self.fp = open( self.filename, 'rb' )
self.fp.seek( currentPointer )
self.fileSize = os.path.getsize( self.filename )
currentSize = self.fileSize - currentPointer
if ( currentSize >= neededSize ):
self.basicHeaderObj.read(self.fp)
return 1
if self.fileSize == self.fileSizeByHeader:
# self.flagEoF = True
return 0
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
sleep( self.delay )
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
return 0
def waitDataBlock(self,pointer_location):
currentPointer = pointer_location
neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
for nTries in range( self.nTries ):
self.fp.close()
self.fp = open( self.filename, 'rb' )
self.fp.seek( currentPointer )
self.fileSize = os.path.getsize( self.filename )
currentSize = self.fileSize - currentPointer
if ( currentSize >= neededSize ):
return 1
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
sleep( self.delay )
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
return 0
def __jumpToLastBlock(self):
if not(self.__isFirstTimeOnline):
return
csize = self.fileSize - self.fp.tell()
blocksize = self.processingHeaderObj.blockSize
#salta el primer bloque de datos
if csize > self.processingHeaderObj.blockSize:
self.fp.seek(self.fp.tell() + blocksize)
else:
return
csize = self.fileSize - self.fp.tell()
neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
while True:
if self.fp.tell()<self.fileSize:
self.fp.seek(self.fp.tell() + neededsize)
else:
self.fp.seek(self.fp.tell() - neededsize)
break
# csize = self.fileSize - self.fp.tell()
# neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
# factor = int(csize/neededsize)
# if factor > 0:
# self.fp.seek(self.fp.tell() + factor*neededsize)
self.flagIsNewFile = 0
self.__isFirstTimeOnline = 0
def __setNewBlock(self):
if self.fp == None:
return 0
if self.online:
self.__jumpToLastBlock()
if self.flagIsNewFile:
return 1
self.lastUTTime = self.basicHeaderObj.utc
currentSize = self.fileSize - self.fp.tell()
neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
if (currentSize >= neededSize):
self.basicHeaderObj.read(self.fp)
return 1
if self.__waitNewBlock():
return 1
if not(self.setNextFile()):
return 0
deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 self.flagDiscontinuousBlock = 0
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
if deltaTime > self.maxTimeStep:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 self.flagDiscontinuousBlock = 1
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
return 1
def readNextBlock(self):
if not(self.__setNewBlock()):
return 0
if not(self.readBlock()):
return 0
return 1
def __readFirstHeader(self):
self.basicHeaderObj.read(self.fp)
self.systemHeaderObj.read(self.fp)
self.radarControllerHeaderObj.read(self.fp)
self.processingHeaderObj.read(self.fp)
self.firstHeaderSize = self.basicHeaderObj.size
datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
if datatype == 0:
datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
elif datatype == 1:
datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
elif datatype == 2:
datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
elif datatype == 3:
datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
elif datatype == 4:
datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
elif datatype == 5:
datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
else:
raise ValueError, 'Data type was not defined'
self.dtype = datatype_str
#self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
# self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
# self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
self.getBlockDimension()
def __verifyFile(self, filename, msgFlag=True):
msg = None
try:
fp = open(filename, 'rb')
currentPosition = fp.tell()
except IOError:
traceback.print_exc()
if msgFlag:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] The file %s can't be opened" % (filename)
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 return False
neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
if neededSize == 0:
basicHeaderObj = BasicHeader(LOCALTIME)
systemHeaderObj = SystemHeader()
radarControllerHeaderObj = RadarControllerHeader()
processingHeaderObj = ProcessingHeader()
try:
if not( basicHeaderObj.read(fp) ): raise IOError
if not( systemHeaderObj.read(fp) ): raise IOError
if not( radarControllerHeaderObj.read(fp) ): raise IOError
if not( processingHeaderObj.read(fp) ): raise IOError
# data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
except IOError:
traceback.print_exc()
if msgFlag:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] The file %s is empty or it hasn't enough data" % filename
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
fp.close()
return False
else:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
fp.close()
fileSize = os.path.getsize(filename)
currentSize = fileSize - currentPosition
if currentSize < neededSize:
if msgFlag and (msg != None):
print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
return False
return True
Miguel Valdez
-USRP Reader was added to Signal Chain GUI...
 r589 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
dateList = []
pathList = []
if not walk:
#pathList.append(path)
multi_path = path.split(',')
for single_path in multi_path:
ok = False
fileList = glob.glob1(single_path, "*"+ext)
for thisFile in fileList:
if not os.path.isfile(os.path.join(single_path, thisFile)):
continue
if not isRadarFile(thisFile):
continue
ok = True
thisDate = getDateFromRadarFile(thisFile)
if thisDate not in dateList:
dateList.append(thisDate)
if ok:
pathList.append(single_path)
return dateList
multi_path = path.split(',')
for single_path in multi_path:
dirList = []
for thisPath in os.listdir(single_path):
if not os.path.isdir(os.path.join(single_path,thisPath)):
continue
if not isRadarFolder(thisPath):
continue
dirList.append(thisPath)
if not dirList:
return dateList
if startDate and endDate:
thisDate = startDate
while(thisDate <= endDate):
year = thisDate.timetuple().tm_year
doy = thisDate.timetuple().tm_yday
matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
if len(matchlist) == 0:
thisDate += datetime.timedelta(1)
continue
for match in matchlist:
pathList.append(os.path.join(single_path,match,expLabel))
dateList.append(thisDate)
thisDate += datetime.timedelta(1)
else:
for thiDir in dirList:
year = int(folder[1:5])
doy = int(folder[5:8])
thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
pathList.append(os.path.join(single_path,thiDir,expLabel))
dateList.append(thisDate)
return dateList
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 def setup(self,
path=None,
startDate=None,
endDate=None,
startTime=datetime.time(0,0,0),
endTime=datetime.time(23,59,59),
set=None,
expLabel = "",
ext = None,
online = False,
delay = 60,
Daniel Valdez
Reading and doing Operation in Blocks to processing radar data from MST_ISR_EEJ Experiment
 r495 walk = True,
Miguel Valdez
Affected:...
 r534 getblock = False,
nTxs = 1):
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
if path == None:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 raise ValueError, "[Reading] The path is not valid"
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
if ext == None:
ext = self.ext
if online:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] Searching files in online mode..."
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
for nTries in range( self.nTries ):
fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
if fullpath:
break
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
sleep( self.delay )
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
if not(fullpath):
Miguel Valdez
A new SendToServer Unit has been created to upload files to a remote server....
 r573 print "[Reading] There 'isn't any valid file in %s" % path
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 return None
self.year = year
self.doy = doy
self.set = set - 1
self.path = path
self.foldercounter = foldercounter
last_set = None
else:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] Searching files in offline mode ..."
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
startTime=startTime, endTime=endTime,
set=set, expLabel=expLabel, ext=ext,
walk=walk)
if not(pathList):
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 datetime.datetime.combine(startDate,startTime).ctime(),
datetime.datetime.combine(endDate,endTime).ctime())
sys.exit(-1)
self.fileIndex = -1
self.pathList = pathList
self.filenameList = filenameList
file_name = os.path.basename(filenameList[-1])
basename, ext = os.path.splitext(file_name)
last_set = int(basename[-3:])
self.online = online
self.delay = delay
ext = ext.lower()
self.ext = ext
Miguel Valdez
JRODATA: timeInterval is a property now...
 r527 self.getByBlock = getblock
Miguel Valdez
Affected:...
 r534 self.nTxs = int(nTxs)
Miguel Valdez
JRODATA: timeInterval is a property now...
 r527
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 if not(self.setNextFile()):
if (startDate!=None) and (endDate!=None):
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 elif startDate != None:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 else:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] No files"
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
sys.exit(-1)
# self.updateDataHeader()
if last_set != None:
self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
return
def getBasicHeader(self):
self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
self.dataOut.timeZone = self.basicHeaderObj.timeZone
self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
self.dataOut.errorCount = self.basicHeaderObj.errorCount
self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
Alexander Valdez
r563 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 def getFirstHeader(self):
raise ValueError, "This method has not been implemented"
def getData(self):
raise ValueError, "This method has not been implemented"
def hasNotDataInBuffer(self):
raise ValueError, "This method has not been implemented"
def readBlock(self):
raise ValueError, "This method has not been implemented"
def isEndProcess(self):
return self.flagNoMoreFiles
def printReadBlocks(self):
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
def printTotalBlocks(self):
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
def printNumberOfBlock(self):
if self.flagIsNewBlock:
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
def printInfo(self):
if self.__printInfo == False:
return
self.basicHeaderObj.printInfo()
self.systemHeaderObj.printInfo()
self.radarControllerHeaderObj.printInfo()
self.processingHeaderObj.printInfo()
self.__printInfo = False
def run(self, **kwargs):
if not(self.isConfig):
# self.dataOut = dataOut
self.setup(**kwargs)
self.isConfig = True
self.getData()
class JRODataWriter(JRODataIO):
"""
Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
de los datos siempre se realiza por bloques.
"""
blockIndex = 0
path = None
setFile = None
profilesPerBlock = None
blocksPerFile = None
nWriteBlocks = 0
def __init__(self, dataOut=None):
raise ValueError, "Not implemented"
def hasAllDataInBuffer(self):
raise ValueError, "Not implemented"
def setBlockDimension(self):
raise ValueError, "Not implemented"
def writeBlock(self):
raise ValueError, "No implemented"
def putData(self):
raise ValueError, "No implemented"
def setBasicHeader(self):
self.basicHeaderObj.size = self.basicHeaderSize #bytes
self.basicHeaderObj.version = self.versionFile
self.basicHeaderObj.dataBlock = self.nTotalBlocks
utc = numpy.floor(self.dataOut.utctime)
milisecond = (self.dataOut.utctime - utc)* 1000.0
self.basicHeaderObj.utc = utc
self.basicHeaderObj.miliSecond = milisecond
self.basicHeaderObj.timeZone = self.dataOut.timeZone
self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
self.basicHeaderObj.errorCount = self.dataOut.errorCount
def setFirstHeader(self):
"""
Obtiene una copia del First Header
Affected:
self.basicHeaderObj
self.systemHeaderObj
self.radarControllerHeaderObj
self.processingHeaderObj self.
Return:
None
"""
raise ValueError, "No implemented"
def __writeFirstHeader(self):
"""
Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
Affected:
__dataType
Return:
None
"""
# CALCULAR PARAMETROS
sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
self.basicHeaderObj.write(self.fp)
self.systemHeaderObj.write(self.fp)
self.radarControllerHeaderObj.write(self.fp)
self.processingHeaderObj.write(self.fp)
self.dtype = self.dataOut.dtype
def __setNewBlock(self):
"""
Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
Return:
0 : si no pudo escribir nada
1 : Si escribio el Basic el First Header
"""
if self.fp == None:
self.setNextFile()
if self.flagIsNewFile:
return 1
if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
self.basicHeaderObj.write(self.fp)
return 1
if not( self.setNextFile() ):
return 0
return 1
def writeNextBlock(self):
"""
Selecciona el bloque siguiente de datos y los escribe en un file
Return:
0 : Si no hizo pudo escribir el bloque de datos
1 : Si no pudo escribir el bloque de datos
"""
if not( self.__setNewBlock() ):
return 0
self.writeBlock()
return 1
def setNextFile(self):
"""
Determina el siguiente file que sera escrito
Affected:
self.filename
self.subfolder
self.fp
self.setFile
self.flagIsNewFile
Return:
0 : Si el archivo no puede ser escrito
1 : Si el archivo esta listo para ser escrito
"""
ext = self.ext
path = self.path
if self.fp != None:
self.fp.close()
timeTuple = time.localtime( self.dataOut.utctime)
subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
fullpath = os.path.join( path, subfolder )
if not( os.path.exists(fullpath) ):
os.mkdir(fullpath)
self.setFile = -1 #inicializo mi contador de seteo
else:
filesList = os.listdir( fullpath )
if len( filesList ) > 0:
filesList = sorted( filesList, key=str.lower )
filen = filesList[-1]
# el filename debera tener el siguiente formato
# 0 1234 567 89A BCDE (hex)
# x YYYY DDD SSS .ext
if isNumber( filen[8:11] ):
self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
else:
self.setFile = -1
else:
self.setFile = -1 #inicializo mi contador de seteo
setFile = self.setFile
setFile += 1
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 timeTuple.tm_year,
timeTuple.tm_yday,
setFile,
ext )
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 filename = os.path.join( path, subfolder, filen )
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
fp = open( filename,'wb' )
self.blockIndex = 0
#guardando atributos
self.filename = filename
self.subfolder = subfolder
self.fp = fp
self.setFile = setFile
self.flagIsNewFile = 1
self.setFirstHeader()
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print '[Writing] file: %s'%self.filename
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487
self.__writeFirstHeader()
return 1
def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
"""
Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
Inputs:
path : el path destino en el cual se escribiran los files a crear
format : formato en el cual sera salvado un file
set : el setebo del file
Return:
0 : Si no realizo un buen seteo
1 : Si realizo un buen seteo
"""
if ext == None:
ext = self.ext
ext = ext.lower()
self.ext = ext
self.path = path
self.setFile = set - 1
self.blocksPerFile = blocksPerFile
self.profilesPerBlock = profilesPerBlock
self.dataOut = dataOut
if not(self.setNextFile()):
Miguel Valdez
Merge with branch schain_julia_drifts from rev. 803 to 995....
 r568 print "[Writing] There isn't a next file"
Daniel Valdez
This is the new organization by packages and scripts for Signal Chain, this version contains new features and bugs fixed until August 2014
 r487 return 0
self.setBlockDimension()
return 1
def run(self, dataOut, **kwargs):
if not(self.isConfig):
self.setup(dataOut, **kwargs)
self.isConfig = True
self.putData()