##// END OF EJS Templates
schain
RSS
New reading option allows to read by blocks of different sizes
Julio Valdez -
r833:26a5421c91c8
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1709 +1,1723
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 print "The file %s can't be opened" %(filename)
71 print "The file %s can't be opened" %(filename)
72 return 0
72 return 0
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isTimeInRange(thisTime, startTime, endTime):
86 def isTimeInRange(thisTime, startTime, endTime):
87
87
88 if endTime >= startTime:
88 if endTime >= startTime:
89 if (thisTime < startTime) or (thisTime > endTime):
89 if (thisTime < startTime) or (thisTime > endTime):
90 return 0
90 return 0
91
91
92 return 1
92 return 1
93 else:
93 else:
94 if (thisTime < startTime) and (thisTime > endTime):
94 if (thisTime < startTime) and (thisTime > endTime):
95 return 0
95 return 0
96
96
97 return 1
97 return 1
98
98
99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 """
100 """
101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102
102
103 Inputs:
103 Inputs:
104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105
105
106 startDate : fecha inicial del rango seleccionado en formato datetime.date
106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107
107
108 endDate : fecha final del rango seleccionado en formato datetime.date
108 endDate : fecha final del rango seleccionado en formato datetime.date
109
109
110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111
111
112 endTime : tiempo final del rango seleccionado en formato datetime.time
112 endTime : tiempo final del rango seleccionado en formato datetime.time
113
113
114 Return:
114 Return:
115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 fecha especificado, de lo contrario retorna False.
116 fecha especificado, de lo contrario retorna False.
117
117
118 Excepciones:
118 Excepciones:
119 Si el archivo no existe o no puede ser abierto
119 Si el archivo no existe o no puede ser abierto
120 Si la cabecera no puede ser leida.
120 Si la cabecera no puede ser leida.
121
121
122 """
122 """
123
123
124
124
125 try:
125 try:
126 fp = open(filename,'rb')
126 fp = open(filename,'rb')
127 except IOError:
127 except IOError:
128 print "The file %s can't be opened" %(filename)
128 print "The file %s can't be opened" %(filename)
129 return None
129 return None
130
130
131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 systemHeaderObj = SystemHeader()
132 systemHeaderObj = SystemHeader()
133 radarControllerHeaderObj = RadarControllerHeader()
133 radarControllerHeaderObj = RadarControllerHeader()
134 processingHeaderObj = ProcessingHeader()
134 processingHeaderObj = ProcessingHeader()
135
135
136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137
137
138 sts = firstBasicHeaderObj.read(fp)
138 sts = firstBasicHeaderObj.read(fp)
139
139
140 if not(sts):
140 if not(sts):
141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 return None
142 return None
143
143
144 if not systemHeaderObj.read(fp):
144 if not systemHeaderObj.read(fp):
145 return None
145 return None
146
146
147 if not radarControllerHeaderObj.read(fp):
147 if not radarControllerHeaderObj.read(fp):
148 return None
148 return None
149
149
150 if not processingHeaderObj.read(fp):
150 if not processingHeaderObj.read(fp):
151 return None
151 return None
152
152
153 filesize = os.path.getsize(filename)
153 filesize = os.path.getsize(filename)
154
154
155 offset = processingHeaderObj.blockSize + 24 #header size
155 offset = processingHeaderObj.blockSize + 24 #header size
156
156
157 if filesize <= offset:
157 if filesize <= offset:
158 print "[Reading] %s: This file has not enough data" %filename
158 print "[Reading] %s: This file has not enough data" %filename
159 return None
159 return None
160
160
161 fp.seek(-offset, 2)
161 fp.seek(-offset, 2)
162
162
163 sts = lastBasicHeaderObj.read(fp)
163 sts = lastBasicHeaderObj.read(fp)
164
164
165 fp.close()
165 fp.close()
166
166
167 thisDatetime = lastBasicHeaderObj.datatime
167 thisDatetime = lastBasicHeaderObj.datatime
168 thisTime_last_block = thisDatetime.time()
168 thisTime_last_block = thisDatetime.time()
169
169
170 thisDatetime = firstBasicHeaderObj.datatime
170 thisDatetime = firstBasicHeaderObj.datatime
171 thisDate = thisDatetime.date()
171 thisDate = thisDatetime.date()
172 thisTime_first_block = thisDatetime.time()
172 thisTime_first_block = thisDatetime.time()
173
173
174 #General case
174 #General case
175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 #-----------o----------------------------o-----------
176 #-----------o----------------------------o-----------
177 # startTime endTime
177 # startTime endTime
178
178
179 if endTime >= startTime:
179 if endTime >= startTime:
180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 return None
181 return None
182
182
183 return thisDatetime
183 return thisDatetime
184
184
185 #If endTime < startTime then endTime belongs to the next day
185 #If endTime < startTime then endTime belongs to the next day
186
186
187
187
188 #<<<<<<<<<<<o o>>>>>>>>>>>
188 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #-----------o----------------------------o-----------
189 #-----------o----------------------------o-----------
190 # endTime startTime
190 # endTime startTime
191
191
192 if (thisDate == startDate) and (thisTime_last_block < startTime):
192 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 return None
193 return None
194
194
195 if (thisDate == endDate) and (thisTime_first_block > endTime):
195 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 return None
196 return None
197
197
198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 return None
199 return None
200
200
201 return thisDatetime
201 return thisDatetime
202
202
203 def isFolderInDateRange(folder, startDate=None, endDate=None):
203 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 """
204 """
205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206
206
207 Inputs:
207 Inputs:
208 folder : nombre completo del directorio.
208 folder : nombre completo del directorio.
209 Su formato deberia ser "/path_root/?YYYYDDD"
209 Su formato deberia ser "/path_root/?YYYYDDD"
210
210
211 siendo:
211 siendo:
212 YYYY : Anio (ejemplo 2015)
212 YYYY : Anio (ejemplo 2015)
213 DDD : Dia del anio (ejemplo 305)
213 DDD : Dia del anio (ejemplo 305)
214
214
215 startDate : fecha inicial del rango seleccionado en formato datetime.date
215 startDate : fecha inicial del rango seleccionado en formato datetime.date
216
216
217 endDate : fecha final del rango seleccionado en formato datetime.date
217 endDate : fecha final del rango seleccionado en formato datetime.date
218
218
219 Return:
219 Return:
220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 fecha especificado, de lo contrario retorna False.
221 fecha especificado, de lo contrario retorna False.
222 Excepciones:
222 Excepciones:
223 Si el directorio no tiene el formato adecuado
223 Si el directorio no tiene el formato adecuado
224 """
224 """
225
225
226 basename = os.path.basename(folder)
226 basename = os.path.basename(folder)
227
227
228 if not isRadarFolder(basename):
228 if not isRadarFolder(basename):
229 print "The folder %s has not the rigth format" %folder
229 print "The folder %s has not the rigth format" %folder
230 return 0
230 return 0
231
231
232 if startDate and endDate:
232 if startDate and endDate:
233 thisDate = getDateFromRadarFolder(basename)
233 thisDate = getDateFromRadarFolder(basename)
234
234
235 if thisDate < startDate:
235 if thisDate < startDate:
236 return 0
236 return 0
237
237
238 if thisDate > endDate:
238 if thisDate > endDate:
239 return 0
239 return 0
240
240
241 return 1
241 return 1
242
242
243 def isFileInDateRange(filename, startDate=None, endDate=None):
243 def isFileInDateRange(filename, startDate=None, endDate=None):
244 """
244 """
245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246
246
247 Inputs:
247 Inputs:
248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249
249
250 Su formato deberia ser "?YYYYDDDsss"
250 Su formato deberia ser "?YYYYDDDsss"
251
251
252 siendo:
252 siendo:
253 YYYY : Anio (ejemplo 2015)
253 YYYY : Anio (ejemplo 2015)
254 DDD : Dia del anio (ejemplo 305)
254 DDD : Dia del anio (ejemplo 305)
255 sss : set
255 sss : set
256
256
257 startDate : fecha inicial del rango seleccionado en formato datetime.date
257 startDate : fecha inicial del rango seleccionado en formato datetime.date
258
258
259 endDate : fecha final del rango seleccionado en formato datetime.date
259 endDate : fecha final del rango seleccionado en formato datetime.date
260
260
261 Return:
261 Return:
262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 fecha especificado, de lo contrario retorna False.
263 fecha especificado, de lo contrario retorna False.
264 Excepciones:
264 Excepciones:
265 Si el archivo no tiene el formato adecuado
265 Si el archivo no tiene el formato adecuado
266 """
266 """
267
267
268 basename = os.path.basename(filename)
268 basename = os.path.basename(filename)
269
269
270 if not isRadarFile(basename):
270 if not isRadarFile(basename):
271 print "The filename %s has not the rigth format" %filename
271 print "The filename %s has not the rigth format" %filename
272 return 0
272 return 0
273
273
274 if startDate and endDate:
274 if startDate and endDate:
275 thisDate = getDateFromRadarFile(basename)
275 thisDate = getDateFromRadarFile(basename)
276
276
277 if thisDate < startDate:
277 if thisDate < startDate:
278 return 0
278 return 0
279
279
280 if thisDate > endDate:
280 if thisDate > endDate:
281 return 0
281 return 0
282
282
283 return 1
283 return 1
284
284
285 def getFileFromSet(path, ext, set):
285 def getFileFromSet(path, ext, set):
286 validFilelist = []
286 validFilelist = []
287 fileList = os.listdir(path)
287 fileList = os.listdir(path)
288
288
289 # 0 1234 567 89A BCDE
289 # 0 1234 567 89A BCDE
290 # H YYYY DDD SSS .ext
290 # H YYYY DDD SSS .ext
291
291
292 for thisFile in fileList:
292 for thisFile in fileList:
293 try:
293 try:
294 year = int(thisFile[1:5])
294 year = int(thisFile[1:5])
295 doy = int(thisFile[5:8])
295 doy = int(thisFile[5:8])
296 except:
296 except:
297 continue
297 continue
298
298
299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 continue
300 continue
301
301
302 validFilelist.append(thisFile)
302 validFilelist.append(thisFile)
303
303
304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305
305
306 if len(myfile)!= 0:
306 if len(myfile)!= 0:
307 return myfile[0]
307 return myfile[0]
308 else:
308 else:
309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 print 'the filename %s does not exist'%filename
310 print 'the filename %s does not exist'%filename
311 print '...going to the last file: '
311 print '...going to the last file: '
312
312
313 if validFilelist:
313 if validFilelist:
314 validFilelist = sorted( validFilelist, key=str.lower )
314 validFilelist = sorted( validFilelist, key=str.lower )
315 return validFilelist[-1]
315 return validFilelist[-1]
316
316
317 return None
317 return None
318
318
319 def getlastFileFromPath(path, ext):
319 def getlastFileFromPath(path, ext):
320 """
320 """
321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323
323
324 Input:
324 Input:
325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 ext : extension de los files contenidos en una carpeta
326 ext : extension de los files contenidos en una carpeta
327
327
328 Return:
328 Return:
329 El ultimo file de una determinada carpeta, no se considera el path.
329 El ultimo file de una determinada carpeta, no se considera el path.
330 """
330 """
331 validFilelist = []
331 validFilelist = []
332 fileList = os.listdir(path)
332 fileList = os.listdir(path)
333
333
334 # 0 1234 567 89A BCDE
334 # 0 1234 567 89A BCDE
335 # H YYYY DDD SSS .ext
335 # H YYYY DDD SSS .ext
336
336
337 for thisFile in fileList:
337 for thisFile in fileList:
338
338
339 year = thisFile[1:5]
339 year = thisFile[1:5]
340 if not isNumber(year):
340 if not isNumber(year):
341 continue
341 continue
342
342
343 doy = thisFile[5:8]
343 doy = thisFile[5:8]
344 if not isNumber(doy):
344 if not isNumber(doy):
345 continue
345 continue
346
346
347 year = int(year)
347 year = int(year)
348 doy = int(doy)
348 doy = int(doy)
349
349
350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 continue
351 continue
352
352
353 validFilelist.append(thisFile)
353 validFilelist.append(thisFile)
354
354
355 if validFilelist:
355 if validFilelist:
356 validFilelist = sorted( validFilelist, key=str.lower )
356 validFilelist = sorted( validFilelist, key=str.lower )
357 return validFilelist[-1]
357 return validFilelist[-1]
358
358
359 return None
359 return None
360
360
361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 """
362 """
363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 el path exacto de un determinado file.
365 el path exacto de un determinado file.
366
366
367 Example :
367 Example :
368 nombre correcto del file es .../.../D2009307/P2009307367.ext
368 nombre correcto del file es .../.../D2009307/P2009307367.ext
369
369
370 Entonces la funcion prueba con las siguientes combinaciones
370 Entonces la funcion prueba con las siguientes combinaciones
371 .../.../y2009307367.ext
371 .../.../y2009307367.ext
372 .../.../Y2009307367.ext
372 .../.../Y2009307367.ext
373 .../.../x2009307/y2009307367.ext
373 .../.../x2009307/y2009307367.ext
374 .../.../x2009307/Y2009307367.ext
374 .../.../x2009307/Y2009307367.ext
375 .../.../X2009307/y2009307367.ext
375 .../.../X2009307/y2009307367.ext
376 .../.../X2009307/Y2009307367.ext
376 .../.../X2009307/Y2009307367.ext
377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378
378
379 Return:
379 Return:
380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 para el filename
382 para el filename
383 """
383 """
384 fullfilename = None
384 fullfilename = None
385 find_flag = False
385 find_flag = False
386 filename = None
386 filename = None
387
387
388 prefixDirList = [None,'d','D']
388 prefixDirList = [None,'d','D']
389 if ext.lower() == ".r": #voltage
389 if ext.lower() == ".r": #voltage
390 prefixFileList = ['d','D']
390 prefixFileList = ['d','D']
391 elif ext.lower() == ".pdata": #spectra
391 elif ext.lower() == ".pdata": #spectra
392 prefixFileList = ['p','P']
392 prefixFileList = ['p','P']
393 else:
393 else:
394 return None, filename
394 return None, filename
395
395
396 #barrido por las combinaciones posibles
396 #barrido por las combinaciones posibles
397 for prefixDir in prefixDirList:
397 for prefixDir in prefixDirList:
398 thispath = path
398 thispath = path
399 if prefixDir != None:
399 if prefixDir != None:
400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 if foldercounter == 0:
401 if foldercounter == 0:
402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 else:
403 else:
404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408
408
409 if os.path.exists( fullfilename ): #verifico que exista
409 if os.path.exists( fullfilename ): #verifico que exista
410 find_flag = True
410 find_flag = True
411 break
411 break
412 if find_flag:
412 if find_flag:
413 break
413 break
414
414
415 if not(find_flag):
415 if not(find_flag):
416 return None, filename
416 return None, filename
417
417
418 return fullfilename, filename
418 return fullfilename, filename
419
419
420 def isRadarFolder(folder):
420 def isRadarFolder(folder):
421 try:
421 try:
422 year = int(folder[1:5])
422 year = int(folder[1:5])
423 doy = int(folder[5:8])
423 doy = int(folder[5:8])
424 except:
424 except:
425 return 0
425 return 0
426
426
427 return 1
427 return 1
428
428
429 def isRadarFile(file):
429 def isRadarFile(file):
430 try:
430 try:
431 year = int(file[1:5])
431 year = int(file[1:5])
432 doy = int(file[5:8])
432 doy = int(file[5:8])
433 set = int(file[8:11])
433 set = int(file[8:11])
434 except:
434 except:
435 return 0
435 return 0
436
436
437 return 1
437 return 1
438
438
439 def getDateFromRadarFile(file):
439 def getDateFromRadarFile(file):
440 try:
440 try:
441 year = int(file[1:5])
441 year = int(file[1:5])
442 doy = int(file[5:8])
442 doy = int(file[5:8])
443 set = int(file[8:11])
443 set = int(file[8:11])
444 except:
444 except:
445 return None
445 return None
446
446
447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 return thisDate
448 return thisDate
449
449
450 def getDateFromRadarFolder(folder):
450 def getDateFromRadarFolder(folder):
451 try:
451 try:
452 year = int(folder[1:5])
452 year = int(folder[1:5])
453 doy = int(folder[5:8])
453 doy = int(folder[5:8])
454 except:
454 except:
455 return None
455 return None
456
456
457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 return thisDate
458 return thisDate
459
459
460 class JRODataIO:
460 class JRODataIO:
461
461
462 c = 3E8
462 c = 3E8
463
463
464 isConfig = False
464 isConfig = False
465
465
466 basicHeaderObj = None
466 basicHeaderObj = None
467
467
468 systemHeaderObj = None
468 systemHeaderObj = None
469
469
470 radarControllerHeaderObj = None
470 radarControllerHeaderObj = None
471
471
472 processingHeaderObj = None
472 processingHeaderObj = None
473
473
474 dtype = None
474 dtype = None
475
475
476 pathList = []
476 pathList = []
477
477
478 filenameList = []
478 filenameList = []
479
479
480 filename = None
480 filename = None
481
481
482 ext = None
482 ext = None
483
483
484 flagIsNewFile = 1
484 flagIsNewFile = 1
485
485
486 flagDiscontinuousBlock = 0
486 flagDiscontinuousBlock = 0
487
487
488 flagIsNewBlock = 0
488 flagIsNewBlock = 0
489
489
490 fp = None
490 fp = None
491
491
492 firstHeaderSize = 0
492 firstHeaderSize = 0
493
493
494 basicHeaderSize = 24
494 basicHeaderSize = 24
495
495
496 versionFile = 1103
496 versionFile = 1103
497
497
498 fileSize = None
498 fileSize = None
499
499
500 # ippSeconds = None
500 # ippSeconds = None
501
501
502 fileSizeByHeader = None
502 fileSizeByHeader = None
503
503
504 fileIndex = None
504 fileIndex = None
505
505
506 profileIndex = None
506 profileIndex = None
507
507
508 blockIndex = None
508 blockIndex = None
509
509
510 nTotalBlocks = None
510 nTotalBlocks = None
511
511
512 maxTimeStep = 30
512 maxTimeStep = 30
513
513
514 lastUTTime = None
514 lastUTTime = None
515
515
516 datablock = None
516 datablock = None
517
517
518 dataOut = None
518 dataOut = None
519
519
520 blocksize = None
520 blocksize = None
521
521
522 getByBlock = False
522 getByBlock = False
523
523
524 def __init__(self):
524 def __init__(self):
525
525
526 raise NotImplementedError
526 raise NotImplementedError
527
527
528 def run(self):
528 def run(self):
529
529
530 raise NotImplementedError
530 raise NotImplementedError
531
531
532 def getDtypeWidth(self):
532 def getDtypeWidth(self):
533
533
534 dtype_index = get_dtype_index(self.dtype)
534 dtype_index = get_dtype_index(self.dtype)
535 dtype_width = get_dtype_width(dtype_index)
535 dtype_width = get_dtype_width(dtype_index)
536
536
537 return dtype_width
537 return dtype_width
538
538
539 class JRODataReader(JRODataIO):
539 class JRODataReader(JRODataIO):
540
540
541
541
542 online = 0
542 online = 0
543
543
544 realtime = 0
544 realtime = 0
545
545
546 nReadBlocks = 0
546 nReadBlocks = 0
547
547
548 delay = 10 #number of seconds waiting a new file
548 delay = 10 #number of seconds waiting a new file
549
549
550 nTries = 3 #quantity tries
550 nTries = 3 #quantity tries
551
551
552 nFiles = 3 #number of files for searching
552 nFiles = 3 #number of files for searching
553
553
554 path = None
554 path = None
555
555
556 foldercounter = 0
556 foldercounter = 0
557
557
558 flagNoMoreFiles = 0
558 flagNoMoreFiles = 0
559
559
560 datetimeList = []
560 datetimeList = []
561
561
562 __isFirstTimeOnline = 1
562 __isFirstTimeOnline = 1
563
563
564 __printInfo = True
564 __printInfo = True
565
565
566 profileIndex = None
566 profileIndex = None
567
567
568 nTxs = 1
568 nTxs = 1
569
569
570 txIndex = None
570 txIndex = None
571
571
572 #Added--------------------
573
574 selBlocksize = None
575
576 selBlocktime = None
577
578
572 def __init__(self):
579 def __init__(self):
573
580
574 """
581 """
575 This class is used to find data files
582 This class is used to find data files
576
583
577 Example:
584 Example:
578 reader = JRODataReader()
585 reader = JRODataReader()
579 fileList = reader.findDataFiles()
586 fileList = reader.findDataFiles()
580
587
581 """
588 """
582 pass
589 pass
583
590
584
591
585 def createObjByDefault(self):
592 def createObjByDefault(self):
586 """
593 """
587
594
588 """
595 """
589 raise NotImplementedError
596 raise NotImplementedError
590
597
591 def getBlockDimension(self):
598 def getBlockDimension(self):
592
599
593 raise NotImplementedError
600 raise NotImplementedError
594
601
595 def __searchFilesOffLine(self,
602 def __searchFilesOffLine(self,
596 path,
603 path,
597 startDate=None,
604 startDate=None,
598 endDate=None,
605 endDate=None,
599 startTime=datetime.time(0,0,0),
606 startTime=datetime.time(0,0,0),
600 endTime=datetime.time(23,59,59),
607 endTime=datetime.time(23,59,59),
601 set=None,
608 set=None,
602 expLabel='',
609 expLabel='',
603 ext='.r',
610 ext='.r',
604 walk=True):
611 walk=True):
605
612
606 self.filenameList = []
613 self.filenameList = []
607 self.datetimeList = []
614 self.datetimeList = []
608
615
609 pathList = []
616 pathList = []
610
617
611 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
618 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
612
619
613 if dateList == []:
620 if dateList == []:
614 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
621 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
615 return None, None
622 return None, None
616
623
617 if len(dateList) > 1:
624 if len(dateList) > 1:
618 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
625 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
619 else:
626 else:
620 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
627 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
621
628
622 filenameList = []
629 filenameList = []
623 datetimeList = []
630 datetimeList = []
624
631
625 for thisPath in pathList:
632 for thisPath in pathList:
626 # thisPath = pathList[pathDict[file]]
633 # thisPath = pathList[pathDict[file]]
627
634
628 fileList = glob.glob1(thisPath, "*%s" %ext)
635 fileList = glob.glob1(thisPath, "*%s" %ext)
629 fileList.sort()
636 fileList.sort()
630
637
631 for file in fileList:
638 for file in fileList:
632
639
633 filename = os.path.join(thisPath,file)
640 filename = os.path.join(thisPath,file)
634
641
635 if not isFileInDateRange(filename, startDate, endDate):
642 if not isFileInDateRange(filename, startDate, endDate):
636 continue
643 continue
637
644
638 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
645 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
639
646
640 if not(thisDatetime):
647 if not(thisDatetime):
641 continue
648 continue
642
649
643 filenameList.append(filename)
650 filenameList.append(filename)
644 datetimeList.append(thisDatetime)
651 datetimeList.append(thisDatetime)
645
652
646 if not(filenameList):
653 if not(filenameList):
647 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
654 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
648 return None, None
655 return None, None
649
656
650 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
657 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
651 print
658 print
652
659
653 for i in range(len(filenameList)):
660 for i in range(len(filenameList)):
654 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
661 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
655
662
656 self.filenameList = filenameList
663 self.filenameList = filenameList
657 self.datetimeList = datetimeList
664 self.datetimeList = datetimeList
658
665
659 return pathList, filenameList
666 return pathList, filenameList
660
667
661 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
668 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
662
669
663 """
670 """
664 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
671 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
665 devuelve el archivo encontrado ademas de otros datos.
672 devuelve el archivo encontrado ademas de otros datos.
666
673
667 Input:
674 Input:
668 path : carpeta donde estan contenidos los files que contiene data
675 path : carpeta donde estan contenidos los files que contiene data
669
676
670 expLabel : Nombre del subexperimento (subfolder)
677 expLabel : Nombre del subexperimento (subfolder)
671
678
672 ext : extension de los files
679 ext : extension de los files
673
680
674 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
681 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
675
682
676 Return:
683 Return:
677 directory : eL directorio donde esta el file encontrado
684 directory : eL directorio donde esta el file encontrado
678 filename : el ultimo file de una determinada carpeta
685 filename : el ultimo file de una determinada carpeta
679 year : el anho
686 year : el anho
680 doy : el numero de dia del anho
687 doy : el numero de dia del anho
681 set : el set del archivo
688 set : el set del archivo
682
689
683
690
684 """
691 """
685 if not os.path.isdir(path):
692 if not os.path.isdir(path):
686 return None, None, None, None, None, None
693 return None, None, None, None, None, None
687
694
688 dirList = []
695 dirList = []
689
696
690 if not walk:
697 if not walk:
691 fullpath = path
698 fullpath = path
692 foldercounter = 0
699 foldercounter = 0
693 else:
700 else:
694 #Filtra solo los directorios
701 #Filtra solo los directorios
695 for thisPath in os.listdir(path):
702 for thisPath in os.listdir(path):
696 if not os.path.isdir(os.path.join(path,thisPath)):
703 if not os.path.isdir(os.path.join(path,thisPath)):
697 continue
704 continue
698 if not isRadarFolder(thisPath):
705 if not isRadarFolder(thisPath):
699 continue
706 continue
700
707
701 dirList.append(thisPath)
708 dirList.append(thisPath)
702
709
703 if not(dirList):
710 if not(dirList):
704 return None, None, None, None, None, None
711 return None, None, None, None, None, None
705
712
706 dirList = sorted( dirList, key=str.lower )
713 dirList = sorted( dirList, key=str.lower )
707
714
708 doypath = dirList[-1]
715 doypath = dirList[-1]
709 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
716 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
710 fullpath = os.path.join(path, doypath, expLabel)
717 fullpath = os.path.join(path, doypath, expLabel)
711
718
712
719
713 print "[Reading] %s folder was found: " %(fullpath )
720 print "[Reading] %s folder was found: " %(fullpath )
714
721
715 if set == None:
722 if set == None:
716 filename = getlastFileFromPath(fullpath, ext)
723 filename = getlastFileFromPath(fullpath, ext)
717 else:
724 else:
718 filename = getFileFromSet(fullpath, ext, set)
725 filename = getFileFromSet(fullpath, ext, set)
719
726
720 if not(filename):
727 if not(filename):
721 return None, None, None, None, None, None
728 return None, None, None, None, None, None
722
729
723 print "[Reading] %s file was found" %(filename)
730 print "[Reading] %s file was found" %(filename)
724
731
725 if not(self.__verifyFile(os.path.join(fullpath, filename))):
732 if not(self.__verifyFile(os.path.join(fullpath, filename))):
726 return None, None, None, None, None, None
733 return None, None, None, None, None, None
727
734
728 year = int( filename[1:5] )
735 year = int( filename[1:5] )
729 doy = int( filename[5:8] )
736 doy = int( filename[5:8] )
730 set = int( filename[8:11] )
737 set = int( filename[8:11] )
731
738
732 return fullpath, foldercounter, filename, year, doy, set
739 return fullpath, foldercounter, filename, year, doy, set
733
740
734 def __setNextFileOffline(self):
741 def __setNextFileOffline(self):
735
742
736 idFile = self.fileIndex
743 idFile = self.fileIndex
737
744
738 while (True):
745 while (True):
739 idFile += 1
746 idFile += 1
740 if not(idFile < len(self.filenameList)):
747 if not(idFile < len(self.filenameList)):
741 self.flagNoMoreFiles = 1
748 self.flagNoMoreFiles = 1
742 # print "[Reading] No more Files"
749 # print "[Reading] No more Files"
743 return 0
750 return 0
744
751
745 filename = self.filenameList[idFile]
752 filename = self.filenameList[idFile]
746
753
747 if not(self.__verifyFile(filename)):
754 if not(self.__verifyFile(filename)):
748 continue
755 continue
749
756
750 fileSize = os.path.getsize(filename)
757 fileSize = os.path.getsize(filename)
751 fp = open(filename,'rb')
758 fp = open(filename,'rb')
752 break
759 break
753
760
754 self.flagIsNewFile = 1
761 self.flagIsNewFile = 1
755 self.fileIndex = idFile
762 self.fileIndex = idFile
756 self.filename = filename
763 self.filename = filename
757 self.fileSize = fileSize
764 self.fileSize = fileSize
758 self.fp = fp
765 self.fp = fp
759
766
760 # print "[Reading] Setting the file: %s"%self.filename
767 # print "[Reading] Setting the file: %s"%self.filename
761
768
762 return 1
769 return 1
763
770
764 def __setNextFileOnline(self):
771 def __setNextFileOnline(self):
765 """
772 """
766 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
773 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
767 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
774 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
768 siguientes.
775 siguientes.
769
776
770 Affected:
777 Affected:
771 self.flagIsNewFile
778 self.flagIsNewFile
772 self.filename
779 self.filename
773 self.fileSize
780 self.fileSize
774 self.fp
781 self.fp
775 self.set
782 self.set
776 self.flagNoMoreFiles
783 self.flagNoMoreFiles
777
784
778 Return:
785 Return:
779 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
786 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
780 1 : si el file fue abierto con exito y esta listo a ser leido
787 1 : si el file fue abierto con exito y esta listo a ser leido
781
788
782 Excepciones:
789 Excepciones:
783 Si un determinado file no puede ser abierto
790 Si un determinado file no puede ser abierto
784 """
791 """
785 nFiles = 0
792 nFiles = 0
786 fileOk_flag = False
793 fileOk_flag = False
787 firstTime_flag = True
794 firstTime_flag = True
788
795
789 self.set += 1
796 self.set += 1
790
797
791 if self.set > 999:
798 if self.set > 999:
792 self.set = 0
799 self.set = 0
793 self.foldercounter += 1
800 self.foldercounter += 1
794
801
795 #busca el 1er file disponible
802 #busca el 1er file disponible
796 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
803 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
797 if fullfilename:
804 if fullfilename:
798 if self.__verifyFile(fullfilename, False):
805 if self.__verifyFile(fullfilename, False):
799 fileOk_flag = True
806 fileOk_flag = True
800
807
801 #si no encuentra un file entonces espera y vuelve a buscar
808 #si no encuentra un file entonces espera y vuelve a buscar
802 if not(fileOk_flag):
809 if not(fileOk_flag):
803 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
810 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
804
811
805 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
812 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
806 tries = self.nTries
813 tries = self.nTries
807 else:
814 else:
808 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
815 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
809
816
810 for nTries in range( tries ):
817 for nTries in range( tries ):
811 if firstTime_flag:
818 if firstTime_flag:
812 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
819 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
813 sleep( self.delay )
820 sleep( self.delay )
814 else:
821 else:
815 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
822 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
816
823
817 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
818 if fullfilename:
825 if fullfilename:
819 if self.__verifyFile(fullfilename):
826 if self.__verifyFile(fullfilename):
820 fileOk_flag = True
827 fileOk_flag = True
821 break
828 break
822
829
823 if fileOk_flag:
830 if fileOk_flag:
824 break
831 break
825
832
826 firstTime_flag = False
833 firstTime_flag = False
827
834
828 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
835 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
829 self.set += 1
836 self.set += 1
830
837
831 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
838 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
832 self.set = 0
839 self.set = 0
833 self.doy += 1
840 self.doy += 1
834 self.foldercounter = 0
841 self.foldercounter = 0
835
842
836 if fileOk_flag:
843 if fileOk_flag:
837 self.fileSize = os.path.getsize( fullfilename )
844 self.fileSize = os.path.getsize( fullfilename )
838 self.filename = fullfilename
845 self.filename = fullfilename
839 self.flagIsNewFile = 1
846 self.flagIsNewFile = 1
840 if self.fp != None: self.fp.close()
847 if self.fp != None: self.fp.close()
841 self.fp = open(fullfilename, 'rb')
848 self.fp = open(fullfilename, 'rb')
842 self.flagNoMoreFiles = 0
849 self.flagNoMoreFiles = 0
843 # print '[Reading] Setting the file: %s' % fullfilename
850 # print '[Reading] Setting the file: %s' % fullfilename
844 else:
851 else:
845 self.fileSize = 0
852 self.fileSize = 0
846 self.filename = None
853 self.filename = None
847 self.flagIsNewFile = 0
854 self.flagIsNewFile = 0
848 self.fp = None
855 self.fp = None
849 self.flagNoMoreFiles = 1
856 self.flagNoMoreFiles = 1
850 # print '[Reading] No more files to read'
857 # print '[Reading] No more files to read'
851
858
852 return fileOk_flag
859 return fileOk_flag
853
860
854 def setNextFile(self):
861 def setNextFile(self):
855 if self.fp != None:
862 if self.fp != None:
856 self.fp.close()
863 self.fp.close()
857
864
858 if self.online:
865 if self.online:
859 newFile = self.__setNextFileOnline()
866 newFile = self.__setNextFileOnline()
860 else:
867 else:
861 newFile = self.__setNextFileOffline()
868 newFile = self.__setNextFileOffline()
862
869
863 if not(newFile):
870 if not(newFile):
864 print '[Reading] No more files to read'
871 print '[Reading] No more files to read'
865 return 0
872 return 0
866
873
867 print '[Reading] Setting the file: %s' % self.filename
874 print '[Reading] Setting the file: %s' % self.filename
868
875
869 self.__readFirstHeader()
876 self.__readFirstHeader()
870 self.nReadBlocks = 0
877 self.nReadBlocks = 0
871 return 1
878 return 1
872
879
873 def __waitNewBlock(self):
880 def __waitNewBlock(self):
874 """
881 """
875 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
882 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
876
883
877 Si el modo de lectura es OffLine siempre retorn 0
884 Si el modo de lectura es OffLine siempre retorn 0
878 """
885 """
879 if not self.online:
886 if not self.online:
880 return 0
887 return 0
881
888
882 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
889 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
883 return 0
890 return 0
884
891
885 currentPointer = self.fp.tell()
892 currentPointer = self.fp.tell()
886
893
887 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
894 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
888
895
889 for nTries in range( self.nTries ):
896 for nTries in range( self.nTries ):
890
897
891 self.fp.close()
898 self.fp.close()
892 self.fp = open( self.filename, 'rb' )
899 self.fp = open( self.filename, 'rb' )
893 self.fp.seek( currentPointer )
900 self.fp.seek( currentPointer )
894
901
895 self.fileSize = os.path.getsize( self.filename )
902 self.fileSize = os.path.getsize( self.filename )
896 currentSize = self.fileSize - currentPointer
903 currentSize = self.fileSize - currentPointer
897
904
898 if ( currentSize >= neededSize ):
905 if ( currentSize >= neededSize ):
899 self.basicHeaderObj.read(self.fp)
906 self.basicHeaderObj.read(self.fp)
900 return 1
907 return 1
901
908
902 if self.fileSize == self.fileSizeByHeader:
909 if self.fileSize == self.fileSizeByHeader:
903 # self.flagEoF = True
910 # self.flagEoF = True
904 return 0
911 return 0
905
912
906 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
913 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
907 sleep( self.delay )
914 sleep( self.delay )
908
915
909
916
910 return 0
917 return 0
911
918
912 def waitDataBlock(self,pointer_location):
919 def waitDataBlock(self,pointer_location):
913
920
914 currentPointer = pointer_location
921 currentPointer = pointer_location
915
922
916 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
923 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
917
924
918 for nTries in range( self.nTries ):
925 for nTries in range( self.nTries ):
919 self.fp.close()
926 self.fp.close()
920 self.fp = open( self.filename, 'rb' )
927 self.fp = open( self.filename, 'rb' )
921 self.fp.seek( currentPointer )
928 self.fp.seek( currentPointer )
922
929
923 self.fileSize = os.path.getsize( self.filename )
930 self.fileSize = os.path.getsize( self.filename )
924 currentSize = self.fileSize - currentPointer
931 currentSize = self.fileSize - currentPointer
925
932
926 if ( currentSize >= neededSize ):
933 if ( currentSize >= neededSize ):
927 return 1
934 return 1
928
935
929 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
936 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
930 sleep( self.delay )
937 sleep( self.delay )
931
938
932 return 0
939 return 0
933
940
934 def __jumpToLastBlock(self):
941 def __jumpToLastBlock(self):
935
942
936 if not(self.__isFirstTimeOnline):
943 if not(self.__isFirstTimeOnline):
937 return
944 return
938
945
939 csize = self.fileSize - self.fp.tell()
946 csize = self.fileSize - self.fp.tell()
940 blocksize = self.processingHeaderObj.blockSize
947 blocksize = self.processingHeaderObj.blockSize
941
948
942 #salta el primer bloque de datos
949 #salta el primer bloque de datos
943 if csize > self.processingHeaderObj.blockSize:
950 if csize > self.processingHeaderObj.blockSize:
944 self.fp.seek(self.fp.tell() + blocksize)
951 self.fp.seek(self.fp.tell() + blocksize)
945 else:
952 else:
946 return
953 return
947
954
948 csize = self.fileSize - self.fp.tell()
955 csize = self.fileSize - self.fp.tell()
949 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
956 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
950 while True:
957 while True:
951
958
952 if self.fp.tell()<self.fileSize:
959 if self.fp.tell()<self.fileSize:
953 self.fp.seek(self.fp.tell() + neededsize)
960 self.fp.seek(self.fp.tell() + neededsize)
954 else:
961 else:
955 self.fp.seek(self.fp.tell() - neededsize)
962 self.fp.seek(self.fp.tell() - neededsize)
956 break
963 break
957
964
958 # csize = self.fileSize - self.fp.tell()
965 # csize = self.fileSize - self.fp.tell()
959 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
966 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
960 # factor = int(csize/neededsize)
967 # factor = int(csize/neededsize)
961 # if factor > 0:
968 # if factor > 0:
962 # self.fp.seek(self.fp.tell() + factor*neededsize)
969 # self.fp.seek(self.fp.tell() + factor*neededsize)
963
970
964 self.flagIsNewFile = 0
971 self.flagIsNewFile = 0
965 self.__isFirstTimeOnline = 0
972 self.__isFirstTimeOnline = 0
966
973
967 def __setNewBlock(self):
974 def __setNewBlock(self):
968
975
969 if self.fp == None:
976 if self.fp == None:
970 return 0
977 return 0
971
978
972 # if self.online:
979 # if self.online:
973 # self.__jumpToLastBlock()
980 # self.__jumpToLastBlock()
974
981
975 if self.flagIsNewFile:
982 if self.flagIsNewFile:
976 self.lastUTTime = self.basicHeaderObj.utc
983 self.lastUTTime = self.basicHeaderObj.utc
977 return 1
984 return 1
978
985
979 if self.realtime:
986 if self.realtime:
980 self.flagDiscontinuousBlock = 1
987 self.flagDiscontinuousBlock = 1
981 if not(self.setNextFile()):
988 if not(self.setNextFile()):
982 return 0
989 return 0
983 else:
990 else:
984 return 1
991 return 1
985
992
986 currentSize = self.fileSize - self.fp.tell()
993 currentSize = self.fileSize - self.fp.tell()
987 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
994 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
988
995
989 if (currentSize >= neededSize):
996 if (currentSize >= neededSize):
990 self.basicHeaderObj.read(self.fp)
997 self.basicHeaderObj.read(self.fp)
991 self.lastUTTime = self.basicHeaderObj.utc
998 self.lastUTTime = self.basicHeaderObj.utc
992 return 1
999 return 1
993
1000
994 if self.__waitNewBlock():
1001 if self.__waitNewBlock():
995 self.lastUTTime = self.basicHeaderObj.utc
1002 self.lastUTTime = self.basicHeaderObj.utc
996 return 1
1003 return 1
997
1004
998 if not(self.setNextFile()):
1005 if not(self.setNextFile()):
999 return 0
1006 return 0
1000
1007
1001 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1008 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1002 self.lastUTTime = self.basicHeaderObj.utc
1009 self.lastUTTime = self.basicHeaderObj.utc
1003
1010
1004 self.flagDiscontinuousBlock = 0
1011 self.flagDiscontinuousBlock = 0
1005
1012
1006 if deltaTime > self.maxTimeStep:
1013 if deltaTime > self.maxTimeStep:
1007 self.flagDiscontinuousBlock = 1
1014 self.flagDiscontinuousBlock = 1
1008
1015
1009 return 1
1016 return 1
1010
1017
1011 def readNextBlock(self):
1018 def readNextBlock(self):
1012
1019
1013 #Skip block out of startTime and endTime
1020 #Skip block out of startTime and endTime
1014 while True:
1021 while True:
1015 if not(self.__setNewBlock()):
1022 if not(self.__setNewBlock()):
1016 return 0
1023 return 0
1017
1024
1018 if not(self.readBlock()):
1025 if not(self.readBlock()):
1019 return 0
1026 return 0
1020
1027
1021 self.getBasicHeader()
1028 self.getBasicHeader()
1022
1029
1023 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1030 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1024
1031
1025 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1032 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1026 self.processingHeaderObj.dataBlocksPerFile,
1033 self.processingHeaderObj.dataBlocksPerFile,
1027 self.dataOut.datatime.ctime())
1034 self.dataOut.datatime.ctime())
1028 continue
1035 continue
1029
1036
1030 break
1037 break
1031
1038
1032 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1039 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1033 self.processingHeaderObj.dataBlocksPerFile,
1040 self.processingHeaderObj.dataBlocksPerFile,
1034 self.dataOut.datatime.ctime())
1041 self.dataOut.datatime.ctime())
1035 return 1
1042 return 1
1036
1043
1037 def __readFirstHeader(self):
1044 def __readFirstHeader(self):
1038
1045
1039 self.basicHeaderObj.read(self.fp)
1046 self.basicHeaderObj.read(self.fp)
1040 self.systemHeaderObj.read(self.fp)
1047 self.systemHeaderObj.read(self.fp)
1041 self.radarControllerHeaderObj.read(self.fp)
1048 self.radarControllerHeaderObj.read(self.fp)
1042 self.processingHeaderObj.read(self.fp)
1049 self.processingHeaderObj.read(self.fp)
1043
1050
1044 self.firstHeaderSize = self.basicHeaderObj.size
1051 self.firstHeaderSize = self.basicHeaderObj.size
1045
1052
1046 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1053 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1047 if datatype == 0:
1054 if datatype == 0:
1048 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1055 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1049 elif datatype == 1:
1056 elif datatype == 1:
1050 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1057 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1051 elif datatype == 2:
1058 elif datatype == 2:
1052 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1059 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1053 elif datatype == 3:
1060 elif datatype == 3:
1054 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1061 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1055 elif datatype == 4:
1062 elif datatype == 4:
1056 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1063 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1057 elif datatype == 5:
1064 elif datatype == 5:
1058 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1065 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1059 else:
1066 else:
1060 raise ValueError, 'Data type was not defined'
1067 raise ValueError, 'Data type was not defined'
1061
1068
1062 self.dtype = datatype_str
1069 self.dtype = datatype_str
1063 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1070 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1064 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1071 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1065 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1072 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1066 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1073 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1067 self.getBlockDimension()
1074 self.getBlockDimension()
1068
1075
1069 def __verifyFile(self, filename, msgFlag=True):
1076 def __verifyFile(self, filename, msgFlag=True):
1070
1077
1071 msg = None
1078 msg = None
1072
1079
1073 try:
1080 try:
1074 fp = open(filename, 'rb')
1081 fp = open(filename, 'rb')
1075 except IOError:
1082 except IOError:
1076
1083
1077 if msgFlag:
1084 if msgFlag:
1078 print "[Reading] File %s can't be opened" % (filename)
1085 print "[Reading] File %s can't be opened" % (filename)
1079
1086
1080 return False
1087 return False
1081
1088
1082 currentPosition = fp.tell()
1089 currentPosition = fp.tell()
1083 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1090 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1084
1091
1085 if neededSize == 0:
1092 if neededSize == 0:
1086 basicHeaderObj = BasicHeader(LOCALTIME)
1093 basicHeaderObj = BasicHeader(LOCALTIME)
1087 systemHeaderObj = SystemHeader()
1094 systemHeaderObj = SystemHeader()
1088 radarControllerHeaderObj = RadarControllerHeader()
1095 radarControllerHeaderObj = RadarControllerHeader()
1089 processingHeaderObj = ProcessingHeader()
1096 processingHeaderObj = ProcessingHeader()
1090
1097
1091 if not( basicHeaderObj.read(fp) ):
1098 if not( basicHeaderObj.read(fp) ):
1092 fp.close()
1099 fp.close()
1093 return False
1100 return False
1094
1101
1095 if not( systemHeaderObj.read(fp) ):
1102 if not( systemHeaderObj.read(fp) ):
1096 fp.close()
1103 fp.close()
1097 return False
1104 return False
1098
1105
1099 if not( radarControllerHeaderObj.read(fp) ):
1106 if not( radarControllerHeaderObj.read(fp) ):
1100 fp.close()
1107 fp.close()
1101 return False
1108 return False
1102
1109
1103 if not( processingHeaderObj.read(fp) ):
1110 if not( processingHeaderObj.read(fp) ):
1104 fp.close()
1111 fp.close()
1105 return False
1112 return False
1106
1113
1107 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1114 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1108 else:
1115 else:
1109 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1116 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1110
1117
1111 fp.close()
1118 fp.close()
1112
1119
1113 fileSize = os.path.getsize(filename)
1120 fileSize = os.path.getsize(filename)
1114 currentSize = fileSize - currentPosition
1121 currentSize = fileSize - currentPosition
1115
1122
1116 if currentSize < neededSize:
1123 if currentSize < neededSize:
1117 if msgFlag and (msg != None):
1124 if msgFlag and (msg != None):
1118 print msg
1125 print msg
1119 return False
1126 return False
1120
1127
1121 return True
1128 return True
1122
1129
1123 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1130 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1124
1131
1125 path_empty = True
1132 path_empty = True
1126
1133
1127 dateList = []
1134 dateList = []
1128 pathList = []
1135 pathList = []
1129
1136
1130 multi_path = path.split(',')
1137 multi_path = path.split(',')
1131
1138
1132 if not walk:
1139 if not walk:
1133
1140
1134 for single_path in multi_path:
1141 for single_path in multi_path:
1135
1142
1136 if not os.path.isdir(single_path):
1143 if not os.path.isdir(single_path):
1137 continue
1144 continue
1138
1145
1139 fileList = glob.glob1(single_path, "*"+ext)
1146 fileList = glob.glob1(single_path, "*"+ext)
1140
1147
1141 if not fileList:
1148 if not fileList:
1142 continue
1149 continue
1143
1150
1144 path_empty = False
1151 path_empty = False
1145
1152
1146 fileList.sort()
1153 fileList.sort()
1147
1154
1148 for thisFile in fileList:
1155 for thisFile in fileList:
1149
1156
1150 if not os.path.isfile(os.path.join(single_path, thisFile)):
1157 if not os.path.isfile(os.path.join(single_path, thisFile)):
1151 continue
1158 continue
1152
1159
1153 if not isRadarFile(thisFile):
1160 if not isRadarFile(thisFile):
1154 continue
1161 continue
1155
1162
1156 if not isFileInDateRange(thisFile, startDate, endDate):
1163 if not isFileInDateRange(thisFile, startDate, endDate):
1157 continue
1164 continue
1158
1165
1159 thisDate = getDateFromRadarFile(thisFile)
1166 thisDate = getDateFromRadarFile(thisFile)
1160
1167
1161 if thisDate in dateList:
1168 if thisDate in dateList:
1162 continue
1169 continue
1163
1170
1164 dateList.append(thisDate)
1171 dateList.append(thisDate)
1165 pathList.append(single_path)
1172 pathList.append(single_path)
1166
1173
1167 else:
1174 else:
1168 for single_path in multi_path:
1175 for single_path in multi_path:
1169
1176
1170 if not os.path.isdir(single_path):
1177 if not os.path.isdir(single_path):
1171 continue
1178 continue
1172
1179
1173 dirList = []
1180 dirList = []
1174
1181
1175 for thisPath in os.listdir(single_path):
1182 for thisPath in os.listdir(single_path):
1176
1183
1177 if not os.path.isdir(os.path.join(single_path,thisPath)):
1184 if not os.path.isdir(os.path.join(single_path,thisPath)):
1178 continue
1185 continue
1179
1186
1180 if not isRadarFolder(thisPath):
1187 if not isRadarFolder(thisPath):
1181 continue
1188 continue
1182
1189
1183 if not isFolderInDateRange(thisPath, startDate, endDate):
1190 if not isFolderInDateRange(thisPath, startDate, endDate):
1184 continue
1191 continue
1185
1192
1186 dirList.append(thisPath)
1193 dirList.append(thisPath)
1187
1194
1188 if not dirList:
1195 if not dirList:
1189 continue
1196 continue
1190
1197
1191 dirList.sort()
1198 dirList.sort()
1192
1199
1193 for thisDir in dirList:
1200 for thisDir in dirList:
1194
1201
1195 datapath = os.path.join(single_path, thisDir, expLabel)
1202 datapath = os.path.join(single_path, thisDir, expLabel)
1196 fileList = glob.glob1(datapath, "*"+ext)
1203 fileList = glob.glob1(datapath, "*"+ext)
1197
1204
1198 if not fileList:
1205 if not fileList:
1199 continue
1206 continue
1200
1207
1201 path_empty = False
1208 path_empty = False
1202
1209
1203 thisDate = getDateFromRadarFolder(thisDir)
1210 thisDate = getDateFromRadarFolder(thisDir)
1204
1211
1205 pathList.append(datapath)
1212 pathList.append(datapath)
1206 dateList.append(thisDate)
1213 dateList.append(thisDate)
1207
1214
1208 dateList.sort()
1215 dateList.sort()
1209
1216
1210 if walk:
1217 if walk:
1211 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1218 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1212 else:
1219 else:
1213 pattern_path = multi_path[0]
1220 pattern_path = multi_path[0]
1214
1221
1215 if path_empty:
1222 if path_empty:
1216 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1223 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1217 else:
1224 else:
1218 if not dateList:
1225 if not dateList:
1219 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1226 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1220
1227
1221 if include_path:
1228 if include_path:
1222 return dateList, pathList
1229 return dateList, pathList
1223
1230
1224 return dateList
1231 return dateList
1225
1232
1226 def setup(self,
1233 def setup(self,
1227 path=None,
1234 path=None,
1228 startDate=None,
1235 startDate=None,
1229 endDate=None,
1236 endDate=None,
1230 startTime=datetime.time(0,0,0),
1237 startTime=datetime.time(0,0,0),
1231 endTime=datetime.time(23,59,59),
1238 endTime=datetime.time(23,59,59),
1232 set=None,
1239 set=None,
1233 expLabel = "",
1240 expLabel = "",
1234 ext = None,
1241 ext = None,
1235 online = False,
1242 online = False,
1236 delay = 60,
1243 delay = 60,
1237 walk = True,
1244 walk = True,
1238 getblock = False,
1245 getblock = False,
1239 nTxs = 1,
1246 nTxs = 1,
1240 realtime=False):
1247 realtime=False,
1248 blocksize=None,
1249 blocktime=None):
1241
1250
1242 if path == None:
1251 if path == None:
1243 raise ValueError, "[Reading] The path is not valid"
1252 raise ValueError, "[Reading] The path is not valid"
1244
1253
1245 if ext == None:
1254 if ext == None:
1246 ext = self.ext
1255 ext = self.ext
1247
1256
1248 if online:
1257 if online:
1249 print "[Reading] Searching files in online mode..."
1258 print "[Reading] Searching files in online mode..."
1250
1259
1251 for nTries in range( self.nTries ):
1260 for nTries in range( self.nTries ):
1252 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1261 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1253
1262
1254 if fullpath:
1263 if fullpath:
1255 break
1264 break
1256
1265
1257 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1266 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1258 sleep( self.delay )
1267 sleep( self.delay )
1259
1268
1260 if not(fullpath):
1269 if not(fullpath):
1261 print "[Reading] There 'isn't any valid file in %s" % path
1270 print "[Reading] There 'isn't any valid file in %s" % path
1262 return
1271 return
1263
1272
1264 self.year = year
1273 self.year = year
1265 self.doy = doy
1274 self.doy = doy
1266 self.set = set - 1
1275 self.set = set - 1
1267 self.path = path
1276 self.path = path
1268 self.foldercounter = foldercounter
1277 self.foldercounter = foldercounter
1269 last_set = None
1278 last_set = None
1270
1279
1271 else:
1280 else:
1272 print "[Reading] Searching files in offline mode ..."
1281 print "[Reading] Searching files in offline mode ..."
1273 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1282 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1274 startTime=startTime, endTime=endTime,
1283 startTime=startTime, endTime=endTime,
1275 set=set, expLabel=expLabel, ext=ext,
1284 set=set, expLabel=expLabel, ext=ext,
1276 walk=walk)
1285 walk=walk)
1277
1286
1278 if not(pathList):
1287 if not(pathList):
1279 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1288 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1280 # datetime.datetime.combine(startDate,startTime).ctime(),
1289 # datetime.datetime.combine(startDate,startTime).ctime(),
1281 # datetime.datetime.combine(endDate,endTime).ctime())
1290 # datetime.datetime.combine(endDate,endTime).ctime())
1282
1291
1283 # sys.exit(-1)
1292 # sys.exit(-1)
1284
1293
1285 self.fileIndex = -1
1294 self.fileIndex = -1
1286 self.pathList = []
1295 self.pathList = []
1287 self.filenameList = []
1296 self.filenameList = []
1288 return
1297 return
1289
1298
1290 self.fileIndex = -1
1299 self.fileIndex = -1
1291 self.pathList = pathList
1300 self.pathList = pathList
1292 self.filenameList = filenameList
1301 self.filenameList = filenameList
1293 file_name = os.path.basename(filenameList[-1])
1302 file_name = os.path.basename(filenameList[-1])
1294 basename, ext = os.path.splitext(file_name)
1303 basename, ext = os.path.splitext(file_name)
1295 last_set = int(basename[-3:])
1304 last_set = int(basename[-3:])
1296
1305
1297 self.online = online
1306 self.online = online
1298 self.realtime = realtime
1307 self.realtime = realtime
1299 self.delay = delay
1308 self.delay = delay
1300 ext = ext.lower()
1309 ext = ext.lower()
1301 self.ext = ext
1310 self.ext = ext
1302 self.getByBlock = getblock
1311 self.getByBlock = getblock
1303 self.nTxs = nTxs
1312 self.nTxs = nTxs
1304 self.startTime = startTime
1313 self.startTime = startTime
1305 self.endTime = endTime
1314 self.endTime = endTime
1306
1315
1316 #Added-----------------
1317 self.selBlocksize = blocksize
1318 self.selBlocktime = blocktime
1319
1320
1307 if not(self.setNextFile()):
1321 if not(self.setNextFile()):
1308 if (startDate!=None) and (endDate!=None):
1322 if (startDate!=None) and (endDate!=None):
1309 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1323 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1310 elif startDate != None:
1324 elif startDate != None:
1311 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1325 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1312 else:
1326 else:
1313 print "[Reading] No files"
1327 print "[Reading] No files"
1314
1328
1315 self.fileIndex = -1
1329 self.fileIndex = -1
1316 self.pathList = []
1330 self.pathList = []
1317 self.filenameList = []
1331 self.filenameList = []
1318 return
1332 return
1319
1333
1320 # self.getBasicHeader()
1334 # self.getBasicHeader()
1321
1335
1322 if last_set != None:
1336 if last_set != None:
1323 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1337 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1324 return
1338 return
1325
1339
1326 def getBasicHeader(self):
1340 def getBasicHeader(self):
1327
1341
1328 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1342 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1329
1343
1330 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1344 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1331
1345
1332 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1346 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1333
1347
1334 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1348 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1335
1349
1336 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1350 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1337
1351
1338 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1352 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1339
1353
1340 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1354 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1341
1355
1342 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1356 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1343
1357
1344
1358
1345 def getFirstHeader(self):
1359 def getFirstHeader(self):
1346
1360
1347 raise NotImplementedError
1361 raise NotImplementedError
1348
1362
1349 def getData(self):
1363 def getData(self):
1350
1364
1351 raise NotImplementedError
1365 raise NotImplementedError
1352
1366
1353 def hasNotDataInBuffer(self):
1367 def hasNotDataInBuffer(self):
1354
1368
1355 raise NotImplementedError
1369 raise NotImplementedError
1356
1370
1357 def readBlock(self):
1371 def readBlock(self):
1358
1372
1359 raise NotImplementedError
1373 raise NotImplementedError
1360
1374
1361 def isEndProcess(self):
1375 def isEndProcess(self):
1362
1376
1363 return self.flagNoMoreFiles
1377 return self.flagNoMoreFiles
1364
1378
1365 def printReadBlocks(self):
1379 def printReadBlocks(self):
1366
1380
1367 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1381 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1368
1382
1369 def printTotalBlocks(self):
1383 def printTotalBlocks(self):
1370
1384
1371 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1385 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1372
1386
1373 def printNumberOfBlock(self):
1387 def printNumberOfBlock(self):
1374
1388
1375 if self.flagIsNewBlock:
1389 if self.flagIsNewBlock:
1376 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1390 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1377 self.processingHeaderObj.dataBlocksPerFile,
1391 self.processingHeaderObj.dataBlocksPerFile,
1378 self.dataOut.datatime.ctime())
1392 self.dataOut.datatime.ctime())
1379
1393
1380 def printInfo(self):
1394 def printInfo(self):
1381
1395
1382 if self.__printInfo == False:
1396 if self.__printInfo == False:
1383 return
1397 return
1384
1398
1385 self.basicHeaderObj.printInfo()
1399 self.basicHeaderObj.printInfo()
1386 self.systemHeaderObj.printInfo()
1400 self.systemHeaderObj.printInfo()
1387 self.radarControllerHeaderObj.printInfo()
1401 self.radarControllerHeaderObj.printInfo()
1388 self.processingHeaderObj.printInfo()
1402 self.processingHeaderObj.printInfo()
1389
1403
1390 self.__printInfo = False
1404 self.__printInfo = False
1391
1405
1392
1406
1393 def run(self, **kwargs):
1407 def run(self, **kwargs):
1394
1408
1395 if not(self.isConfig):
1409 if not(self.isConfig):
1396
1410
1397 # self.dataOut = dataOut
1411 # self.dataOut = dataOut
1398 self.setup(**kwargs)
1412 self.setup(**kwargs)
1399 self.isConfig = True
1413 self.isConfig = True
1400
1414
1401 self.getData()
1415 self.getData()
1402
1416
1403 class JRODataWriter(JRODataIO):
1417 class JRODataWriter(JRODataIO):
1404
1418
1405 """
1419 """
1406 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1420 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1407 de los datos siempre se realiza por bloques.
1421 de los datos siempre se realiza por bloques.
1408 """
1422 """
1409
1423
1410 blockIndex = 0
1424 blockIndex = 0
1411
1425
1412 path = None
1426 path = None
1413
1427
1414 setFile = None
1428 setFile = None
1415
1429
1416 profilesPerBlock = None
1430 profilesPerBlock = None
1417
1431
1418 blocksPerFile = None
1432 blocksPerFile = None
1419
1433
1420 nWriteBlocks = 0
1434 nWriteBlocks = 0
1421
1435
1422 fileDate = None
1436 fileDate = None
1423
1437
1424 def __init__(self, dataOut=None):
1438 def __init__(self, dataOut=None):
1425 raise NotImplementedError
1439 raise NotImplementedError
1426
1440
1427
1441
1428 def hasAllDataInBuffer(self):
1442 def hasAllDataInBuffer(self):
1429 raise NotImplementedError
1443 raise NotImplementedError
1430
1444
1431
1445
1432 def setBlockDimension(self):
1446 def setBlockDimension(self):
1433 raise NotImplementedError
1447 raise NotImplementedError
1434
1448
1435
1449
1436 def writeBlock(self):
1450 def writeBlock(self):
1437 raise NotImplementedError
1451 raise NotImplementedError
1438
1452
1439
1453
1440 def putData(self):
1454 def putData(self):
1441 raise NotImplementedError
1455 raise NotImplementedError
1442
1456
1443
1457
1444 def getProcessFlags(self):
1458 def getProcessFlags(self):
1445
1459
1446 processFlags = 0
1460 processFlags = 0
1447
1461
1448 dtype_index = get_dtype_index(self.dtype)
1462 dtype_index = get_dtype_index(self.dtype)
1449 procflag_dtype = get_procflag_dtype(dtype_index)
1463 procflag_dtype = get_procflag_dtype(dtype_index)
1450
1464
1451 processFlags += procflag_dtype
1465 processFlags += procflag_dtype
1452
1466
1453 if self.dataOut.flagDecodeData:
1467 if self.dataOut.flagDecodeData:
1454 processFlags += PROCFLAG.DECODE_DATA
1468 processFlags += PROCFLAG.DECODE_DATA
1455
1469
1456 if self.dataOut.flagDeflipData:
1470 if self.dataOut.flagDeflipData:
1457 processFlags += PROCFLAG.DEFLIP_DATA
1471 processFlags += PROCFLAG.DEFLIP_DATA
1458
1472
1459 if self.dataOut.code is not None:
1473 if self.dataOut.code is not None:
1460 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1474 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1461
1475
1462 if self.dataOut.nCohInt > 1:
1476 if self.dataOut.nCohInt > 1:
1463 processFlags += PROCFLAG.COHERENT_INTEGRATION
1477 processFlags += PROCFLAG.COHERENT_INTEGRATION
1464
1478
1465 if self.dataOut.type == "Spectra":
1479 if self.dataOut.type == "Spectra":
1466 if self.dataOut.nIncohInt > 1:
1480 if self.dataOut.nIncohInt > 1:
1467 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1481 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1468
1482
1469 if self.dataOut.data_dc is not None:
1483 if self.dataOut.data_dc is not None:
1470 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1484 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1471
1485
1472 if self.dataOut.flagShiftFFT:
1486 if self.dataOut.flagShiftFFT:
1473 processFlags += PROCFLAG.SHIFT_FFT_DATA
1487 processFlags += PROCFLAG.SHIFT_FFT_DATA
1474
1488
1475 return processFlags
1489 return processFlags
1476
1490
1477 def setBasicHeader(self):
1491 def setBasicHeader(self):
1478
1492
1479 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1493 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1480 self.basicHeaderObj.version = self.versionFile
1494 self.basicHeaderObj.version = self.versionFile
1481 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1495 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1482
1496
1483 utc = numpy.floor(self.dataOut.utctime)
1497 utc = numpy.floor(self.dataOut.utctime)
1484 milisecond = (self.dataOut.utctime - utc)* 1000.0
1498 milisecond = (self.dataOut.utctime - utc)* 1000.0
1485
1499
1486 self.basicHeaderObj.utc = utc
1500 self.basicHeaderObj.utc = utc
1487 self.basicHeaderObj.miliSecond = milisecond
1501 self.basicHeaderObj.miliSecond = milisecond
1488 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1502 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1489 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1503 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1490 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1504 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1491
1505
1492 def setFirstHeader(self):
1506 def setFirstHeader(self):
1493 """
1507 """
1494 Obtiene una copia del First Header
1508 Obtiene una copia del First Header
1495
1509
1496 Affected:
1510 Affected:
1497
1511
1498 self.basicHeaderObj
1512 self.basicHeaderObj
1499 self.systemHeaderObj
1513 self.systemHeaderObj
1500 self.radarControllerHeaderObj
1514 self.radarControllerHeaderObj
1501 self.processingHeaderObj self.
1515 self.processingHeaderObj self.
1502
1516
1503 Return:
1517 Return:
1504 None
1518 None
1505 """
1519 """
1506
1520
1507 raise NotImplementedError
1521 raise NotImplementedError
1508
1522
1509 def __writeFirstHeader(self):
1523 def __writeFirstHeader(self):
1510 """
1524 """
1511 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1525 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1512
1526
1513 Affected:
1527 Affected:
1514 __dataType
1528 __dataType
1515
1529
1516 Return:
1530 Return:
1517 None
1531 None
1518 """
1532 """
1519
1533
1520 # CALCULAR PARAMETROS
1534 # CALCULAR PARAMETROS
1521
1535
1522 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1536 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1523 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1537 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1524
1538
1525 self.basicHeaderObj.write(self.fp)
1539 self.basicHeaderObj.write(self.fp)
1526 self.systemHeaderObj.write(self.fp)
1540 self.systemHeaderObj.write(self.fp)
1527 self.radarControllerHeaderObj.write(self.fp)
1541 self.radarControllerHeaderObj.write(self.fp)
1528 self.processingHeaderObj.write(self.fp)
1542 self.processingHeaderObj.write(self.fp)
1529
1543
1530 def __setNewBlock(self):
1544 def __setNewBlock(self):
1531 """
1545 """
1532 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1546 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1533
1547
1534 Return:
1548 Return:
1535 0 : si no pudo escribir nada
1549 0 : si no pudo escribir nada
1536 1 : Si escribio el Basic el First Header
1550 1 : Si escribio el Basic el First Header
1537 """
1551 """
1538 if self.fp == None:
1552 if self.fp == None:
1539 self.setNextFile()
1553 self.setNextFile()
1540
1554
1541 if self.flagIsNewFile:
1555 if self.flagIsNewFile:
1542 return 1
1556 return 1
1543
1557
1544 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1558 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1545 self.basicHeaderObj.write(self.fp)
1559 self.basicHeaderObj.write(self.fp)
1546 return 1
1560 return 1
1547
1561
1548 if not( self.setNextFile() ):
1562 if not( self.setNextFile() ):
1549 return 0
1563 return 0
1550
1564
1551 return 1
1565 return 1
1552
1566
1553
1567
1554 def writeNextBlock(self):
1568 def writeNextBlock(self):
1555 """
1569 """
1556 Selecciona el bloque siguiente de datos y los escribe en un file
1570 Selecciona el bloque siguiente de datos y los escribe en un file
1557
1571
1558 Return:
1572 Return:
1559 0 : Si no hizo pudo escribir el bloque de datos
1573 0 : Si no hizo pudo escribir el bloque de datos
1560 1 : Si no pudo escribir el bloque de datos
1574 1 : Si no pudo escribir el bloque de datos
1561 """
1575 """
1562 if not( self.__setNewBlock() ):
1576 if not( self.__setNewBlock() ):
1563 return 0
1577 return 0
1564
1578
1565 self.writeBlock()
1579 self.writeBlock()
1566
1580
1567 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1581 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1568 self.processingHeaderObj.dataBlocksPerFile)
1582 self.processingHeaderObj.dataBlocksPerFile)
1569
1583
1570 return 1
1584 return 1
1571
1585
1572 def setNextFile(self):
1586 def setNextFile(self):
1573 """
1587 """
1574 Determina el siguiente file que sera escrito
1588 Determina el siguiente file que sera escrito
1575
1589
1576 Affected:
1590 Affected:
1577 self.filename
1591 self.filename
1578 self.subfolder
1592 self.subfolder
1579 self.fp
1593 self.fp
1580 self.setFile
1594 self.setFile
1581 self.flagIsNewFile
1595 self.flagIsNewFile
1582
1596
1583 Return:
1597 Return:
1584 0 : Si el archivo no puede ser escrito
1598 0 : Si el archivo no puede ser escrito
1585 1 : Si el archivo esta listo para ser escrito
1599 1 : Si el archivo esta listo para ser escrito
1586 """
1600 """
1587 ext = self.ext
1601 ext = self.ext
1588 path = self.path
1602 path = self.path
1589
1603
1590 if self.fp != None:
1604 if self.fp != None:
1591 self.fp.close()
1605 self.fp.close()
1592
1606
1593 timeTuple = time.localtime( self.dataOut.utctime)
1607 timeTuple = time.localtime( self.dataOut.utctime)
1594 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1608 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1595
1609
1596 fullpath = os.path.join( path, subfolder )
1610 fullpath = os.path.join( path, subfolder )
1597 setFile = self.setFile
1611 setFile = self.setFile
1598
1612
1599 if not( os.path.exists(fullpath) ):
1613 if not( os.path.exists(fullpath) ):
1600 os.mkdir(fullpath)
1614 os.mkdir(fullpath)
1601 setFile = -1 #inicializo mi contador de seteo
1615 setFile = -1 #inicializo mi contador de seteo
1602 else:
1616 else:
1603 filesList = os.listdir( fullpath )
1617 filesList = os.listdir( fullpath )
1604 if len( filesList ) > 0:
1618 if len( filesList ) > 0:
1605 filesList = sorted( filesList, key=str.lower )
1619 filesList = sorted( filesList, key=str.lower )
1606 filen = filesList[-1]
1620 filen = filesList[-1]
1607 # el filename debera tener el siguiente formato
1621 # el filename debera tener el siguiente formato
1608 # 0 1234 567 89A BCDE (hex)
1622 # 0 1234 567 89A BCDE (hex)
1609 # x YYYY DDD SSS .ext
1623 # x YYYY DDD SSS .ext
1610 if isNumber( filen[8:11] ):
1624 if isNumber( filen[8:11] ):
1611 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1625 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1612 else:
1626 else:
1613 setFile = -1
1627 setFile = -1
1614 else:
1628 else:
1615 setFile = -1 #inicializo mi contador de seteo
1629 setFile = -1 #inicializo mi contador de seteo
1616
1630
1617 setFile += 1
1631 setFile += 1
1618
1632
1619 #If this is a new day it resets some values
1633 #If this is a new day it resets some values
1620 if self.dataOut.datatime.date() > self.fileDate:
1634 if self.dataOut.datatime.date() > self.fileDate:
1621 setFile = 0
1635 setFile = 0
1622 self.nTotalBlocks = 0
1636 self.nTotalBlocks = 0
1623
1637
1624 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1638 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1625
1639
1626 filename = os.path.join( path, subfolder, filen )
1640 filename = os.path.join( path, subfolder, filen )
1627
1641
1628 fp = open( filename,'wb' )
1642 fp = open( filename,'wb' )
1629
1643
1630 self.blockIndex = 0
1644 self.blockIndex = 0
1631
1645
1632 #guardando atributos
1646 #guardando atributos
1633 self.filename = filename
1647 self.filename = filename
1634 self.subfolder = subfolder
1648 self.subfolder = subfolder
1635 self.fp = fp
1649 self.fp = fp
1636 self.setFile = setFile
1650 self.setFile = setFile
1637 self.flagIsNewFile = 1
1651 self.flagIsNewFile = 1
1638 self.fileDate = self.dataOut.datatime.date()
1652 self.fileDate = self.dataOut.datatime.date()
1639
1653
1640 self.setFirstHeader()
1654 self.setFirstHeader()
1641
1655
1642 print '[Writing] Opening file: %s'%self.filename
1656 print '[Writing] Opening file: %s'%self.filename
1643
1657
1644 self.__writeFirstHeader()
1658 self.__writeFirstHeader()
1645
1659
1646 return 1
1660 return 1
1647
1661
1648 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1662 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1649 """
1663 """
1650 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1664 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1651
1665
1652 Inputs:
1666 Inputs:
1653 path : directory where data will be saved
1667 path : directory where data will be saved
1654 profilesPerBlock : number of profiles per block
1668 profilesPerBlock : number of profiles per block
1655 set : initial file set
1669 set : initial file set
1656 datatype : An integer number that defines data type:
1670 datatype : An integer number that defines data type:
1657 0 : int8 (1 byte)
1671 0 : int8 (1 byte)
1658 1 : int16 (2 bytes)
1672 1 : int16 (2 bytes)
1659 2 : int32 (4 bytes)
1673 2 : int32 (4 bytes)
1660 3 : int64 (8 bytes)
1674 3 : int64 (8 bytes)
1661 4 : float32 (4 bytes)
1675 4 : float32 (4 bytes)
1662 5 : double64 (8 bytes)
1676 5 : double64 (8 bytes)
1663
1677
1664 Return:
1678 Return:
1665 0 : Si no realizo un buen seteo
1679 0 : Si no realizo un buen seteo
1666 1 : Si realizo un buen seteo
1680 1 : Si realizo un buen seteo
1667 """
1681 """
1668
1682
1669 if ext == None:
1683 if ext == None:
1670 ext = self.ext
1684 ext = self.ext
1671
1685
1672 self.ext = ext.lower()
1686 self.ext = ext.lower()
1673
1687
1674 self.path = path
1688 self.path = path
1675
1689
1676 if set is None:
1690 if set is None:
1677 self.setFile = -1
1691 self.setFile = -1
1678 else:
1692 else:
1679 self.setFile = set - 1
1693 self.setFile = set - 1
1680
1694
1681 self.blocksPerFile = blocksPerFile
1695 self.blocksPerFile = blocksPerFile
1682
1696
1683 self.profilesPerBlock = profilesPerBlock
1697 self.profilesPerBlock = profilesPerBlock
1684
1698
1685 self.dataOut = dataOut
1699 self.dataOut = dataOut
1686 self.fileDate = self.dataOut.datatime.date()
1700 self.fileDate = self.dataOut.datatime.date()
1687 #By default
1701 #By default
1688 self.dtype = self.dataOut.dtype
1702 self.dtype = self.dataOut.dtype
1689
1703
1690 if datatype is not None:
1704 if datatype is not None:
1691 self.dtype = get_numpy_dtype(datatype)
1705 self.dtype = get_numpy_dtype(datatype)
1692
1706
1693 if not(self.setNextFile()):
1707 if not(self.setNextFile()):
1694 print "[Writing] There isn't a next file"
1708 print "[Writing] There isn't a next file"
1695 return 0
1709 return 0
1696
1710
1697 self.setBlockDimension()
1711 self.setBlockDimension()
1698
1712
1699 return 1
1713 return 1
1700
1714
1701 def run(self, dataOut, **kwargs):
1715 def run(self, dataOut, **kwargs):
1702
1716
1703 if not(self.isConfig):
1717 if not(self.isConfig):
1704
1718
1705 self.setup(dataOut, **kwargs)
1719 self.setup(dataOut, **kwargs)
1706 self.isConfig = True
1720 self.isConfig = True
1707
1721
1708 self.putData()
1722 self.putData()
1709
1723
Show file before | Show file after | Hide comments
@@ -1,596 +1,621
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import numpy
7 import numpy
8
8
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13
13
14 class VoltageReader(JRODataReader, ProcessingUnit):
14 class VoltageReader(JRODataReader, ProcessingUnit):
15 """
15 """
16 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
16 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
17 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
17 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
18 perfiles*alturas*canales) son almacenados en la variable "buffer".
18 perfiles*alturas*canales) son almacenados en la variable "buffer".
19
19
20 perfiles * alturas * canales
20 perfiles * alturas * canales
21
21
22 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
22 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
23 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
23 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
24 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
24 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
25 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
25 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
26
26
27 Example:
27 Example:
28
28
29 dpath = "/home/myuser/data"
29 dpath = "/home/myuser/data"
30
30
31 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
31 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
32
32
33 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
33 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
34
34
35 readerObj = VoltageReader()
35 readerObj = VoltageReader()
36
36
37 readerObj.setup(dpath, startTime, endTime)
37 readerObj.setup(dpath, startTime, endTime)
38
38
39 while(True):
39 while(True):
40
40
41 #to get one profile
41 #to get one profile
42 profile = readerObj.getData()
42 profile = readerObj.getData()
43
43
44 #print the profile
44 #print the profile
45 print profile
45 print profile
46
46
47 #If you want to see all datablock
47 #If you want to see all datablock
48 print readerObj.datablock
48 print readerObj.datablock
49
49
50 if readerObj.flagNoMoreFiles:
50 if readerObj.flagNoMoreFiles:
51 break
51 break
52
52
53 """
53 """
54
54
55 ext = ".r"
55 ext = ".r"
56
56
57 optchar = "D"
57 optchar = "D"
58 dataOut = None
58 dataOut = None
59
59
60
61 def __init__(self):
60 def __init__(self):
62 """
61 """
63 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
62 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
64
63
65 Input:
64 Input:
66 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
65 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
67 almacenar un perfil de datos cada vez que se haga un requerimiento
66 almacenar un perfil de datos cada vez que se haga un requerimiento
68 (getData). El perfil sera obtenido a partir del buffer de datos,
67 (getData). El perfil sera obtenido a partir del buffer de datos,
69 si el buffer esta vacio se hara un nuevo proceso de lectura de un
68 si el buffer esta vacio se hara un nuevo proceso de lectura de un
70 bloque de datos.
69 bloque de datos.
71 Si este parametro no es pasado se creara uno internamente.
70 Si este parametro no es pasado se creara uno internamente.
72
71
73 Variables afectadas:
72 Variables afectadas:
74 self.dataOut
73 self.dataOut
75
74
76 Return:
75 Return:
77 None
76 None
78 """
77 """
79
78
80 ProcessingUnit.__init__(self)
79 ProcessingUnit.__init__(self)
81
80
82 self.isConfig = False
81 self.isConfig = False
83
82
84 self.datablock = None
83 self.datablock = None
85
84
86 self.utc = 0
85 self.utc = 0
87
86
88 self.ext = ".r"
87 self.ext = ".r"
89
88
90 self.optchar = "D"
89 self.optchar = "D"
91
90
92 self.basicHeaderObj = BasicHeader(LOCALTIME)
91 self.basicHeaderObj = BasicHeader(LOCALTIME)
93
92
94 self.systemHeaderObj = SystemHeader()
93 self.systemHeaderObj = SystemHeader()
95
94
96 self.radarControllerHeaderObj = RadarControllerHeader()
95 self.radarControllerHeaderObj = RadarControllerHeader()
97
96
98 self.processingHeaderObj = ProcessingHeader()
97 self.processingHeaderObj = ProcessingHeader()
99
98
100 self.online = 0
99 self.online = 0
101
100
102 self.fp = None
101 self.fp = None
103
102
104 self.idFile = None
103 self.idFile = None
105
104
106 self.dtype = None
105 self.dtype = None
107
106
108 self.fileSizeByHeader = None
107 self.fileSizeByHeader = None
109
108
110 self.filenameList = []
109 self.filenameList = []
111
110
112 self.filename = None
111 self.filename = None
113
112
114 self.fileSize = None
113 self.fileSize = None
115
114
116 self.firstHeaderSize = 0
115 self.firstHeaderSize = 0
117
116
118 self.basicHeaderSize = 24
117 self.basicHeaderSize = 24
119
118
120 self.pathList = []
119 self.pathList = []
121
120
122 self.filenameList = []
121 self.filenameList = []
123
122
124 self.lastUTTime = 0
123 self.lastUTTime = 0
125
124
126 self.maxTimeStep = 30
125 self.maxTimeStep = 30
127
126
128 self.flagNoMoreFiles = 0
127 self.flagNoMoreFiles = 0
129
128
130 self.set = 0
129 self.set = 0
131
130
132 self.path = None
131 self.path = None
133
132
134 self.profileIndex = 2**32-1
133 self.profileIndex = 2**32-1
135
134
136 self.delay = 3 #seconds
135 self.delay = 3 #seconds
137
136
138 self.nTries = 3 #quantity tries
137 self.nTries = 3 #quantity tries
139
138
140 self.nFiles = 3 #number of files for searching
139 self.nFiles = 3 #number of files for searching
141
140
142 self.nReadBlocks = 0
141 self.nReadBlocks = 0
143
142
144 self.flagIsNewFile = 1
143 self.flagIsNewFile = 1
145
144
146 self.__isFirstTimeOnline = 1
145 self.__isFirstTimeOnline = 1
147
146
148 # self.ippSeconds = 0
147 # self.ippSeconds = 0
149
148
150 self.flagDiscontinuousBlock = 0
149 self.flagDiscontinuousBlock = 0
151
150
152 self.flagIsNewBlock = 0
151 self.flagIsNewBlock = 0
153
152
154 self.nTotalBlocks = 0
153 self.nTotalBlocks = 0
155
154
156 self.blocksize = 0
155 self.blocksize = 0
157
156
158 self.dataOut = self.createObjByDefault()
157 self.dataOut = self.createObjByDefault()
159
158
160 self.nTxs = 1
159 self.nTxs = 1
161
160
162 self.txIndex = 0
161 self.txIndex = 0
163
162
164 def createObjByDefault(self):
163 def createObjByDefault(self):
165
164
166 dataObj = Voltage()
165 dataObj = Voltage()
167
166
168 return dataObj
167 return dataObj
169
168
170 def __hasNotDataInBuffer(self):
169 def __hasNotDataInBuffer(self):
171
170
172 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
171 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
173 return 1
172 return 1
174
173
175 return 0
174 return 0
176
175
177
176
178 def getBlockDimension(self):
177 def getBlockDimension(self):
179 """
178 """
180 Obtiene la cantidad de puntos a leer por cada bloque de datos
179 Obtiene la cantidad de puntos a leer por cada bloque de datos
181
180
182 Affected:
181 Affected:
183 self.blocksize
182 self.blocksize
184
183
185 Return:
184 Return:
186 None
185 None
187 """
186 """
188 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
187 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
189 self.blocksize = pts2read
188 self.blocksize = pts2read
190
189
191
190
192 def readBlock(self):
191 def readBlock(self):
193 """
192 """
194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
193 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
194 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
195 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
197 es seteado a 0
196 es seteado a 0
198
197
199 Inputs:
198 Inputs:
200 None
199 None
201
200
202 Return:
201 Return:
203 None
202 None
204
203
205 Affected:
204 Affected:
206 self.profileIndex
205 self.profileIndex
207 self.datablock
206 self.datablock
208 self.flagIsNewFile
207 self.flagIsNewFile
209 self.flagIsNewBlock
208 self.flagIsNewBlock
210 self.nTotalBlocks
209 self.nTotalBlocks
211
210
212 Exceptions:
211 Exceptions:
213 Si un bloque leido no es un bloque valido
212 Si un bloque leido no es un bloque valido
214 """
213 """
215 current_pointer_location = self.fp.tell()
214 current_pointer_location = self.fp.tell()
216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
215 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217
216
218 try:
217 try:
219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
218 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
220 except:
219 except:
221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
220 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222
221
223 if self.waitDataBlock(pointer_location=current_pointer_location):
222 if self.waitDataBlock(pointer_location=current_pointer_location):
224 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
223 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
225 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
224 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
226 # return 0
225 # return 0
227
226
228 #Dimensions : nChannels, nProfiles, nSamples
227 #Dimensions : nChannels, nProfiles, nSamples
229
228
230 junk = numpy.transpose(junk, (2,0,1))
229 junk = numpy.transpose(junk, (2,0,1))
231 self.datablock = junk['real'] + junk['imag']*1j
230 self.datablock = junk['real'] + junk['imag']*1j
232
231
233 self.profileIndex = 0
232 self.profileIndex = 0
234
233
235 self.flagIsNewFile = 0
234 self.flagIsNewFile = 0
236 self.flagIsNewBlock = 1
235 self.flagIsNewBlock = 1
237
236
238 self.nTotalBlocks += 1
237 self.nTotalBlocks += 1
239 self.nReadBlocks += 1
238 self.nReadBlocks += 1
240
239
241 return 1
240 return 1
242
241
243 def getFirstHeader(self):
242 def getFirstHeader(self):
244
243
245 self.getBasicHeader()
244 self.getBasicHeader()
246
245
247 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
246 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
248
247
249 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
248 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
250
249
251 if self.nTxs > 1:
250 if self.nTxs > 1:
252 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
251 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
253
252
254 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
253 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
255
254
256 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
255 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
257 #
256 #
258 # if self.radarControllerHeaderObj.code is not None:
257 # if self.radarControllerHeaderObj.code is not None:
259 #
258 #
260 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
259 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
261 #
260 #
262 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
261 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
263 #
262 #
264 # self.dataOut.code = self.radarControllerHeaderObj.code
263 # self.dataOut.code = self.radarControllerHeaderObj.code
265
264
266 self.dataOut.dtype = self.dtype
265 self.dataOut.dtype = self.dtype
267
266
268 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
267 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
269
268
270 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
269 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
271
270
272 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
271 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
273
272
274 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
273 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
275
274
276 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
275 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
277
276
278 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
277 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
279
278
280 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
279 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
281
280
282 def reshapeData(self):
281 def reshapeData(self):
283
282
284 if self.nTxs < 0:
283 if self.nTxs < 0:
285 return
284 return
286
285
287 if self.nTxs == 1:
286 if self.nTxs == 1:
288 return
287 return
289
288
290 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
289 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
291 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
290 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
292
291
293 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
292 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
294 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
293 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
295
294
296 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
295 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
297
296
298 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
297 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
299 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
298 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
300 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
299 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
301
300
302 return
301 return
303
302
304 def getData(self):
303 def getData(self):
305 """
304 """
306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
305 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
306 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
307 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
309 "readNextBlock"
308 "readNextBlock"
310
309
311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
310 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
312
311
313 Return:
312 Return:
314
313
315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
314 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
316 es igual al total de perfiles leidos desde el archivo.
315 es igual al total de perfiles leidos desde el archivo.
317
316
318 Si self.getByBlock == False:
317 Si self.getByBlock == False:
319
318
320 self.dataOut.data = buffer[:, thisProfile, :]
319 self.dataOut.data = buffer[:, thisProfile, :]
321
320
322 shape = [nChannels, nHeis]
321 shape = [nChannels, nHeis]
323
322
324 Si self.getByBlock == True:
323 Si self.getByBlock == True:
325
324
326 self.dataOut.data = buffer[:, :, :]
325 self.dataOut.data = buffer[:, :, :]
327
326
328 shape = [nChannels, nProfiles, nHeis]
327 shape = [nChannels, nProfiles, nHeis]
329
328
330 Variables afectadas:
329 Variables afectadas:
331 self.dataOut
330 self.dataOut
332 self.profileIndex
331 self.profileIndex
333
332
334 Affected:
333 Affected:
335 self.dataOut
334 self.dataOut
336 self.profileIndex
335 self.profileIndex
337 self.flagDiscontinuousBlock
336 self.flagDiscontinuousBlock
338 self.flagIsNewBlock
337 self.flagIsNewBlock
339 """
338 """
340
339
341 if self.flagNoMoreFiles:
340 if self.flagNoMoreFiles:
342 self.dataOut.flagNoData = True
341 self.dataOut.flagNoData = True
343 print 'Process finished'
342 print 'Process finished'
344 return 0
343 return 0
345
344
346 self.flagDiscontinuousBlock = 0
345 self.flagDiscontinuousBlock = 0
347 self.flagIsNewBlock = 0
346 self.flagIsNewBlock = 0
348
347
349 if self.__hasNotDataInBuffer():
348 if self.__hasNotDataInBuffer():
350
349
351 if not( self.readNextBlock() ):
350 if not( self.readNextBlock() ):
352 return 0
351 return 0
353
352
354 self.getFirstHeader()
353 self.getFirstHeader()
355
354
356 self.reshapeData()
355 self.reshapeData()
357
356
358 if self.datablock is None:
357 if self.datablock is None:
359 self.dataOut.flagNoData = True
358 self.dataOut.flagNoData = True
360 return 0
359 return 0
361
360
362 if not self.getByBlock:
361 if not self.getByBlock:
363
362
364 """
363 """
365 Return profile by profile
364 Return profile by profile
366
365
367 If nTxs > 1 then one profile is divided by nTxs and number of total
366 If nTxs > 1 then one profile is divided by nTxs and number of total
368 blocks is increased by nTxs (nProfiles *= nTxs)
367 blocks is increased by nTxs (nProfiles *= nTxs)
369 """
368 """
370 self.dataOut.flagDataAsBlock = False
369 self.dataOut.flagDataAsBlock = False
371 self.dataOut.data = self.datablock[:,self.profileIndex,:]
370 self.dataOut.data = self.datablock[:,self.profileIndex,:]
372 self.dataOut.profileIndex = self.profileIndex
371 self.dataOut.profileIndex = self.profileIndex
373
372
374 self.profileIndex += 1
373 self.profileIndex += 1
375
374
375 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
376 # """
377 # Return all block
378 # """
379 # self.dataOut.flagDataAsBlock = True
380 # self.dataOut.data = self.datablock
381 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
382 #
383 # self.profileIndex = self.dataOut.nProfiles
384
376 else:
385 else:
377 """
386 """
378 Return all block
387 Return a block
379 """
388 """
380 self.dataOut.flagDataAsBlock = True
389 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
381 self.dataOut.data = self.datablock
390 if self.selBlocktime != None: self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(self.dataOut.ippSeconds*self.dataOut.nProfiles)))
382 self.dataOut.profileIndex = self.dataOut.nProfiles - 1
383
391
384 self.profileIndex = self.dataOut.nProfiles
392 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
393 self.profileIndex += self.selBlocksize
394
395 while self.dataOut.data.shape[1] < self.selBlocksize: #Not enough profiles to fill the block
396 if not( self.readNextBlock() ):
397 return 0
398 self.getFirstHeader()
399 self.reshapeData()
400 if self.datablock is None:
401 self.dataOut.flagNoData = True
402 return 0
403 #stack data
404 indMax = self.selBlocksize - self.dataOut.data.shape[1]
405 self.dataOut.data = numpy.hstack((self.dataOut.data,self.datablock[:,:indMax,:]))
406 self.profileIndex = indMax
407
408 self.dataOut.flagDataAsBlock = True
409 self.dataOut.nProfiles = self.selBlocksize
385
410
386 self.dataOut.flagNoData = False
411 self.dataOut.flagNoData = False
387
412
388 self.getBasicHeader()
413 self.getBasicHeader()
389
414
390 self.dataOut.realtime = self.online
415 self.dataOut.realtime = self.online
391
416
392 return self.dataOut.data
417 return self.dataOut.data
393
418
394 class VoltageWriter(JRODataWriter, Operation):
419 class VoltageWriter(JRODataWriter, Operation):
395 """
420 """
396 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
421 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
397 de los datos siempre se realiza por bloques.
422 de los datos siempre se realiza por bloques.
398 """
423 """
399
424
400 ext = ".r"
425 ext = ".r"
401
426
402 optchar = "D"
427 optchar = "D"
403
428
404 shapeBuffer = None
429 shapeBuffer = None
405
430
406
431
407 def __init__(self):
432 def __init__(self):
408 """
433 """
409 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
434 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
410
435
411 Affected:
436 Affected:
412 self.dataOut
437 self.dataOut
413
438
414 Return: None
439 Return: None
415 """
440 """
416 Operation.__init__(self)
441 Operation.__init__(self)
417
442
418 self.nTotalBlocks = 0
443 self.nTotalBlocks = 0
419
444
420 self.profileIndex = 0
445 self.profileIndex = 0
421
446
422 self.isConfig = False
447 self.isConfig = False
423
448
424 self.fp = None
449 self.fp = None
425
450
426 self.flagIsNewFile = 1
451 self.flagIsNewFile = 1
427
452
428 self.blockIndex = 0
453 self.blockIndex = 0
429
454
430 self.flagIsNewBlock = 0
455 self.flagIsNewBlock = 0
431
456
432 self.setFile = None
457 self.setFile = None
433
458
434 self.dtype = None
459 self.dtype = None
435
460
436 self.path = None
461 self.path = None
437
462
438 self.filename = None
463 self.filename = None
439
464
440 self.basicHeaderObj = BasicHeader(LOCALTIME)
465 self.basicHeaderObj = BasicHeader(LOCALTIME)
441
466
442 self.systemHeaderObj = SystemHeader()
467 self.systemHeaderObj = SystemHeader()
443
468
444 self.radarControllerHeaderObj = RadarControllerHeader()
469 self.radarControllerHeaderObj = RadarControllerHeader()
445
470
446 self.processingHeaderObj = ProcessingHeader()
471 self.processingHeaderObj = ProcessingHeader()
447
472
448 def hasAllDataInBuffer(self):
473 def hasAllDataInBuffer(self):
449 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
474 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
450 return 1
475 return 1
451 return 0
476 return 0
452
477
453
478
454 def setBlockDimension(self):
479 def setBlockDimension(self):
455 """
480 """
456 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
481 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
457
482
458 Affected:
483 Affected:
459 self.shape_spc_Buffer
484 self.shape_spc_Buffer
460 self.shape_cspc_Buffer
485 self.shape_cspc_Buffer
461 self.shape_dc_Buffer
486 self.shape_dc_Buffer
462
487
463 Return: None
488 Return: None
464 """
489 """
465 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
490 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
466 self.processingHeaderObj.nHeights,
491 self.processingHeaderObj.nHeights,
467 self.systemHeaderObj.nChannels)
492 self.systemHeaderObj.nChannels)
468
493
469 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
494 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
470 self.processingHeaderObj.profilesPerBlock,
495 self.processingHeaderObj.profilesPerBlock,
471 self.processingHeaderObj.nHeights),
496 self.processingHeaderObj.nHeights),
472 dtype=numpy.dtype('complex64'))
497 dtype=numpy.dtype('complex64'))
473
498
474 def writeBlock(self):
499 def writeBlock(self):
475 """
500 """
476 Escribe el buffer en el file designado
501 Escribe el buffer en el file designado
477
502
478 Affected:
503 Affected:
479 self.profileIndex
504 self.profileIndex
480 self.flagIsNewFile
505 self.flagIsNewFile
481 self.flagIsNewBlock
506 self.flagIsNewBlock
482 self.nTotalBlocks
507 self.nTotalBlocks
483 self.blockIndex
508 self.blockIndex
484
509
485 Return: None
510 Return: None
486 """
511 """
487 data = numpy.zeros( self.shapeBuffer, self.dtype )
512 data = numpy.zeros( self.shapeBuffer, self.dtype )
488
513
489 junk = numpy.transpose(self.datablock, (1,2,0))
514 junk = numpy.transpose(self.datablock, (1,2,0))
490
515
491 data['real'] = junk.real
516 data['real'] = junk.real
492 data['imag'] = junk.imag
517 data['imag'] = junk.imag
493
518
494 data = data.reshape( (-1) )
519 data = data.reshape( (-1) )
495
520
496 data.tofile( self.fp )
521 data.tofile( self.fp )
497
522
498 self.datablock.fill(0)
523 self.datablock.fill(0)
499
524
500 self.profileIndex = 0
525 self.profileIndex = 0
501 self.flagIsNewFile = 0
526 self.flagIsNewFile = 0
502 self.flagIsNewBlock = 1
527 self.flagIsNewBlock = 1
503
528
504 self.blockIndex += 1
529 self.blockIndex += 1
505 self.nTotalBlocks += 1
530 self.nTotalBlocks += 1
506
531
507 # print "[Writing] Block = %04d" %self.blockIndex
532 # print "[Writing] Block = %04d" %self.blockIndex
508
533
509 def putData(self):
534 def putData(self):
510 """
535 """
511 Setea un bloque de datos y luego los escribe en un file
536 Setea un bloque de datos y luego los escribe en un file
512
537
513 Affected:
538 Affected:
514 self.flagIsNewBlock
539 self.flagIsNewBlock
515 self.profileIndex
540 self.profileIndex
516
541
517 Return:
542 Return:
518 0 : Si no hay data o no hay mas files que puedan escribirse
543 0 : Si no hay data o no hay mas files que puedan escribirse
519 1 : Si se escribio la data de un bloque en un file
544 1 : Si se escribio la data de un bloque en un file
520 """
545 """
521 if self.dataOut.flagNoData:
546 if self.dataOut.flagNoData:
522 return 0
547 return 0
523
548
524 self.flagIsNewBlock = 0
549 self.flagIsNewBlock = 0
525
550
526 if self.dataOut.flagDiscontinuousBlock:
551 if self.dataOut.flagDiscontinuousBlock:
527 self.datablock.fill(0)
552 self.datablock.fill(0)
528 self.profileIndex = 0
553 self.profileIndex = 0
529 self.setNextFile()
554 self.setNextFile()
530
555
531 if self.profileIndex == 0:
556 if self.profileIndex == 0:
532 self.setBasicHeader()
557 self.setBasicHeader()
533
558
534 self.datablock[:,self.profileIndex,:] = self.dataOut.data
559 self.datablock[:,self.profileIndex,:] = self.dataOut.data
535
560
536 self.profileIndex += 1
561 self.profileIndex += 1
537
562
538 if self.hasAllDataInBuffer():
563 if self.hasAllDataInBuffer():
539 #if self.flagIsNewFile:
564 #if self.flagIsNewFile:
540 self.writeNextBlock()
565 self.writeNextBlock()
541 # self.setFirstHeader()
566 # self.setFirstHeader()
542
567
543 return 1
568 return 1
544
569
545 def __getBlockSize(self):
570 def __getBlockSize(self):
546 '''
571 '''
547 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
572 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
548 '''
573 '''
549
574
550 dtype_width = self.getDtypeWidth()
575 dtype_width = self.getDtypeWidth()
551
576
552 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
577 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
553
578
554 return blocksize
579 return blocksize
555
580
556 def setFirstHeader(self):
581 def setFirstHeader(self):
557
582
558 """
583 """
559 Obtiene una copia del First Header
584 Obtiene una copia del First Header
560
585
561 Affected:
586 Affected:
562 self.systemHeaderObj
587 self.systemHeaderObj
563 self.radarControllerHeaderObj
588 self.radarControllerHeaderObj
564 self.dtype
589 self.dtype
565
590
566 Return:
591 Return:
567 None
592 None
568 """
593 """
569
594
570 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
595 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
571 self.systemHeaderObj.nChannels = self.dataOut.nChannels
596 self.systemHeaderObj.nChannels = self.dataOut.nChannels
572 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
597 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
573
598
574 self.processingHeaderObj.dtype = 0 # Voltage
599 self.processingHeaderObj.dtype = 0 # Voltage
575 self.processingHeaderObj.blockSize = self.__getBlockSize()
600 self.processingHeaderObj.blockSize = self.__getBlockSize()
576 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
601 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
577 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
602 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
578 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
603 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
579 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
604 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
580 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
605 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
581 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
606 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
582
607
583 if self.dataOut.code is not None:
608 if self.dataOut.code is not None:
584 self.processingHeaderObj.code = self.dataOut.code
609 self.processingHeaderObj.code = self.dataOut.code
585 self.processingHeaderObj.nCode = self.dataOut.nCode
610 self.processingHeaderObj.nCode = self.dataOut.nCode
586 self.processingHeaderObj.nBaud = self.dataOut.nBaud
611 self.processingHeaderObj.nBaud = self.dataOut.nBaud
587
612
588 if self.processingHeaderObj.nWindows != 0:
613 if self.processingHeaderObj.nWindows != 0:
589 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
614 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
590 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
615 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
591 self.processingHeaderObj.nHeights = self.dataOut.nHeights
616 self.processingHeaderObj.nHeights = self.dataOut.nHeights
592 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
617 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
593
618
594 self.processingHeaderObj.processFlags = self.getProcessFlags()
619 self.processingHeaderObj.processFlags = self.getProcessFlags()
595
620
596 self.setBasicHeader() No newline at end of file
621 self.setBasicHeader()
General Comments 0
You need to be logged in to leave comments. Login now