##// END OF EJS Templates
schain-jc
RSS
realtime argument added to JROReader
Miguel Valdez -
r659:33724e2ee44a
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1596 +1,1611
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 traceback.print_exc()
71 traceback.print_exc()
72 raise IOError, "The file %s can't be opened" %(filename)
72 raise IOError, "The file %s can't be opened" %(filename)
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
90 Inputs:
90 Inputs:
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92
92
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94
94
95 endDate : fecha final del rango seleccionado en formato datetime.date
95 endDate : fecha final del rango seleccionado en formato datetime.date
96
96
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98
98
99 endTime : tiempo final del rango seleccionado en formato datetime.time
99 endTime : tiempo final del rango seleccionado en formato datetime.time
100
100
101 Return:
101 Return:
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 fecha especificado, de lo contrario retorna False.
103 fecha especificado, de lo contrario retorna False.
104
104
105 Excepciones:
105 Excepciones:
106 Si el archivo no existe o no puede ser abierto
106 Si el archivo no existe o no puede ser abierto
107 Si la cabecera no puede ser leida.
107 Si la cabecera no puede ser leida.
108
108
109 """
109 """
110
110
111
111
112 try:
112 try:
113 fp = open(filename,'rb')
113 fp = open(filename,'rb')
114 except IOError:
114 except IOError:
115 traceback.print_exc()
115 traceback.print_exc()
116 raise IOError, "The file %s can't be opened" %(filename)
116 raise IOError, "The file %s can't be opened" %(filename)
117
117
118 basicHeaderObj = BasicHeader(LOCALTIME)
118 basicHeaderObj = BasicHeader(LOCALTIME)
119 sts = basicHeaderObj.read(fp)
119 sts = basicHeaderObj.read(fp)
120 fp.close()
120 fp.close()
121
121
122 thisDatetime = basicHeaderObj.datatime
122 thisDatetime = basicHeaderObj.datatime
123 thisDate = thisDatetime.date()
123 thisDate = thisDatetime.date()
124 thisTime = thisDatetime.time()
124 thisTime = thisDatetime.time()
125
125
126 if not(sts):
126 if not(sts):
127 print "Skipping the file %s because it has not a valid header" %(filename)
127 print "Skipping the file %s because it has not a valid header" %(filename)
128 return None
128 return None
129
129
130 #General case
130 #General case
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 #-----------o----------------------------o-----------
132 #-----------o----------------------------o-----------
133 # startTime endTime
133 # startTime endTime
134
134
135 if endTime >= startTime:
135 if endTime >= startTime:
136 if (thisTime < startTime) or (thisTime > endTime):
136 if (thisTime < startTime) or (thisTime > endTime):
137 return None
137 return None
138
138
139 return thisDatetime
139 return thisDatetime
140
140
141 #If endTime < startTime then endTime belongs to the next day
141 #If endTime < startTime then endTime belongs to the next day
142
142
143
143
144 #<<<<<<<<<<<o o>>>>>>>>>>>
144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 #-----------o----------------------------o-----------
145 #-----------o----------------------------o-----------
146 # endTime startTime
146 # endTime startTime
147
147
148 if (thisDate == startDate) and (thisTime < startTime):
148 if (thisDate == startDate) and (thisTime < startTime):
149 return None
149 return None
150
150
151 if (thisDate == endDate) and (thisTime > endTime):
151 if (thisDate == endDate) and (thisTime > endTime):
152 return None
152 return None
153
153
154 if (thisTime < startTime) and (thisTime > endTime):
154 if (thisTime < startTime) and (thisTime > endTime):
155 return None
155 return None
156
156
157 return thisDatetime
157 return thisDatetime
158
158
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 """
160 """
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162
162
163 Inputs:
163 Inputs:
164 folder : nombre completo del directorio.
164 folder : nombre completo del directorio.
165 Su formato deberia ser "/path_root/?YYYYDDD"
165 Su formato deberia ser "/path_root/?YYYYDDD"
166
166
167 siendo:
167 siendo:
168 YYYY : Anio (ejemplo 2015)
168 YYYY : Anio (ejemplo 2015)
169 DDD : Dia del anio (ejemplo 305)
169 DDD : Dia del anio (ejemplo 305)
170
170
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172
172
173 endDate : fecha final del rango seleccionado en formato datetime.date
173 endDate : fecha final del rango seleccionado en formato datetime.date
174
174
175 Return:
175 Return:
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 fecha especificado, de lo contrario retorna False.
177 fecha especificado, de lo contrario retorna False.
178 Excepciones:
178 Excepciones:
179 Si el directorio no tiene el formato adecuado
179 Si el directorio no tiene el formato adecuado
180 """
180 """
181
181
182 basename = os.path.basename(folder)
182 basename = os.path.basename(folder)
183
183
184 if not isRadarFolder(basename):
184 if not isRadarFolder(basename):
185 raise IOError, "The folder %s has not the rigth format" %folder
185 raise IOError, "The folder %s has not the rigth format" %folder
186
186
187 if startDate and endDate:
187 if startDate and endDate:
188 thisDate = getDateFromRadarFolder(basename)
188 thisDate = getDateFromRadarFolder(basename)
189
189
190 if thisDate < startDate:
190 if thisDate < startDate:
191 return 0
191 return 0
192
192
193 if thisDate > endDate:
193 if thisDate > endDate:
194 return 0
194 return 0
195
195
196 return 1
196 return 1
197
197
198 def isFileInDateRange(filename, startDate=None, endDate=None):
198 def isFileInDateRange(filename, startDate=None, endDate=None):
199 """
199 """
200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
201
201
202 Inputs:
202 Inputs:
203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
204
204
205 Su formato deberia ser "?YYYYDDDsss"
205 Su formato deberia ser "?YYYYDDDsss"
206
206
207 siendo:
207 siendo:
208 YYYY : Anio (ejemplo 2015)
208 YYYY : Anio (ejemplo 2015)
209 DDD : Dia del anio (ejemplo 305)
209 DDD : Dia del anio (ejemplo 305)
210 sss : set
210 sss : set
211
211
212 startDate : fecha inicial del rango seleccionado en formato datetime.date
212 startDate : fecha inicial del rango seleccionado en formato datetime.date
213
213
214 endDate : fecha final del rango seleccionado en formato datetime.date
214 endDate : fecha final del rango seleccionado en formato datetime.date
215
215
216 Return:
216 Return:
217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
218 fecha especificado, de lo contrario retorna False.
218 fecha especificado, de lo contrario retorna False.
219 Excepciones:
219 Excepciones:
220 Si el archivo no tiene el formato adecuado
220 Si el archivo no tiene el formato adecuado
221 """
221 """
222
222
223 basename = os.path.basename(filename)
223 basename = os.path.basename(filename)
224
224
225 if not isRadarFile(basename):
225 if not isRadarFile(basename):
226 raise IOError, "The filename %s has not the rigth format" %filename
226 raise IOError, "The filename %s has not the rigth format" %filename
227
227
228 if startDate and endDate:
228 if startDate and endDate:
229 thisDate = getDateFromRadarFile(basename)
229 thisDate = getDateFromRadarFile(basename)
230
230
231 if thisDate < startDate:
231 if thisDate < startDate:
232 return 0
232 return 0
233
233
234 if thisDate > endDate:
234 if thisDate > endDate:
235 return 0
235 return 0
236
236
237 return 1
237 return 1
238
238
239 def getFileFromSet(path, ext, set):
239 def getFileFromSet(path, ext, set):
240 validFilelist = []
240 validFilelist = []
241 fileList = os.listdir(path)
241 fileList = os.listdir(path)
242
242
243 # 0 1234 567 89A BCDE
243 # 0 1234 567 89A BCDE
244 # H YYYY DDD SSS .ext
244 # H YYYY DDD SSS .ext
245
245
246 for thisFile in fileList:
246 for thisFile in fileList:
247 try:
247 try:
248 year = int(thisFile[1:5])
248 year = int(thisFile[1:5])
249 doy = int(thisFile[5:8])
249 doy = int(thisFile[5:8])
250 except:
250 except:
251 continue
251 continue
252
252
253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
254 continue
254 continue
255
255
256 validFilelist.append(thisFile)
256 validFilelist.append(thisFile)
257
257
258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
259
259
260 if len(myfile)!= 0:
260 if len(myfile)!= 0:
261 return myfile[0]
261 return myfile[0]
262 else:
262 else:
263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
264 print 'the filename %s does not exist'%filename
264 print 'the filename %s does not exist'%filename
265 print '...going to the last file: '
265 print '...going to the last file: '
266
266
267 if validFilelist:
267 if validFilelist:
268 validFilelist = sorted( validFilelist, key=str.lower )
268 validFilelist = sorted( validFilelist, key=str.lower )
269 return validFilelist[-1]
269 return validFilelist[-1]
270
270
271 return None
271 return None
272
272
273 def getlastFileFromPath(path, ext):
273 def getlastFileFromPath(path, ext):
274 """
274 """
275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
277
277
278 Input:
278 Input:
279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
280 ext : extension de los files contenidos en una carpeta
280 ext : extension de los files contenidos en una carpeta
281
281
282 Return:
282 Return:
283 El ultimo file de una determinada carpeta, no se considera el path.
283 El ultimo file de una determinada carpeta, no se considera el path.
284 """
284 """
285 validFilelist = []
285 validFilelist = []
286 fileList = os.listdir(path)
286 fileList = os.listdir(path)
287
287
288 # 0 1234 567 89A BCDE
288 # 0 1234 567 89A BCDE
289 # H YYYY DDD SSS .ext
289 # H YYYY DDD SSS .ext
290
290
291 for thisFile in fileList:
291 for thisFile in fileList:
292
292
293 year = thisFile[1:5]
293 year = thisFile[1:5]
294 if not isNumber(year):
294 if not isNumber(year):
295 continue
295 continue
296
296
297 doy = thisFile[5:8]
297 doy = thisFile[5:8]
298 if not isNumber(doy):
298 if not isNumber(doy):
299 continue
299 continue
300
300
301 year = int(year)
301 year = int(year)
302 doy = int(doy)
302 doy = int(doy)
303
303
304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 continue
305 continue
306
306
307 validFilelist.append(thisFile)
307 validFilelist.append(thisFile)
308
308
309 if validFilelist:
309 if validFilelist:
310 validFilelist = sorted( validFilelist, key=str.lower )
310 validFilelist = sorted( validFilelist, key=str.lower )
311 return validFilelist[-1]
311 return validFilelist[-1]
312
312
313 return None
313 return None
314
314
315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
316 """
316 """
317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
319 el path exacto de un determinado file.
319 el path exacto de un determinado file.
320
320
321 Example :
321 Example :
322 nombre correcto del file es .../.../D2009307/P2009307367.ext
322 nombre correcto del file es .../.../D2009307/P2009307367.ext
323
323
324 Entonces la funcion prueba con las siguientes combinaciones
324 Entonces la funcion prueba con las siguientes combinaciones
325 .../.../y2009307367.ext
325 .../.../y2009307367.ext
326 .../.../Y2009307367.ext
326 .../.../Y2009307367.ext
327 .../.../x2009307/y2009307367.ext
327 .../.../x2009307/y2009307367.ext
328 .../.../x2009307/Y2009307367.ext
328 .../.../x2009307/Y2009307367.ext
329 .../.../X2009307/y2009307367.ext
329 .../.../X2009307/y2009307367.ext
330 .../.../X2009307/Y2009307367.ext
330 .../.../X2009307/Y2009307367.ext
331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
332
332
333 Return:
333 Return:
334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
336 para el filename
336 para el filename
337 """
337 """
338 fullfilename = None
338 fullfilename = None
339 find_flag = False
339 find_flag = False
340 filename = None
340 filename = None
341
341
342 prefixDirList = [None,'d','D']
342 prefixDirList = [None,'d','D']
343 if ext.lower() == ".r": #voltage
343 if ext.lower() == ".r": #voltage
344 prefixFileList = ['d','D']
344 prefixFileList = ['d','D']
345 elif ext.lower() == ".pdata": #spectra
345 elif ext.lower() == ".pdata": #spectra
346 prefixFileList = ['p','P']
346 prefixFileList = ['p','P']
347 else:
347 else:
348 return None, filename
348 return None, filename
349
349
350 #barrido por las combinaciones posibles
350 #barrido por las combinaciones posibles
351 for prefixDir in prefixDirList:
351 for prefixDir in prefixDirList:
352 thispath = path
352 thispath = path
353 if prefixDir != None:
353 if prefixDir != None:
354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
355 if foldercounter == 0:
355 if foldercounter == 0:
356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
357 else:
357 else:
358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
362
362
363 if os.path.exists( fullfilename ): #verifico que exista
363 if os.path.exists( fullfilename ): #verifico que exista
364 find_flag = True
364 find_flag = True
365 break
365 break
366 if find_flag:
366 if find_flag:
367 break
367 break
368
368
369 if not(find_flag):
369 if not(find_flag):
370 return None, filename
370 return None, filename
371
371
372 return fullfilename, filename
372 return fullfilename, filename
373
373
374 def isRadarFolder(folder):
374 def isRadarFolder(folder):
375 try:
375 try:
376 year = int(folder[1:5])
376 year = int(folder[1:5])
377 doy = int(folder[5:8])
377 doy = int(folder[5:8])
378 except:
378 except:
379 return 0
379 return 0
380
380
381 return 1
381 return 1
382
382
383 def isRadarFile(file):
383 def isRadarFile(file):
384 try:
384 try:
385 year = int(file[1:5])
385 year = int(file[1:5])
386 doy = int(file[5:8])
386 doy = int(file[5:8])
387 set = int(file[8:11])
387 set = int(file[8:11])
388 except:
388 except:
389 return 0
389 return 0
390
390
391 return 1
391 return 1
392
392
393 def getDateFromRadarFile(file):
393 def getDateFromRadarFile(file):
394 try:
394 try:
395 year = int(file[1:5])
395 year = int(file[1:5])
396 doy = int(file[5:8])
396 doy = int(file[5:8])
397 set = int(file[8:11])
397 set = int(file[8:11])
398 except:
398 except:
399 return None
399 return None
400
400
401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
402 return thisDate
402 return thisDate
403
403
404 def getDateFromRadarFolder(folder):
404 def getDateFromRadarFolder(folder):
405 try:
405 try:
406 year = int(folder[1:5])
406 year = int(folder[1:5])
407 doy = int(folder[5:8])
407 doy = int(folder[5:8])
408 except:
408 except:
409 return None
409 return None
410
410
411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
412 return thisDate
412 return thisDate
413
413
414 class JRODataIO:
414 class JRODataIO:
415
415
416 c = 3E8
416 c = 3E8
417
417
418 isConfig = False
418 isConfig = False
419
419
420 basicHeaderObj = None
420 basicHeaderObj = None
421
421
422 systemHeaderObj = None
422 systemHeaderObj = None
423
423
424 radarControllerHeaderObj = None
424 radarControllerHeaderObj = None
425
425
426 processingHeaderObj = None
426 processingHeaderObj = None
427
427
428 online = 0
429
430 dtype = None
428 dtype = None
431
429
432 pathList = []
430 pathList = []
433
431
434 filenameList = []
432 filenameList = []
435
433
436 filename = None
434 filename = None
437
435
438 ext = None
436 ext = None
439
437
440 flagIsNewFile = 1
438 flagIsNewFile = 1
441
439
442 flagDiscontinuousBlock = 0
440 flagDiscontinuousBlock = 0
443
441
444 flagIsNewBlock = 0
442 flagIsNewBlock = 0
445
443
446 fp = None
444 fp = None
447
445
448 firstHeaderSize = 0
446 firstHeaderSize = 0
449
447
450 basicHeaderSize = 24
448 basicHeaderSize = 24
451
449
452 versionFile = 1103
450 versionFile = 1103
453
451
454 fileSize = None
452 fileSize = None
455
453
456 # ippSeconds = None
454 # ippSeconds = None
457
455
458 fileSizeByHeader = None
456 fileSizeByHeader = None
459
457
460 fileIndex = None
458 fileIndex = None
461
459
462 profileIndex = None
460 profileIndex = None
463
461
464 blockIndex = None
462 blockIndex = None
465
463
466 nTotalBlocks = None
464 nTotalBlocks = None
467
465
468 maxTimeStep = 30
466 maxTimeStep = 30
469
467
470 lastUTTime = None
468 lastUTTime = None
471
469
472 datablock = None
470 datablock = None
473
471
474 dataOut = None
472 dataOut = None
475
473
476 blocksize = None
474 blocksize = None
477
475
478 getByBlock = False
476 getByBlock = False
479
477
480 def __init__(self):
478 def __init__(self):
481
479
482 raise ValueError, "Not implemented"
480 raise ValueError, "Not implemented"
483
481
484 def run(self):
482 def run(self):
485
483
486 raise ValueError, "Not implemented"
484 raise ValueError, "Not implemented"
487
485
488 def getDtypeWidth(self):
486 def getDtypeWidth(self):
489
487
490 dtype_index = get_dtype_index(self.dtype)
488 dtype_index = get_dtype_index(self.dtype)
491 dtype_width = get_dtype_width(dtype_index)
489 dtype_width = get_dtype_width(dtype_index)
492
490
493 return dtype_width
491 return dtype_width
494
492
495 class JRODataReader(JRODataIO):
493 class JRODataReader(JRODataIO):
496
494
495
496 online = 0
497
498 realtime = 0
499
497 nReadBlocks = 0
500 nReadBlocks = 0
498
501
499 delay = 10 #number of seconds waiting a new file
502 delay = 10 #number of seconds waiting a new file
500
503
501 nTries = 3 #quantity tries
504 nTries = 3 #quantity tries
502
505
503 nFiles = 3 #number of files for searching
506 nFiles = 3 #number of files for searching
504
507
505 path = None
508 path = None
506
509
507 foldercounter = 0
510 foldercounter = 0
508
511
509 flagNoMoreFiles = 0
512 flagNoMoreFiles = 0
510
513
511 datetimeList = []
514 datetimeList = []
512
515
513 __isFirstTimeOnline = 1
516 __isFirstTimeOnline = 1
514
517
515 __printInfo = True
518 __printInfo = True
516
519
517 profileIndex = None
520 profileIndex = None
518
521
519 nTxs = 1
522 nTxs = 1
520
523
521 txIndex = None
524 txIndex = None
522
525
523 def __init__(self):
526 def __init__(self):
524
527
525 """
528 """
526
529
527 """
530 """
528
531
529 # raise NotImplementedError, "This method has not been implemented"
532 # raise NotImplementedError, "This method has not been implemented"
530
533
531
534
532 def createObjByDefault(self):
535 def createObjByDefault(self):
533 """
536 """
534
537
535 """
538 """
536 raise NotImplementedError, "This method has not been implemented"
539 raise NotImplementedError, "This method has not been implemented"
537
540
538 def getBlockDimension(self):
541 def getBlockDimension(self):
539
542
540 raise NotImplementedError, "No implemented"
543 raise NotImplementedError, "No implemented"
541
544
542 def __searchFilesOffLine(self,
545 def __searchFilesOffLine(self,
543 path,
546 path,
544 startDate=None,
547 startDate=None,
545 endDate=None,
548 endDate=None,
546 startTime=datetime.time(0,0,0),
549 startTime=datetime.time(0,0,0),
547 endTime=datetime.time(23,59,59),
550 endTime=datetime.time(23,59,59),
548 set=None,
551 set=None,
549 expLabel='',
552 expLabel='',
550 ext='.r',
553 ext='.r',
551 walk=True):
554 walk=True):
552
555
553 self.filenameList = []
556 self.filenameList = []
554 self.datetimeList = []
557 self.datetimeList = []
555
558
556 pathList = []
559 pathList = []
557
560
558 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
561 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
559
562
560 if dateList == []:
563 if dateList == []:
561 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
564 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
562 datetime.datetime.combine(startDate,startTime).ctime(),
565 datetime.datetime.combine(startDate,startTime).ctime(),
563 datetime.datetime.combine(endDate,endTime).ctime())
566 datetime.datetime.combine(endDate,endTime).ctime())
564
567
565 return None, None
568 return None, None
566
569
567 if len(dateList) > 1:
570 if len(dateList) > 1:
568 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
571 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
569 else:
572 else:
570 print "[Reading] data was found for the date %s" %(dateList[0])
573 print "[Reading] data was found for the date %s" %(dateList[0])
571
574
572 filenameList = []
575 filenameList = []
573 datetimeList = []
576 datetimeList = []
574
577
575 for thisPath in pathList:
578 for thisPath in pathList:
576 # thisPath = pathList[pathDict[file]]
579 # thisPath = pathList[pathDict[file]]
577
580
578 fileList = glob.glob1(thisPath, "*%s" %ext)
581 fileList = glob.glob1(thisPath, "*%s" %ext)
579 fileList.sort()
582 fileList.sort()
580
583
581 for file in fileList:
584 for file in fileList:
582
585
583 filename = os.path.join(thisPath,file)
586 filename = os.path.join(thisPath,file)
584
587
585 if not isFileInDateRange(filename, startDate, endDate):
588 if not isFileInDateRange(filename, startDate, endDate):
586 continue
589 continue
587
590
588 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
591 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
589
592
590 if not(thisDatetime):
593 if not(thisDatetime):
591 continue
594 continue
592
595
593 filenameList.append(filename)
596 filenameList.append(filename)
594 datetimeList.append(thisDatetime)
597 datetimeList.append(thisDatetime)
595
598
596 if not(filenameList):
599 if not(filenameList):
597 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
600 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
598 return None, None
601 return None, None
599
602
600 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
603 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
601 print
604 print
602
605
603 for i in range(len(filenameList)):
606 for i in range(len(filenameList)):
604 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
607 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
605
608
606 self.filenameList = filenameList
609 self.filenameList = filenameList
607 self.datetimeList = datetimeList
610 self.datetimeList = datetimeList
608
611
609 return pathList, filenameList
612 return pathList, filenameList
610
613
611 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
614 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
612
615
613 """
616 """
614 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
617 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
615 devuelve el archivo encontrado ademas de otros datos.
618 devuelve el archivo encontrado ademas de otros datos.
616
619
617 Input:
620 Input:
618 path : carpeta donde estan contenidos los files que contiene data
621 path : carpeta donde estan contenidos los files que contiene data
619
622
620 expLabel : Nombre del subexperimento (subfolder)
623 expLabel : Nombre del subexperimento (subfolder)
621
624
622 ext : extension de los files
625 ext : extension de los files
623
626
624 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
627 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
625
628
626 Return:
629 Return:
627 directory : eL directorio donde esta el file encontrado
630 directory : eL directorio donde esta el file encontrado
628 filename : el ultimo file de una determinada carpeta
631 filename : el ultimo file de una determinada carpeta
629 year : el anho
632 year : el anho
630 doy : el numero de dia del anho
633 doy : el numero de dia del anho
631 set : el set del archivo
634 set : el set del archivo
632
635
633
636
634 """
637 """
635 dirList = []
638 dirList = []
636
639
637 if not walk:
640 if not walk:
638 fullpath = path
641 fullpath = path
639 foldercounter = 0
642 foldercounter = 0
640 else:
643 else:
641 #Filtra solo los directorios
644 #Filtra solo los directorios
642 for thisPath in os.listdir(path):
645 for thisPath in os.listdir(path):
643 if not os.path.isdir(os.path.join(path,thisPath)):
646 if not os.path.isdir(os.path.join(path,thisPath)):
644 continue
647 continue
645 if not isRadarFolder(thisPath):
648 if not isRadarFolder(thisPath):
646 continue
649 continue
647
650
648 dirList.append(thisPath)
651 dirList.append(thisPath)
649
652
650 if not(dirList):
653 if not(dirList):
651 return None, None, None, None, None, None
654 return None, None, None, None, None, None
652
655
653 dirList = sorted( dirList, key=str.lower )
656 dirList = sorted( dirList, key=str.lower )
654
657
655 doypath = dirList[-1]
658 doypath = dirList[-1]
656 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
659 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
657 fullpath = os.path.join(path, doypath, expLabel)
660 fullpath = os.path.join(path, doypath, expLabel)
658
661
659
662
660 print "[Reading] %s folder was found: " %(fullpath )
663 print "[Reading] %s folder was found: " %(fullpath )
661
664
662 if set == None:
665 if set == None:
663 filename = getlastFileFromPath(fullpath, ext)
666 filename = getlastFileFromPath(fullpath, ext)
664 else:
667 else:
665 filename = getFileFromSet(fullpath, ext, set)
668 filename = getFileFromSet(fullpath, ext, set)
666
669
667 if not(filename):
670 if not(filename):
668 return None, None, None, None, None, None
671 return None, None, None, None, None, None
669
672
670 print "[Reading] %s file was found" %(filename)
673 print "[Reading] %s file was found" %(filename)
671
674
672 if not(self.__verifyFile(os.path.join(fullpath, filename))):
675 if not(self.__verifyFile(os.path.join(fullpath, filename))):
673 return None, None, None, None, None, None
676 return None, None, None, None, None, None
674
677
675 year = int( filename[1:5] )
678 year = int( filename[1:5] )
676 doy = int( filename[5:8] )
679 doy = int( filename[5:8] )
677 set = int( filename[8:11] )
680 set = int( filename[8:11] )
678
681
679 return fullpath, foldercounter, filename, year, doy, set
682 return fullpath, foldercounter, filename, year, doy, set
680
683
681 def __setNextFileOffline(self):
684 def __setNextFileOffline(self):
682
685
683 idFile = self.fileIndex
686 idFile = self.fileIndex
684
687
685 while (True):
688 while (True):
686 idFile += 1
689 idFile += 1
687 if not(idFile < len(self.filenameList)):
690 if not(idFile < len(self.filenameList)):
688 self.flagNoMoreFiles = 1
691 self.flagNoMoreFiles = 1
689 # print "[Reading] No more Files"
692 # print "[Reading] No more Files"
690 return 0
693 return 0
691
694
692 filename = self.filenameList[idFile]
695 filename = self.filenameList[idFile]
693
696
694 if not(self.__verifyFile(filename)):
697 if not(self.__verifyFile(filename)):
695 continue
698 continue
696
699
697 fileSize = os.path.getsize(filename)
700 fileSize = os.path.getsize(filename)
698 fp = open(filename,'rb')
701 fp = open(filename,'rb')
699 break
702 break
700
703
701 self.flagIsNewFile = 1
704 self.flagIsNewFile = 1
702 self.fileIndex = idFile
705 self.fileIndex = idFile
703 self.filename = filename
706 self.filename = filename
704 self.fileSize = fileSize
707 self.fileSize = fileSize
705 self.fp = fp
708 self.fp = fp
706
709
707 # print "[Reading] Setting the file: %s"%self.filename
710 # print "[Reading] Setting the file: %s"%self.filename
708
711
709 return 1
712 return 1
710
713
711 def __setNextFileOnline(self):
714 def __setNextFileOnline(self):
712 """
715 """
713 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
716 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
714 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
717 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
715 siguientes.
718 siguientes.
716
719
717 Affected:
720 Affected:
718 self.flagIsNewFile
721 self.flagIsNewFile
719 self.filename
722 self.filename
720 self.fileSize
723 self.fileSize
721 self.fp
724 self.fp
722 self.set
725 self.set
723 self.flagNoMoreFiles
726 self.flagNoMoreFiles
724
727
725 Return:
728 Return:
726 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
729 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
727 1 : si el file fue abierto con exito y esta listo a ser leido
730 1 : si el file fue abierto con exito y esta listo a ser leido
728
731
729 Excepciones:
732 Excepciones:
730 Si un determinado file no puede ser abierto
733 Si un determinado file no puede ser abierto
731 """
734 """
732 nFiles = 0
735 nFiles = 0
733 fileOk_flag = False
736 fileOk_flag = False
734 firstTime_flag = True
737 firstTime_flag = True
735
738
736 self.set += 1
739 self.set += 1
737
740
738 if self.set > 999:
741 if self.set > 999:
739 self.set = 0
742 self.set = 0
740 self.foldercounter += 1
743 self.foldercounter += 1
741
744
742 #busca el 1er file disponible
745 #busca el 1er file disponible
743 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
746 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
744 if fullfilename:
747 if fullfilename:
745 if self.__verifyFile(fullfilename, False):
748 if self.__verifyFile(fullfilename, False):
746 fileOk_flag = True
749 fileOk_flag = True
747
750
748 #si no encuentra un file entonces espera y vuelve a buscar
751 #si no encuentra un file entonces espera y vuelve a buscar
749 if not(fileOk_flag):
752 if not(fileOk_flag):
750 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
753 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
751
754
752 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
755 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
753 tries = self.nTries
756 tries = self.nTries
754 else:
757 else:
755 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
758 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
756
759
757 for nTries in range( tries ):
760 for nTries in range( tries ):
758 if firstTime_flag:
761 if firstTime_flag:
759 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
762 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
760 sleep( self.delay )
763 sleep( self.delay )
761 else:
764 else:
762 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
765 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
763
766
764 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
767 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
765 if fullfilename:
768 if fullfilename:
766 if self.__verifyFile(fullfilename):
769 if self.__verifyFile(fullfilename):
767 fileOk_flag = True
770 fileOk_flag = True
768 break
771 break
769
772
770 if fileOk_flag:
773 if fileOk_flag:
771 break
774 break
772
775
773 firstTime_flag = False
776 firstTime_flag = False
774
777
775 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
778 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
776 self.set += 1
779 self.set += 1
777
780
778 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
781 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
779 self.set = 0
782 self.set = 0
780 self.doy += 1
783 self.doy += 1
781 self.foldercounter = 0
784 self.foldercounter = 0
782
785
783 if fileOk_flag:
786 if fileOk_flag:
784 self.fileSize = os.path.getsize( fullfilename )
787 self.fileSize = os.path.getsize( fullfilename )
785 self.filename = fullfilename
788 self.filename = fullfilename
786 self.flagIsNewFile = 1
789 self.flagIsNewFile = 1
787 if self.fp != None: self.fp.close()
790 if self.fp != None: self.fp.close()
788 self.fp = open(fullfilename, 'rb')
791 self.fp = open(fullfilename, 'rb')
789 self.flagNoMoreFiles = 0
792 self.flagNoMoreFiles = 0
790 # print '[Reading] Setting the file: %s' % fullfilename
793 # print '[Reading] Setting the file: %s' % fullfilename
791 else:
794 else:
792 self.fileSize = 0
795 self.fileSize = 0
793 self.filename = None
796 self.filename = None
794 self.flagIsNewFile = 0
797 self.flagIsNewFile = 0
795 self.fp = None
798 self.fp = None
796 self.flagNoMoreFiles = 1
799 self.flagNoMoreFiles = 1
797 # print '[Reading] No more files to read'
800 # print '[Reading] No more files to read'
798
801
799 return fileOk_flag
802 return fileOk_flag
800
803
801 def setNextFile(self):
804 def setNextFile(self):
802 if self.fp != None:
805 if self.fp != None:
803 self.fp.close()
806 self.fp.close()
804
807
805 if self.online:
808 if self.online:
806 newFile = self.__setNextFileOnline()
809 newFile = self.__setNextFileOnline()
807 else:
810 else:
808 newFile = self.__setNextFileOffline()
811 newFile = self.__setNextFileOffline()
809
812
810 if not(newFile):
813 if not(newFile):
811 print '[Reading] No more files to read'
814 print '[Reading] No more files to read'
812 return 0
815 return 0
813
816
814 print '[Reading] Setting the file: %s' % self.filename
817 print '[Reading] Setting the file: %s' % self.filename
815
818
816 self.__readFirstHeader()
819 self.__readFirstHeader()
817 self.nReadBlocks = 0
820 self.nReadBlocks = 0
818 return 1
821 return 1
819
822
820 def __waitNewBlock(self):
823 def __waitNewBlock(self):
821 """
824 """
822 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
825 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
823
826
824 Si el modo de lectura es OffLine siempre retorn 0
827 Si el modo de lectura es OffLine siempre retorn 0
825 """
828 """
826 if not self.online:
829 if not self.online:
827 return 0
830 return 0
828
831
829 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
832 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
830 return 0
833 return 0
831
834
832 currentPointer = self.fp.tell()
835 currentPointer = self.fp.tell()
833
836
834 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
837 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
835
838
836 for nTries in range( self.nTries ):
839 for nTries in range( self.nTries ):
837
840
838 self.fp.close()
841 self.fp.close()
839 self.fp = open( self.filename, 'rb' )
842 self.fp = open( self.filename, 'rb' )
840 self.fp.seek( currentPointer )
843 self.fp.seek( currentPointer )
841
844
842 self.fileSize = os.path.getsize( self.filename )
845 self.fileSize = os.path.getsize( self.filename )
843 currentSize = self.fileSize - currentPointer
846 currentSize = self.fileSize - currentPointer
844
847
845 if ( currentSize >= neededSize ):
848 if ( currentSize >= neededSize ):
846 self.basicHeaderObj.read(self.fp)
849 self.basicHeaderObj.read(self.fp)
847 return 1
850 return 1
848
851
849 if self.fileSize == self.fileSizeByHeader:
852 if self.fileSize == self.fileSizeByHeader:
850 # self.flagEoF = True
853 # self.flagEoF = True
851 return 0
854 return 0
852
855
853 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
856 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
854 sleep( self.delay )
857 sleep( self.delay )
855
858
856
859
857 return 0
860 return 0
858
861
859 def waitDataBlock(self,pointer_location):
862 def waitDataBlock(self,pointer_location):
860
863
861 currentPointer = pointer_location
864 currentPointer = pointer_location
862
865
863 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
866 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
864
867
865 for nTries in range( self.nTries ):
868 for nTries in range( self.nTries ):
866 self.fp.close()
869 self.fp.close()
867 self.fp = open( self.filename, 'rb' )
870 self.fp = open( self.filename, 'rb' )
868 self.fp.seek( currentPointer )
871 self.fp.seek( currentPointer )
869
872
870 self.fileSize = os.path.getsize( self.filename )
873 self.fileSize = os.path.getsize( self.filename )
871 currentSize = self.fileSize - currentPointer
874 currentSize = self.fileSize - currentPointer
872
875
873 if ( currentSize >= neededSize ):
876 if ( currentSize >= neededSize ):
874 return 1
877 return 1
875
878
876 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
879 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
877 sleep( self.delay )
880 sleep( self.delay )
878
881
879 return 0
882 return 0
880
883
881 def __jumpToLastBlock(self):
884 def __jumpToLastBlock(self):
882
885
883 if not(self.__isFirstTimeOnline):
886 if not(self.__isFirstTimeOnline):
884 return
887 return
885
888
886 csize = self.fileSize - self.fp.tell()
889 csize = self.fileSize - self.fp.tell()
887 blocksize = self.processingHeaderObj.blockSize
890 blocksize = self.processingHeaderObj.blockSize
888
891
889 #salta el primer bloque de datos
892 #salta el primer bloque de datos
890 if csize > self.processingHeaderObj.blockSize:
893 if csize > self.processingHeaderObj.blockSize:
891 self.fp.seek(self.fp.tell() + blocksize)
894 self.fp.seek(self.fp.tell() + blocksize)
892 else:
895 else:
893 return
896 return
894
897
895 csize = self.fileSize - self.fp.tell()
898 csize = self.fileSize - self.fp.tell()
896 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
899 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
897 while True:
900 while True:
898
901
899 if self.fp.tell()<self.fileSize:
902 if self.fp.tell()<self.fileSize:
900 self.fp.seek(self.fp.tell() + neededsize)
903 self.fp.seek(self.fp.tell() + neededsize)
901 else:
904 else:
902 self.fp.seek(self.fp.tell() - neededsize)
905 self.fp.seek(self.fp.tell() - neededsize)
903 break
906 break
904
907
905 # csize = self.fileSize - self.fp.tell()
908 # csize = self.fileSize - self.fp.tell()
906 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
909 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
907 # factor = int(csize/neededsize)
910 # factor = int(csize/neededsize)
908 # if factor > 0:
911 # if factor > 0:
909 # self.fp.seek(self.fp.tell() + factor*neededsize)
912 # self.fp.seek(self.fp.tell() + factor*neededsize)
910
913
911 self.flagIsNewFile = 0
914 self.flagIsNewFile = 0
912 self.__isFirstTimeOnline = 0
915 self.__isFirstTimeOnline = 0
913
916
914 def __setNewBlock(self):
917 def __setNewBlock(self):
915
918
916 if self.fp == None:
919 if self.fp == None:
917 return 0
920 return 0
918
921
919 if self.online:
922 # if self.online:
920 self.__jumpToLastBlock()
923 # self.__jumpToLastBlock()
921
924
922 if self.flagIsNewFile:
925 if self.flagIsNewFile:
926 self.lastUTTime = self.basicHeaderObj.utc
923 return 1
927 return 1
924
928
925 self.lastUTTime = self.basicHeaderObj.utc
929 if self.realtime:
930 self.flagDiscontinuousBlock = 1
931 if not(self.setNextFile()):
932 return 0
933 else:
934 return 1
935
926 currentSize = self.fileSize - self.fp.tell()
936 currentSize = self.fileSize - self.fp.tell()
927 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
937 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
928
938
929 if (currentSize >= neededSize):
939 if (currentSize >= neededSize):
930 self.basicHeaderObj.read(self.fp)
940 self.basicHeaderObj.read(self.fp)
941 self.lastUTTime = self.basicHeaderObj.utc
931 return 1
942 return 1
932
943
933 if self.__waitNewBlock():
944 if self.__waitNewBlock():
945 self.lastUTTime = self.basicHeaderObj.utc
934 return 1
946 return 1
935
947
936 if not(self.setNextFile()):
948 if not(self.setNextFile()):
937 return 0
949 return 0
938
950
939 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
951 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
952 self.lastUTTime = self.basicHeaderObj.utc
940
953
941 self.flagDiscontinuousBlock = 0
954 self.flagDiscontinuousBlock = 0
942
955
943 if deltaTime > self.maxTimeStep:
956 if deltaTime > self.maxTimeStep:
944 self.flagDiscontinuousBlock = 1
957 self.flagDiscontinuousBlock = 1
945
958
946 return 1
959 return 1
947
960
948 def readNextBlock(self):
961 def readNextBlock(self):
949
962
950 if not(self.__setNewBlock()):
963 if not(self.__setNewBlock()):
951 return 0
964 return 0
952
965
953 if not(self.readBlock()):
966 if not(self.readBlock()):
954 return 0
967 return 0
955
968
956 self.getBasicHeader()
969 self.getBasicHeader()
957
970
958 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
971 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
959 self.processingHeaderObj.dataBlocksPerFile,
972 self.processingHeaderObj.dataBlocksPerFile,
960 self.dataOut.datatime.ctime())
973 self.dataOut.datatime.ctime())
961 return 1
974 return 1
962
975
963 def __readFirstHeader(self):
976 def __readFirstHeader(self):
964
977
965 self.basicHeaderObj.read(self.fp)
978 self.basicHeaderObj.read(self.fp)
966 self.systemHeaderObj.read(self.fp)
979 self.systemHeaderObj.read(self.fp)
967 self.radarControllerHeaderObj.read(self.fp)
980 self.radarControllerHeaderObj.read(self.fp)
968 self.processingHeaderObj.read(self.fp)
981 self.processingHeaderObj.read(self.fp)
969
982
970 self.firstHeaderSize = self.basicHeaderObj.size
983 self.firstHeaderSize = self.basicHeaderObj.size
971
984
972 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
985 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
973 if datatype == 0:
986 if datatype == 0:
974 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
987 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
975 elif datatype == 1:
988 elif datatype == 1:
976 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
989 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
977 elif datatype == 2:
990 elif datatype == 2:
978 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
991 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
979 elif datatype == 3:
992 elif datatype == 3:
980 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
993 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
981 elif datatype == 4:
994 elif datatype == 4:
982 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
995 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
983 elif datatype == 5:
996 elif datatype == 5:
984 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
997 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
985 else:
998 else:
986 raise ValueError, 'Data type was not defined'
999 raise ValueError, 'Data type was not defined'
987
1000
988 self.dtype = datatype_str
1001 self.dtype = datatype_str
989 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1002 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
990 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1003 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
991 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1004 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
992 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1005 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
993 self.getBlockDimension()
1006 self.getBlockDimension()
994
1007
995 def __verifyFile(self, filename, msgFlag=True):
1008 def __verifyFile(self, filename, msgFlag=True):
996 msg = None
1009 msg = None
997 try:
1010 try:
998 fp = open(filename, 'rb')
1011 fp = open(filename, 'rb')
999 currentPosition = fp.tell()
1012 currentPosition = fp.tell()
1000 except IOError:
1013 except IOError:
1001 traceback.print_exc()
1014 traceback.print_exc()
1002 if msgFlag:
1015 if msgFlag:
1003 print "[Reading] The file %s can't be opened" % (filename)
1016 print "[Reading] The file %s can't be opened" % (filename)
1004 return False
1017 return False
1005
1018
1006 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1019 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1007
1020
1008 if neededSize == 0:
1021 if neededSize == 0:
1009 basicHeaderObj = BasicHeader(LOCALTIME)
1022 basicHeaderObj = BasicHeader(LOCALTIME)
1010 systemHeaderObj = SystemHeader()
1023 systemHeaderObj = SystemHeader()
1011 radarControllerHeaderObj = RadarControllerHeader()
1024 radarControllerHeaderObj = RadarControllerHeader()
1012 processingHeaderObj = ProcessingHeader()
1025 processingHeaderObj = ProcessingHeader()
1013
1026
1014 try:
1027 try:
1015 if not( basicHeaderObj.read(fp) ): raise IOError
1028 if not( basicHeaderObj.read(fp) ): raise IOError
1016 if not( systemHeaderObj.read(fp) ): raise IOError
1029 if not( systemHeaderObj.read(fp) ): raise IOError
1017 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1030 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1018 if not( processingHeaderObj.read(fp) ): raise IOError
1031 if not( processingHeaderObj.read(fp) ): raise IOError
1019 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1032 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1020
1033
1021 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1034 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1022
1035
1023 except IOError:
1036 except IOError:
1024 traceback.print_exc()
1037 traceback.print_exc()
1025 sys.exit(0)
1038 sys.exit(0)
1026
1039
1027 if msgFlag:
1040 if msgFlag:
1028 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1041 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1029
1042
1030 fp.close()
1043 fp.close()
1031 return False
1044 return False
1032 else:
1045 else:
1033 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1046 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1034
1047
1035 fp.close()
1048 fp.close()
1036 fileSize = os.path.getsize(filename)
1049 fileSize = os.path.getsize(filename)
1037 currentSize = fileSize - currentPosition
1050 currentSize = fileSize - currentPosition
1038 if currentSize < neededSize:
1051 if currentSize < neededSize:
1039 if msgFlag and (msg != None):
1052 if msgFlag and (msg != None):
1040 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1053 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1041 return False
1054 return False
1042
1055
1043 return True
1056 return True
1044
1057
1045 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1058 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1046
1059
1047 dateList = []
1060 dateList = []
1048 pathList = []
1061 pathList = []
1049
1062
1050 multi_path = path.split(',')
1063 multi_path = path.split(',')
1051
1064
1052 if not walk:
1065 if not walk:
1053
1066
1054 for single_path in multi_path:
1067 for single_path in multi_path:
1055
1068
1056 if not os.path.isdir(single_path):
1069 if not os.path.isdir(single_path):
1057 continue
1070 continue
1058
1071
1059 fileList = glob.glob1(single_path, "*"+ext)
1072 fileList = glob.glob1(single_path, "*"+ext)
1060
1073
1061 for thisFile in fileList:
1074 for thisFile in fileList:
1062
1075
1063 if not os.path.isfile(os.path.join(single_path, thisFile)):
1076 if not os.path.isfile(os.path.join(single_path, thisFile)):
1064 continue
1077 continue
1065
1078
1066 if not isRadarFile(thisFile):
1079 if not isRadarFile(thisFile):
1067 continue
1080 continue
1068
1081
1069 if not isFileInDateRange(thisFile, startDate, endDate):
1082 if not isFileInDateRange(thisFile, startDate, endDate):
1070 continue
1083 continue
1071
1084
1072 thisDate = getDateFromRadarFile(thisFile)
1085 thisDate = getDateFromRadarFile(thisFile)
1073
1086
1074 if thisDate in dateList:
1087 if thisDate in dateList:
1075 continue
1088 continue
1076
1089
1077 dateList.append(thisDate)
1090 dateList.append(thisDate)
1078 pathList.append(single_path)
1091 pathList.append(single_path)
1079
1092
1080 else:
1093 else:
1081 for single_path in multi_path:
1094 for single_path in multi_path:
1082
1095
1083 if not os.path.isdir(single_path):
1096 if not os.path.isdir(single_path):
1084 continue
1097 continue
1085
1098
1086 dirList = []
1099 dirList = []
1087
1100
1088 for thisPath in os.listdir(single_path):
1101 for thisPath in os.listdir(single_path):
1089
1102
1090 if not os.path.isdir(os.path.join(single_path,thisPath)):
1103 if not os.path.isdir(os.path.join(single_path,thisPath)):
1091 continue
1104 continue
1092
1105
1093 if not isRadarFolder(thisPath):
1106 if not isRadarFolder(thisPath):
1094 continue
1107 continue
1095
1108
1096 if not isFolderInDateRange(thisPath, startDate, endDate):
1109 if not isFolderInDateRange(thisPath, startDate, endDate):
1097 continue
1110 continue
1098
1111
1099 dirList.append(thisPath)
1112 dirList.append(thisPath)
1100
1113
1101 if not dirList:
1114 if not dirList:
1102 continue
1115 continue
1103
1116
1104 for thisDir in dirList:
1117 for thisDir in dirList:
1105
1118
1106 datapath = os.path.join(single_path, thisDir, expLabel)
1119 datapath = os.path.join(single_path, thisDir, expLabel)
1107 fileList = glob.glob1(datapath, "*"+ext)
1120 fileList = glob.glob1(datapath, "*"+ext)
1108
1121
1109 if len(fileList) < 1:
1122 if len(fileList) < 1:
1110 continue
1123 continue
1111
1124
1112 thisDate = getDateFromRadarFolder(thisDir)
1125 thisDate = getDateFromRadarFolder(thisDir)
1113
1126
1114 pathList.append(datapath)
1127 pathList.append(datapath)
1115 dateList.append(thisDate)
1128 dateList.append(thisDate)
1116
1129
1117 dateList.sort()
1130 dateList.sort()
1118
1131
1119 if include_path:
1132 if include_path:
1120 return dateList, pathList
1133 return dateList, pathList
1121
1134
1122 return dateList
1135 return dateList
1123
1136
1124 def setup(self,
1137 def setup(self,
1125 path=None,
1138 path=None,
1126 startDate=None,
1139 startDate=None,
1127 endDate=None,
1140 endDate=None,
1128 startTime=datetime.time(0,0,0),
1141 startTime=datetime.time(0,0,0),
1129 endTime=datetime.time(23,59,59),
1142 endTime=datetime.time(23,59,59),
1130 set=None,
1143 set=None,
1131 expLabel = "",
1144 expLabel = "",
1132 ext = None,
1145 ext = None,
1133 online = False,
1146 online = False,
1134 delay = 60,
1147 delay = 60,
1135 walk = True,
1148 walk = True,
1136 getblock = False,
1149 getblock = False,
1137 nTxs = 1):
1150 nTxs = 1,
1151 realtime=False):
1138
1152
1139 if path == None:
1153 if path == None:
1140 raise ValueError, "[Reading] The path is not valid"
1154 raise ValueError, "[Reading] The path is not valid"
1141
1155
1142 if ext == None:
1156 if ext == None:
1143 ext = self.ext
1157 ext = self.ext
1144
1158
1145 if online:
1159 if online:
1146 print "[Reading] Searching files in online mode..."
1160 print "[Reading] Searching files in online mode..."
1147
1161
1148 for nTries in range( self.nTries ):
1162 for nTries in range( self.nTries ):
1149 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1163 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1150
1164
1151 if fullpath:
1165 if fullpath:
1152 break
1166 break
1153
1167
1154 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1168 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1155 sleep( self.delay )
1169 sleep( self.delay )
1156
1170
1157 if not(fullpath):
1171 if not(fullpath):
1158 print "[Reading] There 'isn't any valid file in %s" % path
1172 print "[Reading] There 'isn't any valid file in %s" % path
1159 return None
1173 return None
1160
1174
1161 self.year = year
1175 self.year = year
1162 self.doy = doy
1176 self.doy = doy
1163 self.set = set - 1
1177 self.set = set - 1
1164 self.path = path
1178 self.path = path
1165 self.foldercounter = foldercounter
1179 self.foldercounter = foldercounter
1166 last_set = None
1180 last_set = None
1167
1181
1168 else:
1182 else:
1169 print "[Reading] Searching files in offline mode ..."
1183 print "[Reading] Searching files in offline mode ..."
1170 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1184 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1171 startTime=startTime, endTime=endTime,
1185 startTime=startTime, endTime=endTime,
1172 set=set, expLabel=expLabel, ext=ext,
1186 set=set, expLabel=expLabel, ext=ext,
1173 walk=walk)
1187 walk=walk)
1174
1188
1175 if not(pathList):
1189 if not(pathList):
1176 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1190 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1177 # datetime.datetime.combine(startDate,startTime).ctime(),
1191 # datetime.datetime.combine(startDate,startTime).ctime(),
1178 # datetime.datetime.combine(endDate,endTime).ctime())
1192 # datetime.datetime.combine(endDate,endTime).ctime())
1179 #
1193 #
1180 sys.exit(-1)
1194 sys.exit(-1)
1181
1195
1182
1196
1183 self.fileIndex = -1
1197 self.fileIndex = -1
1184 self.pathList = pathList
1198 self.pathList = pathList
1185 self.filenameList = filenameList
1199 self.filenameList = filenameList
1186 file_name = os.path.basename(filenameList[-1])
1200 file_name = os.path.basename(filenameList[-1])
1187 basename, ext = os.path.splitext(file_name)
1201 basename, ext = os.path.splitext(file_name)
1188 last_set = int(basename[-3:])
1202 last_set = int(basename[-3:])
1189
1203
1190 self.online = online
1204 self.online = online
1205 self.realtime = realtime
1191 self.delay = delay
1206 self.delay = delay
1192 ext = ext.lower()
1207 ext = ext.lower()
1193 self.ext = ext
1208 self.ext = ext
1194 self.getByBlock = getblock
1209 self.getByBlock = getblock
1195 self.nTxs = int(nTxs)
1210 self.nTxs = int(nTxs)
1196
1211
1197 if not(self.setNextFile()):
1212 if not(self.setNextFile()):
1198 if (startDate!=None) and (endDate!=None):
1213 if (startDate!=None) and (endDate!=None):
1199 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1214 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1200 elif startDate != None:
1215 elif startDate != None:
1201 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1216 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1202 else:
1217 else:
1203 print "[Reading] No files"
1218 print "[Reading] No files"
1204
1219
1205 sys.exit(-1)
1220 sys.exit(-1)
1206
1221
1207 # self.getBasicHeader()
1222 # self.getBasicHeader()
1208
1223
1209 if last_set != None:
1224 if last_set != None:
1210 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1225 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1211 return
1226 return
1212
1227
1213 def getBasicHeader(self):
1228 def getBasicHeader(self):
1214
1229
1215 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1230 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1216
1231
1217 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1232 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1218
1233
1219 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1234 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1220
1235
1221 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1236 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1222
1237
1223 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1238 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1224
1239
1225 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1240 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1226
1241
1227 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1242 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1228
1243
1229 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1244 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1230
1245
1231
1246
1232 def getFirstHeader(self):
1247 def getFirstHeader(self):
1233
1248
1234 raise ValueError, "This method has not been implemented"
1249 raise ValueError, "This method has not been implemented"
1235
1250
1236 def getData(self):
1251 def getData(self):
1237
1252
1238 raise ValueError, "This method has not been implemented"
1253 raise ValueError, "This method has not been implemented"
1239
1254
1240 def hasNotDataInBuffer(self):
1255 def hasNotDataInBuffer(self):
1241
1256
1242 raise ValueError, "This method has not been implemented"
1257 raise ValueError, "This method has not been implemented"
1243
1258
1244 def readBlock(self):
1259 def readBlock(self):
1245
1260
1246 raise ValueError, "This method has not been implemented"
1261 raise ValueError, "This method has not been implemented"
1247
1262
1248 def isEndProcess(self):
1263 def isEndProcess(self):
1249
1264
1250 return self.flagNoMoreFiles
1265 return self.flagNoMoreFiles
1251
1266
1252 def printReadBlocks(self):
1267 def printReadBlocks(self):
1253
1268
1254 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1269 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1255
1270
1256 def printTotalBlocks(self):
1271 def printTotalBlocks(self):
1257
1272
1258 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1273 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1259
1274
1260 def printNumberOfBlock(self):
1275 def printNumberOfBlock(self):
1261
1276
1262 if self.flagIsNewBlock:
1277 if self.flagIsNewBlock:
1263 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1278 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1264 self.processingHeaderObj.dataBlocksPerFile,
1279 self.processingHeaderObj.dataBlocksPerFile,
1265 self.dataOut.datatime.ctime())
1280 self.dataOut.datatime.ctime())
1266
1281
1267 def printInfo(self):
1282 def printInfo(self):
1268
1283
1269 if self.__printInfo == False:
1284 if self.__printInfo == False:
1270 return
1285 return
1271
1286
1272 self.basicHeaderObj.printInfo()
1287 self.basicHeaderObj.printInfo()
1273 self.systemHeaderObj.printInfo()
1288 self.systemHeaderObj.printInfo()
1274 self.radarControllerHeaderObj.printInfo()
1289 self.radarControllerHeaderObj.printInfo()
1275 self.processingHeaderObj.printInfo()
1290 self.processingHeaderObj.printInfo()
1276
1291
1277 self.__printInfo = False
1292 self.__printInfo = False
1278
1293
1279
1294
1280 def run(self, **kwargs):
1295 def run(self, **kwargs):
1281
1296
1282 if not(self.isConfig):
1297 if not(self.isConfig):
1283
1298
1284 # self.dataOut = dataOut
1299 # self.dataOut = dataOut
1285 self.setup(**kwargs)
1300 self.setup(**kwargs)
1286 self.isConfig = True
1301 self.isConfig = True
1287
1302
1288 self.getData()
1303 self.getData()
1289
1304
1290 class JRODataWriter(JRODataIO):
1305 class JRODataWriter(JRODataIO):
1291
1306
1292 """
1307 """
1293 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1308 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1294 de los datos siempre se realiza por bloques.
1309 de los datos siempre se realiza por bloques.
1295 """
1310 """
1296
1311
1297 blockIndex = 0
1312 blockIndex = 0
1298
1313
1299 path = None
1314 path = None
1300
1315
1301 setFile = None
1316 setFile = None
1302
1317
1303 profilesPerBlock = None
1318 profilesPerBlock = None
1304
1319
1305 blocksPerFile = None
1320 blocksPerFile = None
1306
1321
1307 nWriteBlocks = 0
1322 nWriteBlocks = 0
1308
1323
1309 fileDate = None
1324 fileDate = None
1310
1325
1311 def __init__(self, dataOut=None):
1326 def __init__(self, dataOut=None):
1312 raise ValueError, "Not implemented"
1327 raise ValueError, "Not implemented"
1313
1328
1314
1329
1315 def hasAllDataInBuffer(self):
1330 def hasAllDataInBuffer(self):
1316 raise ValueError, "Not implemented"
1331 raise ValueError, "Not implemented"
1317
1332
1318
1333
1319 def setBlockDimension(self):
1334 def setBlockDimension(self):
1320 raise ValueError, "Not implemented"
1335 raise ValueError, "Not implemented"
1321
1336
1322
1337
1323 def writeBlock(self):
1338 def writeBlock(self):
1324 raise ValueError, "No implemented"
1339 raise ValueError, "No implemented"
1325
1340
1326
1341
1327 def putData(self):
1342 def putData(self):
1328 raise ValueError, "No implemented"
1343 raise ValueError, "No implemented"
1329
1344
1330
1345
1331 def getProcessFlags(self):
1346 def getProcessFlags(self):
1332
1347
1333 processFlags = 0
1348 processFlags = 0
1334
1349
1335 dtype_index = get_dtype_index(self.dtype)
1350 dtype_index = get_dtype_index(self.dtype)
1336 procflag_dtype = get_procflag_dtype(dtype_index)
1351 procflag_dtype = get_procflag_dtype(dtype_index)
1337
1352
1338 processFlags += procflag_dtype
1353 processFlags += procflag_dtype
1339
1354
1340 if self.dataOut.flagDecodeData:
1355 if self.dataOut.flagDecodeData:
1341 processFlags += PROCFLAG.DECODE_DATA
1356 processFlags += PROCFLAG.DECODE_DATA
1342
1357
1343 if self.dataOut.flagDeflipData:
1358 if self.dataOut.flagDeflipData:
1344 processFlags += PROCFLAG.DEFLIP_DATA
1359 processFlags += PROCFLAG.DEFLIP_DATA
1345
1360
1346 if self.dataOut.code is not None:
1361 if self.dataOut.code is not None:
1347 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1362 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1348
1363
1349 if self.dataOut.nCohInt > 1:
1364 if self.dataOut.nCohInt > 1:
1350 processFlags += PROCFLAG.COHERENT_INTEGRATION
1365 processFlags += PROCFLAG.COHERENT_INTEGRATION
1351
1366
1352 if self.dataOut.type == "Spectra":
1367 if self.dataOut.type == "Spectra":
1353 if self.dataOut.nIncohInt > 1:
1368 if self.dataOut.nIncohInt > 1:
1354 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1369 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1355
1370
1356 if self.dataOut.data_dc is not None:
1371 if self.dataOut.data_dc is not None:
1357 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1372 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1358
1373
1359 if self.dataOut.flagShiftFFT:
1374 if self.dataOut.flagShiftFFT:
1360 processFlags += PROCFLAG.SHIFT_FFT_DATA
1375 processFlags += PROCFLAG.SHIFT_FFT_DATA
1361
1376
1362 return processFlags
1377 return processFlags
1363
1378
1364 def setBasicHeader(self):
1379 def setBasicHeader(self):
1365
1380
1366 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1381 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1367 self.basicHeaderObj.version = self.versionFile
1382 self.basicHeaderObj.version = self.versionFile
1368 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1383 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1369
1384
1370 utc = numpy.floor(self.dataOut.utctime)
1385 utc = numpy.floor(self.dataOut.utctime)
1371 milisecond = (self.dataOut.utctime - utc)* 1000.0
1386 milisecond = (self.dataOut.utctime - utc)* 1000.0
1372
1387
1373 self.basicHeaderObj.utc = utc
1388 self.basicHeaderObj.utc = utc
1374 self.basicHeaderObj.miliSecond = milisecond
1389 self.basicHeaderObj.miliSecond = milisecond
1375 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1390 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1376 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1391 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1377 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1392 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1378
1393
1379 def setFirstHeader(self):
1394 def setFirstHeader(self):
1380 """
1395 """
1381 Obtiene una copia del First Header
1396 Obtiene una copia del First Header
1382
1397
1383 Affected:
1398 Affected:
1384
1399
1385 self.basicHeaderObj
1400 self.basicHeaderObj
1386 self.systemHeaderObj
1401 self.systemHeaderObj
1387 self.radarControllerHeaderObj
1402 self.radarControllerHeaderObj
1388 self.processingHeaderObj self.
1403 self.processingHeaderObj self.
1389
1404
1390 Return:
1405 Return:
1391 None
1406 None
1392 """
1407 """
1393
1408
1394 raise ValueError, "No implemented"
1409 raise ValueError, "No implemented"
1395
1410
1396 def __writeFirstHeader(self):
1411 def __writeFirstHeader(self):
1397 """
1412 """
1398 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1413 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1399
1414
1400 Affected:
1415 Affected:
1401 __dataType
1416 __dataType
1402
1417
1403 Return:
1418 Return:
1404 None
1419 None
1405 """
1420 """
1406
1421
1407 # CALCULAR PARAMETROS
1422 # CALCULAR PARAMETROS
1408
1423
1409 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1424 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1410 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1425 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1411
1426
1412 self.basicHeaderObj.write(self.fp)
1427 self.basicHeaderObj.write(self.fp)
1413 self.systemHeaderObj.write(self.fp)
1428 self.systemHeaderObj.write(self.fp)
1414 self.radarControllerHeaderObj.write(self.fp)
1429 self.radarControllerHeaderObj.write(self.fp)
1415 self.processingHeaderObj.write(self.fp)
1430 self.processingHeaderObj.write(self.fp)
1416
1431
1417 def __setNewBlock(self):
1432 def __setNewBlock(self):
1418 """
1433 """
1419 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1434 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1420
1435
1421 Return:
1436 Return:
1422 0 : si no pudo escribir nada
1437 0 : si no pudo escribir nada
1423 1 : Si escribio el Basic el First Header
1438 1 : Si escribio el Basic el First Header
1424 """
1439 """
1425 if self.fp == None:
1440 if self.fp == None:
1426 self.setNextFile()
1441 self.setNextFile()
1427
1442
1428 if self.flagIsNewFile:
1443 if self.flagIsNewFile:
1429 return 1
1444 return 1
1430
1445
1431 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1446 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1432 self.basicHeaderObj.write(self.fp)
1447 self.basicHeaderObj.write(self.fp)
1433 return 1
1448 return 1
1434
1449
1435 if not( self.setNextFile() ):
1450 if not( self.setNextFile() ):
1436 return 0
1451 return 0
1437
1452
1438 return 1
1453 return 1
1439
1454
1440
1455
1441 def writeNextBlock(self):
1456 def writeNextBlock(self):
1442 """
1457 """
1443 Selecciona el bloque siguiente de datos y los escribe en un file
1458 Selecciona el bloque siguiente de datos y los escribe en un file
1444
1459
1445 Return:
1460 Return:
1446 0 : Si no hizo pudo escribir el bloque de datos
1461 0 : Si no hizo pudo escribir el bloque de datos
1447 1 : Si no pudo escribir el bloque de datos
1462 1 : Si no pudo escribir el bloque de datos
1448 """
1463 """
1449 if not( self.__setNewBlock() ):
1464 if not( self.__setNewBlock() ):
1450 return 0
1465 return 0
1451
1466
1452 self.writeBlock()
1467 self.writeBlock()
1453
1468
1454 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1469 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1455 self.processingHeaderObj.dataBlocksPerFile)
1470 self.processingHeaderObj.dataBlocksPerFile)
1456
1471
1457 return 1
1472 return 1
1458
1473
1459 def setNextFile(self):
1474 def setNextFile(self):
1460 """
1475 """
1461 Determina el siguiente file que sera escrito
1476 Determina el siguiente file que sera escrito
1462
1477
1463 Affected:
1478 Affected:
1464 self.filename
1479 self.filename
1465 self.subfolder
1480 self.subfolder
1466 self.fp
1481 self.fp
1467 self.setFile
1482 self.setFile
1468 self.flagIsNewFile
1483 self.flagIsNewFile
1469
1484
1470 Return:
1485 Return:
1471 0 : Si el archivo no puede ser escrito
1486 0 : Si el archivo no puede ser escrito
1472 1 : Si el archivo esta listo para ser escrito
1487 1 : Si el archivo esta listo para ser escrito
1473 """
1488 """
1474 ext = self.ext
1489 ext = self.ext
1475 path = self.path
1490 path = self.path
1476
1491
1477 if self.fp != None:
1492 if self.fp != None:
1478 self.fp.close()
1493 self.fp.close()
1479
1494
1480 timeTuple = time.localtime( self.dataOut.utctime)
1495 timeTuple = time.localtime( self.dataOut.utctime)
1481 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1496 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1482
1497
1483 fullpath = os.path.join( path, subfolder )
1498 fullpath = os.path.join( path, subfolder )
1484 setFile = self.setFile
1499 setFile = self.setFile
1485
1500
1486 if not( os.path.exists(fullpath) ):
1501 if not( os.path.exists(fullpath) ):
1487 os.mkdir(fullpath)
1502 os.mkdir(fullpath)
1488 setFile = -1 #inicializo mi contador de seteo
1503 setFile = -1 #inicializo mi contador de seteo
1489 else:
1504 else:
1490 filesList = os.listdir( fullpath )
1505 filesList = os.listdir( fullpath )
1491 if len( filesList ) > 0:
1506 if len( filesList ) > 0:
1492 filesList = sorted( filesList, key=str.lower )
1507 filesList = sorted( filesList, key=str.lower )
1493 filen = filesList[-1]
1508 filen = filesList[-1]
1494 # el filename debera tener el siguiente formato
1509 # el filename debera tener el siguiente formato
1495 # 0 1234 567 89A BCDE (hex)
1510 # 0 1234 567 89A BCDE (hex)
1496 # x YYYY DDD SSS .ext
1511 # x YYYY DDD SSS .ext
1497 if isNumber( filen[8:11] ):
1512 if isNumber( filen[8:11] ):
1498 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1513 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1499 else:
1514 else:
1500 setFile = -1
1515 setFile = -1
1501 else:
1516 else:
1502 setFile = -1 #inicializo mi contador de seteo
1517 setFile = -1 #inicializo mi contador de seteo
1503
1518
1504 setFile += 1
1519 setFile += 1
1505
1520
1506 #If this is a new day it resets some values
1521 #If this is a new day it resets some values
1507 if self.dataOut.datatime.date() > self.fileDate:
1522 if self.dataOut.datatime.date() > self.fileDate:
1508 setFile = 0
1523 setFile = 0
1509 self.nTotalBlocks = 0
1524 self.nTotalBlocks = 0
1510
1525
1511 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1526 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1512
1527
1513 filename = os.path.join( path, subfolder, filen )
1528 filename = os.path.join( path, subfolder, filen )
1514
1529
1515 fp = open( filename,'wb' )
1530 fp = open( filename,'wb' )
1516
1531
1517 self.blockIndex = 0
1532 self.blockIndex = 0
1518
1533
1519 #guardando atributos
1534 #guardando atributos
1520 self.filename = filename
1535 self.filename = filename
1521 self.subfolder = subfolder
1536 self.subfolder = subfolder
1522 self.fp = fp
1537 self.fp = fp
1523 self.setFile = setFile
1538 self.setFile = setFile
1524 self.flagIsNewFile = 1
1539 self.flagIsNewFile = 1
1525 self.fileDate = self.dataOut.datatime.date()
1540 self.fileDate = self.dataOut.datatime.date()
1526
1541
1527 self.setFirstHeader()
1542 self.setFirstHeader()
1528
1543
1529 print '[Writing] Opening file: %s'%self.filename
1544 print '[Writing] Opening file: %s'%self.filename
1530
1545
1531 self.__writeFirstHeader()
1546 self.__writeFirstHeader()
1532
1547
1533 return 1
1548 return 1
1534
1549
1535 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1550 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1536 """
1551 """
1537 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1552 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1538
1553
1539 Inputs:
1554 Inputs:
1540 path : directory where data will be saved
1555 path : directory where data will be saved
1541 profilesPerBlock : number of profiles per block
1556 profilesPerBlock : number of profiles per block
1542 set : initial file set
1557 set : initial file set
1543 datatype : An integer number that defines data type:
1558 datatype : An integer number that defines data type:
1544 0 : int8 (1 byte)
1559 0 : int8 (1 byte)
1545 1 : int16 (2 bytes)
1560 1 : int16 (2 bytes)
1546 2 : int32 (4 bytes)
1561 2 : int32 (4 bytes)
1547 3 : int64 (8 bytes)
1562 3 : int64 (8 bytes)
1548 4 : float32 (4 bytes)
1563 4 : float32 (4 bytes)
1549 5 : double64 (8 bytes)
1564 5 : double64 (8 bytes)
1550
1565
1551 Return:
1566 Return:
1552 0 : Si no realizo un buen seteo
1567 0 : Si no realizo un buen seteo
1553 1 : Si realizo un buen seteo
1568 1 : Si realizo un buen seteo
1554 """
1569 """
1555
1570
1556 if ext == None:
1571 if ext == None:
1557 ext = self.ext
1572 ext = self.ext
1558
1573
1559 self.ext = ext.lower()
1574 self.ext = ext.lower()
1560
1575
1561 self.path = path
1576 self.path = path
1562
1577
1563 if set is None:
1578 if set is None:
1564 self.setFile = -1
1579 self.setFile = -1
1565 else:
1580 else:
1566 self.setFile = set - 1
1581 self.setFile = set - 1
1567
1582
1568 self.blocksPerFile = blocksPerFile
1583 self.blocksPerFile = blocksPerFile
1569
1584
1570 self.profilesPerBlock = profilesPerBlock
1585 self.profilesPerBlock = profilesPerBlock
1571
1586
1572 self.dataOut = dataOut
1587 self.dataOut = dataOut
1573 self.fileDate = self.dataOut.datatime.date()
1588 self.fileDate = self.dataOut.datatime.date()
1574 #By default
1589 #By default
1575 self.dtype = self.dataOut.dtype
1590 self.dtype = self.dataOut.dtype
1576
1591
1577 if datatype is not None:
1592 if datatype is not None:
1578 self.dtype = get_numpy_dtype(datatype)
1593 self.dtype = get_numpy_dtype(datatype)
1579
1594
1580 if not(self.setNextFile()):
1595 if not(self.setNextFile()):
1581 print "[Writing] There isn't a next file"
1596 print "[Writing] There isn't a next file"
1582 return 0
1597 return 0
1583
1598
1584 self.setBlockDimension()
1599 self.setBlockDimension()
1585
1600
1586 return 1
1601 return 1
1587
1602
1588 def run(self, dataOut, **kwargs):
1603 def run(self, dataOut, **kwargs):
1589
1604
1590 if not(self.isConfig):
1605 if not(self.isConfig):
1591
1606
1592 self.setup(dataOut, **kwargs)
1607 self.setup(dataOut, **kwargs)
1593 self.isConfig = True
1608 self.isConfig = True
1594
1609
1595 self.putData()
1610 self.putData()
1596
1611
General Comments 0
You need to be logged in to leave comments. Login now