##// END OF EJS Templates
schain
RSS
sys.exit(-1) was deleted from jroIO_base.py
Miguel Valdez -
r676:526ad8aef3b2
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1611 +1,1618
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 traceback.print_exc()
71 traceback.print_exc()
72 raise IOError, "The file %s can't be opened" %(filename)
72 raise IOError, "The file %s can't be opened" %(filename)
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
90 Inputs:
90 Inputs:
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92
92
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94
94
95 endDate : fecha final del rango seleccionado en formato datetime.date
95 endDate : fecha final del rango seleccionado en formato datetime.date
96
96
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98
98
99 endTime : tiempo final del rango seleccionado en formato datetime.time
99 endTime : tiempo final del rango seleccionado en formato datetime.time
100
100
101 Return:
101 Return:
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 fecha especificado, de lo contrario retorna False.
103 fecha especificado, de lo contrario retorna False.
104
104
105 Excepciones:
105 Excepciones:
106 Si el archivo no existe o no puede ser abierto
106 Si el archivo no existe o no puede ser abierto
107 Si la cabecera no puede ser leida.
107 Si la cabecera no puede ser leida.
108
108
109 """
109 """
110
110
111
111
112 try:
112 try:
113 fp = open(filename,'rb')
113 fp = open(filename,'rb')
114 except IOError:
114 except IOError:
115 traceback.print_exc()
115 traceback.print_exc()
116 raise IOError, "The file %s can't be opened" %(filename)
116 raise IOError, "The file %s can't be opened" %(filename)
117
117
118 basicHeaderObj = BasicHeader(LOCALTIME)
118 basicHeaderObj = BasicHeader(LOCALTIME)
119 sts = basicHeaderObj.read(fp)
119 sts = basicHeaderObj.read(fp)
120 fp.close()
120 fp.close()
121
121
122 thisDatetime = basicHeaderObj.datatime
122 thisDatetime = basicHeaderObj.datatime
123 thisDate = thisDatetime.date()
123 thisDate = thisDatetime.date()
124 thisTime = thisDatetime.time()
124 thisTime = thisDatetime.time()
125
125
126 if not(sts):
126 if not(sts):
127 print "Skipping the file %s because it has not a valid header" %(filename)
127 print "Skipping the file %s because it has not a valid header" %(filename)
128 return None
128 return None
129
129
130 #General case
130 #General case
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 #-----------o----------------------------o-----------
132 #-----------o----------------------------o-----------
133 # startTime endTime
133 # startTime endTime
134
134
135 if endTime >= startTime:
135 if endTime >= startTime:
136 if (thisTime < startTime) or (thisTime > endTime):
136 if (thisTime < startTime) or (thisTime > endTime):
137 return None
137 return None
138
138
139 return thisDatetime
139 return thisDatetime
140
140
141 #If endTime < startTime then endTime belongs to the next day
141 #If endTime < startTime then endTime belongs to the next day
142
142
143
143
144 #<<<<<<<<<<<o o>>>>>>>>>>>
144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 #-----------o----------------------------o-----------
145 #-----------o----------------------------o-----------
146 # endTime startTime
146 # endTime startTime
147
147
148 if (thisDate == startDate) and (thisTime < startTime):
148 if (thisDate == startDate) and (thisTime < startTime):
149 return None
149 return None
150
150
151 if (thisDate == endDate) and (thisTime > endTime):
151 if (thisDate == endDate) and (thisTime > endTime):
152 return None
152 return None
153
153
154 if (thisTime < startTime) and (thisTime > endTime):
154 if (thisTime < startTime) and (thisTime > endTime):
155 return None
155 return None
156
156
157 return thisDatetime
157 return thisDatetime
158
158
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 """
160 """
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162
162
163 Inputs:
163 Inputs:
164 folder : nombre completo del directorio.
164 folder : nombre completo del directorio.
165 Su formato deberia ser "/path_root/?YYYYDDD"
165 Su formato deberia ser "/path_root/?YYYYDDD"
166
166
167 siendo:
167 siendo:
168 YYYY : Anio (ejemplo 2015)
168 YYYY : Anio (ejemplo 2015)
169 DDD : Dia del anio (ejemplo 305)
169 DDD : Dia del anio (ejemplo 305)
170
170
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172
172
173 endDate : fecha final del rango seleccionado en formato datetime.date
173 endDate : fecha final del rango seleccionado en formato datetime.date
174
174
175 Return:
175 Return:
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 fecha especificado, de lo contrario retorna False.
177 fecha especificado, de lo contrario retorna False.
178 Excepciones:
178 Excepciones:
179 Si el directorio no tiene el formato adecuado
179 Si el directorio no tiene el formato adecuado
180 """
180 """
181
181
182 basename = os.path.basename(folder)
182 basename = os.path.basename(folder)
183
183
184 if not isRadarFolder(basename):
184 if not isRadarFolder(basename):
185 raise IOError, "The folder %s has not the rigth format" %folder
185 raise IOError, "The folder %s has not the rigth format" %folder
186
186
187 if startDate and endDate:
187 if startDate and endDate:
188 thisDate = getDateFromRadarFolder(basename)
188 thisDate = getDateFromRadarFolder(basename)
189
189
190 if thisDate < startDate:
190 if thisDate < startDate:
191 return 0
191 return 0
192
192
193 if thisDate > endDate:
193 if thisDate > endDate:
194 return 0
194 return 0
195
195
196 return 1
196 return 1
197
197
198 def isFileInDateRange(filename, startDate=None, endDate=None):
198 def isFileInDateRange(filename, startDate=None, endDate=None):
199 """
199 """
200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
201
201
202 Inputs:
202 Inputs:
203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
204
204
205 Su formato deberia ser "?YYYYDDDsss"
205 Su formato deberia ser "?YYYYDDDsss"
206
206
207 siendo:
207 siendo:
208 YYYY : Anio (ejemplo 2015)
208 YYYY : Anio (ejemplo 2015)
209 DDD : Dia del anio (ejemplo 305)
209 DDD : Dia del anio (ejemplo 305)
210 sss : set
210 sss : set
211
211
212 startDate : fecha inicial del rango seleccionado en formato datetime.date
212 startDate : fecha inicial del rango seleccionado en formato datetime.date
213
213
214 endDate : fecha final del rango seleccionado en formato datetime.date
214 endDate : fecha final del rango seleccionado en formato datetime.date
215
215
216 Return:
216 Return:
217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
218 fecha especificado, de lo contrario retorna False.
218 fecha especificado, de lo contrario retorna False.
219 Excepciones:
219 Excepciones:
220 Si el archivo no tiene el formato adecuado
220 Si el archivo no tiene el formato adecuado
221 """
221 """
222
222
223 basename = os.path.basename(filename)
223 basename = os.path.basename(filename)
224
224
225 if not isRadarFile(basename):
225 if not isRadarFile(basename):
226 raise IOError, "The filename %s has not the rigth format" %filename
226 raise IOError, "The filename %s has not the rigth format" %filename
227
227
228 if startDate and endDate:
228 if startDate and endDate:
229 thisDate = getDateFromRadarFile(basename)
229 thisDate = getDateFromRadarFile(basename)
230
230
231 if thisDate < startDate:
231 if thisDate < startDate:
232 return 0
232 return 0
233
233
234 if thisDate > endDate:
234 if thisDate > endDate:
235 return 0
235 return 0
236
236
237 return 1
237 return 1
238
238
239 def getFileFromSet(path, ext, set):
239 def getFileFromSet(path, ext, set):
240 validFilelist = []
240 validFilelist = []
241 fileList = os.listdir(path)
241 fileList = os.listdir(path)
242
242
243 # 0 1234 567 89A BCDE
243 # 0 1234 567 89A BCDE
244 # H YYYY DDD SSS .ext
244 # H YYYY DDD SSS .ext
245
245
246 for thisFile in fileList:
246 for thisFile in fileList:
247 try:
247 try:
248 year = int(thisFile[1:5])
248 year = int(thisFile[1:5])
249 doy = int(thisFile[5:8])
249 doy = int(thisFile[5:8])
250 except:
250 except:
251 continue
251 continue
252
252
253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
254 continue
254 continue
255
255
256 validFilelist.append(thisFile)
256 validFilelist.append(thisFile)
257
257
258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
259
259
260 if len(myfile)!= 0:
260 if len(myfile)!= 0:
261 return myfile[0]
261 return myfile[0]
262 else:
262 else:
263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
264 print 'the filename %s does not exist'%filename
264 print 'the filename %s does not exist'%filename
265 print '...going to the last file: '
265 print '...going to the last file: '
266
266
267 if validFilelist:
267 if validFilelist:
268 validFilelist = sorted( validFilelist, key=str.lower )
268 validFilelist = sorted( validFilelist, key=str.lower )
269 return validFilelist[-1]
269 return validFilelist[-1]
270
270
271 return None
271 return None
272
272
273 def getlastFileFromPath(path, ext):
273 def getlastFileFromPath(path, ext):
274 """
274 """
275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
277
277
278 Input:
278 Input:
279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
280 ext : extension de los files contenidos en una carpeta
280 ext : extension de los files contenidos en una carpeta
281
281
282 Return:
282 Return:
283 El ultimo file de una determinada carpeta, no se considera el path.
283 El ultimo file de una determinada carpeta, no se considera el path.
284 """
284 """
285 validFilelist = []
285 validFilelist = []
286 fileList = os.listdir(path)
286 fileList = os.listdir(path)
287
287
288 # 0 1234 567 89A BCDE
288 # 0 1234 567 89A BCDE
289 # H YYYY DDD SSS .ext
289 # H YYYY DDD SSS .ext
290
290
291 for thisFile in fileList:
291 for thisFile in fileList:
292
292
293 year = thisFile[1:5]
293 year = thisFile[1:5]
294 if not isNumber(year):
294 if not isNumber(year):
295 continue
295 continue
296
296
297 doy = thisFile[5:8]
297 doy = thisFile[5:8]
298 if not isNumber(doy):
298 if not isNumber(doy):
299 continue
299 continue
300
300
301 year = int(year)
301 year = int(year)
302 doy = int(doy)
302 doy = int(doy)
303
303
304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 continue
305 continue
306
306
307 validFilelist.append(thisFile)
307 validFilelist.append(thisFile)
308
308
309 if validFilelist:
309 if validFilelist:
310 validFilelist = sorted( validFilelist, key=str.lower )
310 validFilelist = sorted( validFilelist, key=str.lower )
311 return validFilelist[-1]
311 return validFilelist[-1]
312
312
313 return None
313 return None
314
314
315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
316 """
316 """
317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
319 el path exacto de un determinado file.
319 el path exacto de un determinado file.
320
320
321 Example :
321 Example :
322 nombre correcto del file es .../.../D2009307/P2009307367.ext
322 nombre correcto del file es .../.../D2009307/P2009307367.ext
323
323
324 Entonces la funcion prueba con las siguientes combinaciones
324 Entonces la funcion prueba con las siguientes combinaciones
325 .../.../y2009307367.ext
325 .../.../y2009307367.ext
326 .../.../Y2009307367.ext
326 .../.../Y2009307367.ext
327 .../.../x2009307/y2009307367.ext
327 .../.../x2009307/y2009307367.ext
328 .../.../x2009307/Y2009307367.ext
328 .../.../x2009307/Y2009307367.ext
329 .../.../X2009307/y2009307367.ext
329 .../.../X2009307/y2009307367.ext
330 .../.../X2009307/Y2009307367.ext
330 .../.../X2009307/Y2009307367.ext
331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
332
332
333 Return:
333 Return:
334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
336 para el filename
336 para el filename
337 """
337 """
338 fullfilename = None
338 fullfilename = None
339 find_flag = False
339 find_flag = False
340 filename = None
340 filename = None
341
341
342 prefixDirList = [None,'d','D']
342 prefixDirList = [None,'d','D']
343 if ext.lower() == ".r": #voltage
343 if ext.lower() == ".r": #voltage
344 prefixFileList = ['d','D']
344 prefixFileList = ['d','D']
345 elif ext.lower() == ".pdata": #spectra
345 elif ext.lower() == ".pdata": #spectra
346 prefixFileList = ['p','P']
346 prefixFileList = ['p','P']
347 else:
347 else:
348 return None, filename
348 return None, filename
349
349
350 #barrido por las combinaciones posibles
350 #barrido por las combinaciones posibles
351 for prefixDir in prefixDirList:
351 for prefixDir in prefixDirList:
352 thispath = path
352 thispath = path
353 if prefixDir != None:
353 if prefixDir != None:
354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
355 if foldercounter == 0:
355 if foldercounter == 0:
356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
357 else:
357 else:
358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
362
362
363 if os.path.exists( fullfilename ): #verifico que exista
363 if os.path.exists( fullfilename ): #verifico que exista
364 find_flag = True
364 find_flag = True
365 break
365 break
366 if find_flag:
366 if find_flag:
367 break
367 break
368
368
369 if not(find_flag):
369 if not(find_flag):
370 return None, filename
370 return None, filename
371
371
372 return fullfilename, filename
372 return fullfilename, filename
373
373
374 def isRadarFolder(folder):
374 def isRadarFolder(folder):
375 try:
375 try:
376 year = int(folder[1:5])
376 year = int(folder[1:5])
377 doy = int(folder[5:8])
377 doy = int(folder[5:8])
378 except:
378 except:
379 return 0
379 return 0
380
380
381 return 1
381 return 1
382
382
383 def isRadarFile(file):
383 def isRadarFile(file):
384 try:
384 try:
385 year = int(file[1:5])
385 year = int(file[1:5])
386 doy = int(file[5:8])
386 doy = int(file[5:8])
387 set = int(file[8:11])
387 set = int(file[8:11])
388 except:
388 except:
389 return 0
389 return 0
390
390
391 return 1
391 return 1
392
392
393 def getDateFromRadarFile(file):
393 def getDateFromRadarFile(file):
394 try:
394 try:
395 year = int(file[1:5])
395 year = int(file[1:5])
396 doy = int(file[5:8])
396 doy = int(file[5:8])
397 set = int(file[8:11])
397 set = int(file[8:11])
398 except:
398 except:
399 return None
399 return None
400
400
401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
402 return thisDate
402 return thisDate
403
403
404 def getDateFromRadarFolder(folder):
404 def getDateFromRadarFolder(folder):
405 try:
405 try:
406 year = int(folder[1:5])
406 year = int(folder[1:5])
407 doy = int(folder[5:8])
407 doy = int(folder[5:8])
408 except:
408 except:
409 return None
409 return None
410
410
411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
412 return thisDate
412 return thisDate
413
413
414 class JRODataIO:
414 class JRODataIO:
415
415
416 c = 3E8
416 c = 3E8
417
417
418 isConfig = False
418 isConfig = False
419
419
420 basicHeaderObj = None
420 basicHeaderObj = None
421
421
422 systemHeaderObj = None
422 systemHeaderObj = None
423
423
424 radarControllerHeaderObj = None
424 radarControllerHeaderObj = None
425
425
426 processingHeaderObj = None
426 processingHeaderObj = None
427
427
428 dtype = None
428 dtype = None
429
429
430 pathList = []
430 pathList = []
431
431
432 filenameList = []
432 filenameList = []
433
433
434 filename = None
434 filename = None
435
435
436 ext = None
436 ext = None
437
437
438 flagIsNewFile = 1
438 flagIsNewFile = 1
439
439
440 flagDiscontinuousBlock = 0
440 flagDiscontinuousBlock = 0
441
441
442 flagIsNewBlock = 0
442 flagIsNewBlock = 0
443
443
444 fp = None
444 fp = None
445
445
446 firstHeaderSize = 0
446 firstHeaderSize = 0
447
447
448 basicHeaderSize = 24
448 basicHeaderSize = 24
449
449
450 versionFile = 1103
450 versionFile = 1103
451
451
452 fileSize = None
452 fileSize = None
453
453
454 # ippSeconds = None
454 # ippSeconds = None
455
455
456 fileSizeByHeader = None
456 fileSizeByHeader = None
457
457
458 fileIndex = None
458 fileIndex = None
459
459
460 profileIndex = None
460 profileIndex = None
461
461
462 blockIndex = None
462 blockIndex = None
463
463
464 nTotalBlocks = None
464 nTotalBlocks = None
465
465
466 maxTimeStep = 30
466 maxTimeStep = 30
467
467
468 lastUTTime = None
468 lastUTTime = None
469
469
470 datablock = None
470 datablock = None
471
471
472 dataOut = None
472 dataOut = None
473
473
474 blocksize = None
474 blocksize = None
475
475
476 getByBlock = False
476 getByBlock = False
477
477
478 def __init__(self):
478 def __init__(self):
479
479
480 raise ValueError, "Not implemented"
480 raise ValueError, "Not implemented"
481
481
482 def run(self):
482 def run(self):
483
483
484 raise ValueError, "Not implemented"
484 raise ValueError, "Not implemented"
485
485
486 def getDtypeWidth(self):
486 def getDtypeWidth(self):
487
487
488 dtype_index = get_dtype_index(self.dtype)
488 dtype_index = get_dtype_index(self.dtype)
489 dtype_width = get_dtype_width(dtype_index)
489 dtype_width = get_dtype_width(dtype_index)
490
490
491 return dtype_width
491 return dtype_width
492
492
493 class JRODataReader(JRODataIO):
493 class JRODataReader(JRODataIO):
494
494
495
495
496 online = 0
496 online = 0
497
497
498 realtime = 0
498 realtime = 0
499
499
500 nReadBlocks = 0
500 nReadBlocks = 0
501
501
502 delay = 10 #number of seconds waiting a new file
502 delay = 10 #number of seconds waiting a new file
503
503
504 nTries = 3 #quantity tries
504 nTries = 3 #quantity tries
505
505
506 nFiles = 3 #number of files for searching
506 nFiles = 3 #number of files for searching
507
507
508 path = None
508 path = None
509
509
510 foldercounter = 0
510 foldercounter = 0
511
511
512 flagNoMoreFiles = 0
512 flagNoMoreFiles = 0
513
513
514 datetimeList = []
514 datetimeList = []
515
515
516 __isFirstTimeOnline = 1
516 __isFirstTimeOnline = 1
517
517
518 __printInfo = True
518 __printInfo = True
519
519
520 profileIndex = None
520 profileIndex = None
521
521
522 nTxs = 1
522 nTxs = 1
523
523
524 txIndex = None
524 txIndex = None
525
525
526 def __init__(self):
526 def __init__(self):
527
527
528 """
528 """
529
529
530 """
530 """
531
531
532 # raise NotImplementedError, "This method has not been implemented"
532 # raise NotImplementedError, "This method has not been implemented"
533
533
534
534
535 def createObjByDefault(self):
535 def createObjByDefault(self):
536 """
536 """
537
537
538 """
538 """
539 raise NotImplementedError, "This method has not been implemented"
539 raise NotImplementedError, "This method has not been implemented"
540
540
541 def getBlockDimension(self):
541 def getBlockDimension(self):
542
542
543 raise NotImplementedError, "No implemented"
543 raise NotImplementedError, "No implemented"
544
544
545 def __searchFilesOffLine(self,
545 def __searchFilesOffLine(self,
546 path,
546 path,
547 startDate=None,
547 startDate=None,
548 endDate=None,
548 endDate=None,
549 startTime=datetime.time(0,0,0),
549 startTime=datetime.time(0,0,0),
550 endTime=datetime.time(23,59,59),
550 endTime=datetime.time(23,59,59),
551 set=None,
551 set=None,
552 expLabel='',
552 expLabel='',
553 ext='.r',
553 ext='.r',
554 walk=True):
554 walk=True):
555
555
556 self.filenameList = []
556 self.filenameList = []
557 self.datetimeList = []
557 self.datetimeList = []
558
558
559 pathList = []
559 pathList = []
560
560
561 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
561 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
562
562
563 if dateList == []:
563 if dateList == []:
564 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
564 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
565 datetime.datetime.combine(startDate,startTime).ctime(),
565 datetime.datetime.combine(startDate,startTime).ctime(),
566 datetime.datetime.combine(endDate,endTime).ctime())
566 datetime.datetime.combine(endDate,endTime).ctime())
567
567
568 return None, None
568 return None, None
569
569
570 if len(dateList) > 1:
570 if len(dateList) > 1:
571 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
571 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
572 else:
572 else:
573 print "[Reading] data was found for the date %s" %(dateList[0])
573 print "[Reading] data was found for the date %s" %(dateList[0])
574
574
575 filenameList = []
575 filenameList = []
576 datetimeList = []
576 datetimeList = []
577
577
578 for thisPath in pathList:
578 for thisPath in pathList:
579 # thisPath = pathList[pathDict[file]]
579 # thisPath = pathList[pathDict[file]]
580
580
581 fileList = glob.glob1(thisPath, "*%s" %ext)
581 fileList = glob.glob1(thisPath, "*%s" %ext)
582 fileList.sort()
582 fileList.sort()
583
583
584 for file in fileList:
584 for file in fileList:
585
585
586 filename = os.path.join(thisPath,file)
586 filename = os.path.join(thisPath,file)
587
587
588 if not isFileInDateRange(filename, startDate, endDate):
588 if not isFileInDateRange(filename, startDate, endDate):
589 continue
589 continue
590
590
591 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
591 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
592
592
593 if not(thisDatetime):
593 if not(thisDatetime):
594 continue
594 continue
595
595
596 filenameList.append(filename)
596 filenameList.append(filename)
597 datetimeList.append(thisDatetime)
597 datetimeList.append(thisDatetime)
598
598
599 if not(filenameList):
599 if not(filenameList):
600 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
600 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
601 return None, None
601 return None, None
602
602
603 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
603 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
604 print
604 print
605
605
606 for i in range(len(filenameList)):
606 for i in range(len(filenameList)):
607 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
607 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
608
608
609 self.filenameList = filenameList
609 self.filenameList = filenameList
610 self.datetimeList = datetimeList
610 self.datetimeList = datetimeList
611
611
612 return pathList, filenameList
612 return pathList, filenameList
613
613
614 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
614 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
615
615
616 """
616 """
617 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
617 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
618 devuelve el archivo encontrado ademas de otros datos.
618 devuelve el archivo encontrado ademas de otros datos.
619
619
620 Input:
620 Input:
621 path : carpeta donde estan contenidos los files que contiene data
621 path : carpeta donde estan contenidos los files que contiene data
622
622
623 expLabel : Nombre del subexperimento (subfolder)
623 expLabel : Nombre del subexperimento (subfolder)
624
624
625 ext : extension de los files
625 ext : extension de los files
626
626
627 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
627 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
628
628
629 Return:
629 Return:
630 directory : eL directorio donde esta el file encontrado
630 directory : eL directorio donde esta el file encontrado
631 filename : el ultimo file de una determinada carpeta
631 filename : el ultimo file de una determinada carpeta
632 year : el anho
632 year : el anho
633 doy : el numero de dia del anho
633 doy : el numero de dia del anho
634 set : el set del archivo
634 set : el set del archivo
635
635
636
636
637 """
637 """
638 dirList = []
638 dirList = []
639
639
640 if not walk:
640 if not walk:
641 fullpath = path
641 fullpath = path
642 foldercounter = 0
642 foldercounter = 0
643 else:
643 else:
644 #Filtra solo los directorios
644 #Filtra solo los directorios
645 for thisPath in os.listdir(path):
645 for thisPath in os.listdir(path):
646 if not os.path.isdir(os.path.join(path,thisPath)):
646 if not os.path.isdir(os.path.join(path,thisPath)):
647 continue
647 continue
648 if not isRadarFolder(thisPath):
648 if not isRadarFolder(thisPath):
649 continue
649 continue
650
650
651 dirList.append(thisPath)
651 dirList.append(thisPath)
652
652
653 if not(dirList):
653 if not(dirList):
654 return None, None, None, None, None, None
654 return None, None, None, None, None, None
655
655
656 dirList = sorted( dirList, key=str.lower )
656 dirList = sorted( dirList, key=str.lower )
657
657
658 doypath = dirList[-1]
658 doypath = dirList[-1]
659 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
659 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
660 fullpath = os.path.join(path, doypath, expLabel)
660 fullpath = os.path.join(path, doypath, expLabel)
661
661
662
662
663 print "[Reading] %s folder was found: " %(fullpath )
663 print "[Reading] %s folder was found: " %(fullpath )
664
664
665 if set == None:
665 if set == None:
666 filename = getlastFileFromPath(fullpath, ext)
666 filename = getlastFileFromPath(fullpath, ext)
667 else:
667 else:
668 filename = getFileFromSet(fullpath, ext, set)
668 filename = getFileFromSet(fullpath, ext, set)
669
669
670 if not(filename):
670 if not(filename):
671 return None, None, None, None, None, None
671 return None, None, None, None, None, None
672
672
673 print "[Reading] %s file was found" %(filename)
673 print "[Reading] %s file was found" %(filename)
674
674
675 if not(self.__verifyFile(os.path.join(fullpath, filename))):
675 if not(self.__verifyFile(os.path.join(fullpath, filename))):
676 return None, None, None, None, None, None
676 return None, None, None, None, None, None
677
677
678 year = int( filename[1:5] )
678 year = int( filename[1:5] )
679 doy = int( filename[5:8] )
679 doy = int( filename[5:8] )
680 set = int( filename[8:11] )
680 set = int( filename[8:11] )
681
681
682 return fullpath, foldercounter, filename, year, doy, set
682 return fullpath, foldercounter, filename, year, doy, set
683
683
684 def __setNextFileOffline(self):
684 def __setNextFileOffline(self):
685
685
686 idFile = self.fileIndex
686 idFile = self.fileIndex
687
687
688 while (True):
688 while (True):
689 idFile += 1
689 idFile += 1
690 if not(idFile < len(self.filenameList)):
690 if not(idFile < len(self.filenameList)):
691 self.flagNoMoreFiles = 1
691 self.flagNoMoreFiles = 1
692 # print "[Reading] No more Files"
692 # print "[Reading] No more Files"
693 return 0
693 return 0
694
694
695 filename = self.filenameList[idFile]
695 filename = self.filenameList[idFile]
696
696
697 if not(self.__verifyFile(filename)):
697 if not(self.__verifyFile(filename)):
698 continue
698 continue
699
699
700 fileSize = os.path.getsize(filename)
700 fileSize = os.path.getsize(filename)
701 fp = open(filename,'rb')
701 fp = open(filename,'rb')
702 break
702 break
703
703
704 self.flagIsNewFile = 1
704 self.flagIsNewFile = 1
705 self.fileIndex = idFile
705 self.fileIndex = idFile
706 self.filename = filename
706 self.filename = filename
707 self.fileSize = fileSize
707 self.fileSize = fileSize
708 self.fp = fp
708 self.fp = fp
709
709
710 # print "[Reading] Setting the file: %s"%self.filename
710 # print "[Reading] Setting the file: %s"%self.filename
711
711
712 return 1
712 return 1
713
713
714 def __setNextFileOnline(self):
714 def __setNextFileOnline(self):
715 """
715 """
716 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
716 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
717 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
717 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
718 siguientes.
718 siguientes.
719
719
720 Affected:
720 Affected:
721 self.flagIsNewFile
721 self.flagIsNewFile
722 self.filename
722 self.filename
723 self.fileSize
723 self.fileSize
724 self.fp
724 self.fp
725 self.set
725 self.set
726 self.flagNoMoreFiles
726 self.flagNoMoreFiles
727
727
728 Return:
728 Return:
729 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
729 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
730 1 : si el file fue abierto con exito y esta listo a ser leido
730 1 : si el file fue abierto con exito y esta listo a ser leido
731
731
732 Excepciones:
732 Excepciones:
733 Si un determinado file no puede ser abierto
733 Si un determinado file no puede ser abierto
734 """
734 """
735 nFiles = 0
735 nFiles = 0
736 fileOk_flag = False
736 fileOk_flag = False
737 firstTime_flag = True
737 firstTime_flag = True
738
738
739 self.set += 1
739 self.set += 1
740
740
741 if self.set > 999:
741 if self.set > 999:
742 self.set = 0
742 self.set = 0
743 self.foldercounter += 1
743 self.foldercounter += 1
744
744
745 #busca el 1er file disponible
745 #busca el 1er file disponible
746 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
746 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
747 if fullfilename:
747 if fullfilename:
748 if self.__verifyFile(fullfilename, False):
748 if self.__verifyFile(fullfilename, False):
749 fileOk_flag = True
749 fileOk_flag = True
750
750
751 #si no encuentra un file entonces espera y vuelve a buscar
751 #si no encuentra un file entonces espera y vuelve a buscar
752 if not(fileOk_flag):
752 if not(fileOk_flag):
753 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
753 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
754
754
755 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
755 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
756 tries = self.nTries
756 tries = self.nTries
757 else:
757 else:
758 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
758 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
759
759
760 for nTries in range( tries ):
760 for nTries in range( tries ):
761 if firstTime_flag:
761 if firstTime_flag:
762 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
762 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
763 sleep( self.delay )
763 sleep( self.delay )
764 else:
764 else:
765 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
765 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
766
766
767 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
767 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
768 if fullfilename:
768 if fullfilename:
769 if self.__verifyFile(fullfilename):
769 if self.__verifyFile(fullfilename):
770 fileOk_flag = True
770 fileOk_flag = True
771 break
771 break
772
772
773 if fileOk_flag:
773 if fileOk_flag:
774 break
774 break
775
775
776 firstTime_flag = False
776 firstTime_flag = False
777
777
778 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
778 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
779 self.set += 1
779 self.set += 1
780
780
781 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
781 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
782 self.set = 0
782 self.set = 0
783 self.doy += 1
783 self.doy += 1
784 self.foldercounter = 0
784 self.foldercounter = 0
785
785
786 if fileOk_flag:
786 if fileOk_flag:
787 self.fileSize = os.path.getsize( fullfilename )
787 self.fileSize = os.path.getsize( fullfilename )
788 self.filename = fullfilename
788 self.filename = fullfilename
789 self.flagIsNewFile = 1
789 self.flagIsNewFile = 1
790 if self.fp != None: self.fp.close()
790 if self.fp != None: self.fp.close()
791 self.fp = open(fullfilename, 'rb')
791 self.fp = open(fullfilename, 'rb')
792 self.flagNoMoreFiles = 0
792 self.flagNoMoreFiles = 0
793 # print '[Reading] Setting the file: %s' % fullfilename
793 # print '[Reading] Setting the file: %s' % fullfilename
794 else:
794 else:
795 self.fileSize = 0
795 self.fileSize = 0
796 self.filename = None
796 self.filename = None
797 self.flagIsNewFile = 0
797 self.flagIsNewFile = 0
798 self.fp = None
798 self.fp = None
799 self.flagNoMoreFiles = 1
799 self.flagNoMoreFiles = 1
800 # print '[Reading] No more files to read'
800 # print '[Reading] No more files to read'
801
801
802 return fileOk_flag
802 return fileOk_flag
803
803
804 def setNextFile(self):
804 def setNextFile(self):
805 if self.fp != None:
805 if self.fp != None:
806 self.fp.close()
806 self.fp.close()
807
807
808 if self.online:
808 if self.online:
809 newFile = self.__setNextFileOnline()
809 newFile = self.__setNextFileOnline()
810 else:
810 else:
811 newFile = self.__setNextFileOffline()
811 newFile = self.__setNextFileOffline()
812
812
813 if not(newFile):
813 if not(newFile):
814 print '[Reading] No more files to read'
814 print '[Reading] No more files to read'
815 return 0
815 return 0
816
816
817 print '[Reading] Setting the file: %s' % self.filename
817 print '[Reading] Setting the file: %s' % self.filename
818
818
819 self.__readFirstHeader()
819 self.__readFirstHeader()
820 self.nReadBlocks = 0
820 self.nReadBlocks = 0
821 return 1
821 return 1
822
822
823 def __waitNewBlock(self):
823 def __waitNewBlock(self):
824 """
824 """
825 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
825 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
826
826
827 Si el modo de lectura es OffLine siempre retorn 0
827 Si el modo de lectura es OffLine siempre retorn 0
828 """
828 """
829 if not self.online:
829 if not self.online:
830 return 0
830 return 0
831
831
832 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
832 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
833 return 0
833 return 0
834
834
835 currentPointer = self.fp.tell()
835 currentPointer = self.fp.tell()
836
836
837 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
837 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
838
838
839 for nTries in range( self.nTries ):
839 for nTries in range( self.nTries ):
840
840
841 self.fp.close()
841 self.fp.close()
842 self.fp = open( self.filename, 'rb' )
842 self.fp = open( self.filename, 'rb' )
843 self.fp.seek( currentPointer )
843 self.fp.seek( currentPointer )
844
844
845 self.fileSize = os.path.getsize( self.filename )
845 self.fileSize = os.path.getsize( self.filename )
846 currentSize = self.fileSize - currentPointer
846 currentSize = self.fileSize - currentPointer
847
847
848 if ( currentSize >= neededSize ):
848 if ( currentSize >= neededSize ):
849 self.basicHeaderObj.read(self.fp)
849 self.basicHeaderObj.read(self.fp)
850 return 1
850 return 1
851
851
852 if self.fileSize == self.fileSizeByHeader:
852 if self.fileSize == self.fileSizeByHeader:
853 # self.flagEoF = True
853 # self.flagEoF = True
854 return 0
854 return 0
855
855
856 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
856 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
857 sleep( self.delay )
857 sleep( self.delay )
858
858
859
859
860 return 0
860 return 0
861
861
862 def waitDataBlock(self,pointer_location):
862 def waitDataBlock(self,pointer_location):
863
863
864 currentPointer = pointer_location
864 currentPointer = pointer_location
865
865
866 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
866 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
867
867
868 for nTries in range( self.nTries ):
868 for nTries in range( self.nTries ):
869 self.fp.close()
869 self.fp.close()
870 self.fp = open( self.filename, 'rb' )
870 self.fp = open( self.filename, 'rb' )
871 self.fp.seek( currentPointer )
871 self.fp.seek( currentPointer )
872
872
873 self.fileSize = os.path.getsize( self.filename )
873 self.fileSize = os.path.getsize( self.filename )
874 currentSize = self.fileSize - currentPointer
874 currentSize = self.fileSize - currentPointer
875
875
876 if ( currentSize >= neededSize ):
876 if ( currentSize >= neededSize ):
877 return 1
877 return 1
878
878
879 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
879 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
880 sleep( self.delay )
880 sleep( self.delay )
881
881
882 return 0
882 return 0
883
883
884 def __jumpToLastBlock(self):
884 def __jumpToLastBlock(self):
885
885
886 if not(self.__isFirstTimeOnline):
886 if not(self.__isFirstTimeOnline):
887 return
887 return
888
888
889 csize = self.fileSize - self.fp.tell()
889 csize = self.fileSize - self.fp.tell()
890 blocksize = self.processingHeaderObj.blockSize
890 blocksize = self.processingHeaderObj.blockSize
891
891
892 #salta el primer bloque de datos
892 #salta el primer bloque de datos
893 if csize > self.processingHeaderObj.blockSize:
893 if csize > self.processingHeaderObj.blockSize:
894 self.fp.seek(self.fp.tell() + blocksize)
894 self.fp.seek(self.fp.tell() + blocksize)
895 else:
895 else:
896 return
896 return
897
897
898 csize = self.fileSize - self.fp.tell()
898 csize = self.fileSize - self.fp.tell()
899 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
899 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
900 while True:
900 while True:
901
901
902 if self.fp.tell()<self.fileSize:
902 if self.fp.tell()<self.fileSize:
903 self.fp.seek(self.fp.tell() + neededsize)
903 self.fp.seek(self.fp.tell() + neededsize)
904 else:
904 else:
905 self.fp.seek(self.fp.tell() - neededsize)
905 self.fp.seek(self.fp.tell() - neededsize)
906 break
906 break
907
907
908 # csize = self.fileSize - self.fp.tell()
908 # csize = self.fileSize - self.fp.tell()
909 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
909 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
910 # factor = int(csize/neededsize)
910 # factor = int(csize/neededsize)
911 # if factor > 0:
911 # if factor > 0:
912 # self.fp.seek(self.fp.tell() + factor*neededsize)
912 # self.fp.seek(self.fp.tell() + factor*neededsize)
913
913
914 self.flagIsNewFile = 0
914 self.flagIsNewFile = 0
915 self.__isFirstTimeOnline = 0
915 self.__isFirstTimeOnline = 0
916
916
917 def __setNewBlock(self):
917 def __setNewBlock(self):
918
918
919 if self.fp == None:
919 if self.fp == None:
920 return 0
920 return 0
921
921
922 # if self.online:
922 # if self.online:
923 # self.__jumpToLastBlock()
923 # self.__jumpToLastBlock()
924
924
925 if self.flagIsNewFile:
925 if self.flagIsNewFile:
926 self.lastUTTime = self.basicHeaderObj.utc
926 self.lastUTTime = self.basicHeaderObj.utc
927 return 1
927 return 1
928
928
929 if self.realtime:
929 if self.realtime:
930 self.flagDiscontinuousBlock = 1
930 self.flagDiscontinuousBlock = 1
931 if not(self.setNextFile()):
931 if not(self.setNextFile()):
932 return 0
932 return 0
933 else:
933 else:
934 return 1
934 return 1
935
935
936 currentSize = self.fileSize - self.fp.tell()
936 currentSize = self.fileSize - self.fp.tell()
937 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
937 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
938
938
939 if (currentSize >= neededSize):
939 if (currentSize >= neededSize):
940 self.basicHeaderObj.read(self.fp)
940 self.basicHeaderObj.read(self.fp)
941 self.lastUTTime = self.basicHeaderObj.utc
941 self.lastUTTime = self.basicHeaderObj.utc
942 return 1
942 return 1
943
943
944 if self.__waitNewBlock():
944 if self.__waitNewBlock():
945 self.lastUTTime = self.basicHeaderObj.utc
945 self.lastUTTime = self.basicHeaderObj.utc
946 return 1
946 return 1
947
947
948 if not(self.setNextFile()):
948 if not(self.setNextFile()):
949 return 0
949 return 0
950
950
951 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
951 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
952 self.lastUTTime = self.basicHeaderObj.utc
952 self.lastUTTime = self.basicHeaderObj.utc
953
953
954 self.flagDiscontinuousBlock = 0
954 self.flagDiscontinuousBlock = 0
955
955
956 if deltaTime > self.maxTimeStep:
956 if deltaTime > self.maxTimeStep:
957 self.flagDiscontinuousBlock = 1
957 self.flagDiscontinuousBlock = 1
958
958
959 return 1
959 return 1
960
960
961 def readNextBlock(self):
961 def readNextBlock(self):
962
962
963 if not(self.__setNewBlock()):
963 if not(self.__setNewBlock()):
964 return 0
964 return 0
965
965
966 if not(self.readBlock()):
966 if not(self.readBlock()):
967 return 0
967 return 0
968
968
969 self.getBasicHeader()
969 self.getBasicHeader()
970
970
971 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
971 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
972 self.processingHeaderObj.dataBlocksPerFile,
972 self.processingHeaderObj.dataBlocksPerFile,
973 self.dataOut.datatime.ctime())
973 self.dataOut.datatime.ctime())
974 return 1
974 return 1
975
975
976 def __readFirstHeader(self):
976 def __readFirstHeader(self):
977
977
978 self.basicHeaderObj.read(self.fp)
978 self.basicHeaderObj.read(self.fp)
979 self.systemHeaderObj.read(self.fp)
979 self.systemHeaderObj.read(self.fp)
980 self.radarControllerHeaderObj.read(self.fp)
980 self.radarControllerHeaderObj.read(self.fp)
981 self.processingHeaderObj.read(self.fp)
981 self.processingHeaderObj.read(self.fp)
982
982
983 self.firstHeaderSize = self.basicHeaderObj.size
983 self.firstHeaderSize = self.basicHeaderObj.size
984
984
985 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
985 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
986 if datatype == 0:
986 if datatype == 0:
987 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
987 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
988 elif datatype == 1:
988 elif datatype == 1:
989 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
989 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
990 elif datatype == 2:
990 elif datatype == 2:
991 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
991 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
992 elif datatype == 3:
992 elif datatype == 3:
993 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
993 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
994 elif datatype == 4:
994 elif datatype == 4:
995 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
995 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
996 elif datatype == 5:
996 elif datatype == 5:
997 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
997 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
998 else:
998 else:
999 raise ValueError, 'Data type was not defined'
999 raise ValueError, 'Data type was not defined'
1000
1000
1001 self.dtype = datatype_str
1001 self.dtype = datatype_str
1002 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1002 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1003 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1003 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1004 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1004 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1005 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1005 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1006 self.getBlockDimension()
1006 self.getBlockDimension()
1007
1007
1008 def __verifyFile(self, filename, msgFlag=True):
1008 def __verifyFile(self, filename, msgFlag=True):
1009 msg = None
1009 msg = None
1010 try:
1010 try:
1011 fp = open(filename, 'rb')
1011 fp = open(filename, 'rb')
1012 currentPosition = fp.tell()
1012 currentPosition = fp.tell()
1013 except IOError:
1013 except IOError:
1014 traceback.print_exc()
1014 traceback.print_exc()
1015 if msgFlag:
1015 if msgFlag:
1016 print "[Reading] The file %s can't be opened" % (filename)
1016 print "[Reading] The file %s can't be opened" % (filename)
1017 return False
1017 return False
1018
1018
1019 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1019 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1020
1020
1021 if neededSize == 0:
1021 if neededSize == 0:
1022 basicHeaderObj = BasicHeader(LOCALTIME)
1022 basicHeaderObj = BasicHeader(LOCALTIME)
1023 systemHeaderObj = SystemHeader()
1023 systemHeaderObj = SystemHeader()
1024 radarControllerHeaderObj = RadarControllerHeader()
1024 radarControllerHeaderObj = RadarControllerHeader()
1025 processingHeaderObj = ProcessingHeader()
1025 processingHeaderObj = ProcessingHeader()
1026
1026
1027 try:
1027 try:
1028 if not( basicHeaderObj.read(fp) ): raise IOError
1028 if not( basicHeaderObj.read(fp) ): raise IOError
1029 if not( systemHeaderObj.read(fp) ): raise IOError
1029 if not( systemHeaderObj.read(fp) ): raise IOError
1030 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1030 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1031 if not( processingHeaderObj.read(fp) ): raise IOError
1031 if not( processingHeaderObj.read(fp) ): raise IOError
1032 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1032 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1033
1033
1034 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1034 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1035
1035
1036 except IOError:
1036 except IOError:
1037 traceback.print_exc()
1037 traceback.print_exc()
1038 sys.exit(0)
1038 # sys.exit(0)
1039
1039
1040 if msgFlag:
1040 if msgFlag:
1041 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1041 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1042
1042
1043 fp.close()
1043 fp.close()
1044 return False
1044 return False
1045 else:
1045 else:
1046 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1046 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1047
1047
1048 fp.close()
1048 fp.close()
1049 fileSize = os.path.getsize(filename)
1049 fileSize = os.path.getsize(filename)
1050 currentSize = fileSize - currentPosition
1050 currentSize = fileSize - currentPosition
1051 if currentSize < neededSize:
1051 if currentSize < neededSize:
1052 if msgFlag and (msg != None):
1052 if msgFlag and (msg != None):
1053 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1053 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1054 return False
1054 return False
1055
1055
1056 return True
1056 return True
1057
1057
1058 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1058 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1059
1059
1060 dateList = []
1060 dateList = []
1061 pathList = []
1061 pathList = []
1062
1062
1063 multi_path = path.split(',')
1063 multi_path = path.split(',')
1064
1064
1065 if not walk:
1065 if not walk:
1066
1066
1067 for single_path in multi_path:
1067 for single_path in multi_path:
1068
1068
1069 if not os.path.isdir(single_path):
1069 if not os.path.isdir(single_path):
1070 continue
1070 continue
1071
1071
1072 fileList = glob.glob1(single_path, "*"+ext)
1072 fileList = glob.glob1(single_path, "*"+ext)
1073
1073
1074 for thisFile in fileList:
1074 for thisFile in fileList:
1075
1075
1076 if not os.path.isfile(os.path.join(single_path, thisFile)):
1076 if not os.path.isfile(os.path.join(single_path, thisFile)):
1077 continue
1077 continue
1078
1078
1079 if not isRadarFile(thisFile):
1079 if not isRadarFile(thisFile):
1080 continue
1080 continue
1081
1081
1082 if not isFileInDateRange(thisFile, startDate, endDate):
1082 if not isFileInDateRange(thisFile, startDate, endDate):
1083 continue
1083 continue
1084
1084
1085 thisDate = getDateFromRadarFile(thisFile)
1085 thisDate = getDateFromRadarFile(thisFile)
1086
1086
1087 if thisDate in dateList:
1087 if thisDate in dateList:
1088 continue
1088 continue
1089
1089
1090 dateList.append(thisDate)
1090 dateList.append(thisDate)
1091 pathList.append(single_path)
1091 pathList.append(single_path)
1092
1092
1093 else:
1093 else:
1094 for single_path in multi_path:
1094 for single_path in multi_path:
1095
1095
1096 if not os.path.isdir(single_path):
1096 if not os.path.isdir(single_path):
1097 continue
1097 continue
1098
1098
1099 dirList = []
1099 dirList = []
1100
1100
1101 for thisPath in os.listdir(single_path):
1101 for thisPath in os.listdir(single_path):
1102
1102
1103 if not os.path.isdir(os.path.join(single_path,thisPath)):
1103 if not os.path.isdir(os.path.join(single_path,thisPath)):
1104 continue
1104 continue
1105
1105
1106 if not isRadarFolder(thisPath):
1106 if not isRadarFolder(thisPath):
1107 continue
1107 continue
1108
1108
1109 if not isFolderInDateRange(thisPath, startDate, endDate):
1109 if not isFolderInDateRange(thisPath, startDate, endDate):
1110 continue
1110 continue
1111
1111
1112 dirList.append(thisPath)
1112 dirList.append(thisPath)
1113
1113
1114 if not dirList:
1114 if not dirList:
1115 continue
1115 continue
1116
1116
1117 for thisDir in dirList:
1117 for thisDir in dirList:
1118
1118
1119 datapath = os.path.join(single_path, thisDir, expLabel)
1119 datapath = os.path.join(single_path, thisDir, expLabel)
1120 fileList = glob.glob1(datapath, "*"+ext)
1120 fileList = glob.glob1(datapath, "*"+ext)
1121
1121
1122 if len(fileList) < 1:
1122 if len(fileList) < 1:
1123 continue
1123 continue
1124
1124
1125 thisDate = getDateFromRadarFolder(thisDir)
1125 thisDate = getDateFromRadarFolder(thisDir)
1126
1126
1127 pathList.append(datapath)
1127 pathList.append(datapath)
1128 dateList.append(thisDate)
1128 dateList.append(thisDate)
1129
1129
1130 dateList.sort()
1130 dateList.sort()
1131
1131
1132 if include_path:
1132 if include_path:
1133 return dateList, pathList
1133 return dateList, pathList
1134
1134
1135 return dateList
1135 return dateList
1136
1136
1137 def setup(self,
1137 def setup(self,
1138 path=None,
1138 path=None,
1139 startDate=None,
1139 startDate=None,
1140 endDate=None,
1140 endDate=None,
1141 startTime=datetime.time(0,0,0),
1141 startTime=datetime.time(0,0,0),
1142 endTime=datetime.time(23,59,59),
1142 endTime=datetime.time(23,59,59),
1143 set=None,
1143 set=None,
1144 expLabel = "",
1144 expLabel = "",
1145 ext = None,
1145 ext = None,
1146 online = False,
1146 online = False,
1147 delay = 60,
1147 delay = 60,
1148 walk = True,
1148 walk = True,
1149 getblock = False,
1149 getblock = False,
1150 nTxs = 1,
1150 nTxs = 1,
1151 realtime=False):
1151 realtime=False):
1152
1152
1153 if path == None:
1153 if path == None:
1154 raise ValueError, "[Reading] The path is not valid"
1154 raise ValueError, "[Reading] The path is not valid"
1155
1155
1156 if ext == None:
1156 if ext == None:
1157 ext = self.ext
1157 ext = self.ext
1158
1158
1159 if online:
1159 if online:
1160 print "[Reading] Searching files in online mode..."
1160 print "[Reading] Searching files in online mode..."
1161
1161
1162 for nTries in range( self.nTries ):
1162 for nTries in range( self.nTries ):
1163 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1163 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1164
1164
1165 if fullpath:
1165 if fullpath:
1166 break
1166 break
1167
1167
1168 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1168 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1169 sleep( self.delay )
1169 sleep( self.delay )
1170
1170
1171 if not(fullpath):
1171 if not(fullpath):
1172 print "[Reading] There 'isn't any valid file in %s" % path
1172 print "[Reading] There 'isn't any valid file in %s" % path
1173 return None
1173 return
1174
1174
1175 self.year = year
1175 self.year = year
1176 self.doy = doy
1176 self.doy = doy
1177 self.set = set - 1
1177 self.set = set - 1
1178 self.path = path
1178 self.path = path
1179 self.foldercounter = foldercounter
1179 self.foldercounter = foldercounter
1180 last_set = None
1180 last_set = None
1181
1181
1182 else:
1182 else:
1183 print "[Reading] Searching files in offline mode ..."
1183 print "[Reading] Searching files in offline mode ..."
1184 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1184 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1185 startTime=startTime, endTime=endTime,
1185 startTime=startTime, endTime=endTime,
1186 set=set, expLabel=expLabel, ext=ext,
1186 set=set, expLabel=expLabel, ext=ext,
1187 walk=walk)
1187 walk=walk)
1188
1188
1189 if not(pathList):
1189 if not(pathList):
1190 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1190 print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1191 # datetime.datetime.combine(startDate,startTime).ctime(),
1191 datetime.datetime.combine(startDate,startTime).ctime(),
1192 # datetime.datetime.combine(endDate,endTime).ctime())
1192 datetime.datetime.combine(endDate,endTime).ctime())
1193 #
1193
1194 sys.exit(-1)
1194 # sys.exit(-1)
1195
1195
1196 self.fileIndex = -1
1197 self.pathList = []
1198 self.filenameList = []
1199 return
1196
1200
1197 self.fileIndex = -1
1201 self.fileIndex = -1
1198 self.pathList = pathList
1202 self.pathList = pathList
1199 self.filenameList = filenameList
1203 self.filenameList = filenameList
1200 file_name = os.path.basename(filenameList[-1])
1204 file_name = os.path.basename(filenameList[-1])
1201 basename, ext = os.path.splitext(file_name)
1205 basename, ext = os.path.splitext(file_name)
1202 last_set = int(basename[-3:])
1206 last_set = int(basename[-3:])
1203
1207
1204 self.online = online
1208 self.online = online
1205 self.realtime = realtime
1209 self.realtime = realtime
1206 self.delay = delay
1210 self.delay = delay
1207 ext = ext.lower()
1211 ext = ext.lower()
1208 self.ext = ext
1212 self.ext = ext
1209 self.getByBlock = getblock
1213 self.getByBlock = getblock
1210 self.nTxs = int(nTxs)
1214 self.nTxs = int(nTxs)
1211
1215
1212 if not(self.setNextFile()):
1216 if not(self.setNextFile()):
1213 if (startDate!=None) and (endDate!=None):
1217 if (startDate!=None) and (endDate!=None):
1214 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1218 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1215 elif startDate != None:
1219 elif startDate != None:
1216 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1220 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1217 else:
1221 else:
1218 print "[Reading] No files"
1222 print "[Reading] No files"
1219
1223
1220 sys.exit(-1)
1224 self.fileIndex = -1
1225 self.pathList = []
1226 self.filenameList = []
1227 return
1221
1228
1222 # self.getBasicHeader()
1229 # self.getBasicHeader()
1223
1230
1224 if last_set != None:
1231 if last_set != None:
1225 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1232 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1226 return
1233 return
1227
1234
1228 def getBasicHeader(self):
1235 def getBasicHeader(self):
1229
1236
1230 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1237 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1231
1238
1232 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1239 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1233
1240
1234 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1241 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1235
1242
1236 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1243 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1237
1244
1238 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1245 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1239
1246
1240 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1247 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1241
1248
1242 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1249 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1243
1250
1244 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1251 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1245
1252
1246
1253
1247 def getFirstHeader(self):
1254 def getFirstHeader(self):
1248
1255
1249 raise ValueError, "This method has not been implemented"
1256 raise ValueError, "This method has not been implemented"
1250
1257
1251 def getData(self):
1258 def getData(self):
1252
1259
1253 raise ValueError, "This method has not been implemented"
1260 raise ValueError, "This method has not been implemented"
1254
1261
1255 def hasNotDataInBuffer(self):
1262 def hasNotDataInBuffer(self):
1256
1263
1257 raise ValueError, "This method has not been implemented"
1264 raise ValueError, "This method has not been implemented"
1258
1265
1259 def readBlock(self):
1266 def readBlock(self):
1260
1267
1261 raise ValueError, "This method has not been implemented"
1268 raise ValueError, "This method has not been implemented"
1262
1269
1263 def isEndProcess(self):
1270 def isEndProcess(self):
1264
1271
1265 return self.flagNoMoreFiles
1272 return self.flagNoMoreFiles
1266
1273
1267 def printReadBlocks(self):
1274 def printReadBlocks(self):
1268
1275
1269 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1276 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1270
1277
1271 def printTotalBlocks(self):
1278 def printTotalBlocks(self):
1272
1279
1273 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1280 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1274
1281
1275 def printNumberOfBlock(self):
1282 def printNumberOfBlock(self):
1276
1283
1277 if self.flagIsNewBlock:
1284 if self.flagIsNewBlock:
1278 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1285 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1279 self.processingHeaderObj.dataBlocksPerFile,
1286 self.processingHeaderObj.dataBlocksPerFile,
1280 self.dataOut.datatime.ctime())
1287 self.dataOut.datatime.ctime())
1281
1288
1282 def printInfo(self):
1289 def printInfo(self):
1283
1290
1284 if self.__printInfo == False:
1291 if self.__printInfo == False:
1285 return
1292 return
1286
1293
1287 self.basicHeaderObj.printInfo()
1294 self.basicHeaderObj.printInfo()
1288 self.systemHeaderObj.printInfo()
1295 self.systemHeaderObj.printInfo()
1289 self.radarControllerHeaderObj.printInfo()
1296 self.radarControllerHeaderObj.printInfo()
1290 self.processingHeaderObj.printInfo()
1297 self.processingHeaderObj.printInfo()
1291
1298
1292 self.__printInfo = False
1299 self.__printInfo = False
1293
1300
1294
1301
1295 def run(self, **kwargs):
1302 def run(self, **kwargs):
1296
1303
1297 if not(self.isConfig):
1304 if not(self.isConfig):
1298
1305
1299 # self.dataOut = dataOut
1306 # self.dataOut = dataOut
1300 self.setup(**kwargs)
1307 self.setup(**kwargs)
1301 self.isConfig = True
1308 self.isConfig = True
1302
1309
1303 self.getData()
1310 self.getData()
1304
1311
1305 class JRODataWriter(JRODataIO):
1312 class JRODataWriter(JRODataIO):
1306
1313
1307 """
1314 """
1308 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1315 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1309 de los datos siempre se realiza por bloques.
1316 de los datos siempre se realiza por bloques.
1310 """
1317 """
1311
1318
1312 blockIndex = 0
1319 blockIndex = 0
1313
1320
1314 path = None
1321 path = None
1315
1322
1316 setFile = None
1323 setFile = None
1317
1324
1318 profilesPerBlock = None
1325 profilesPerBlock = None
1319
1326
1320 blocksPerFile = None
1327 blocksPerFile = None
1321
1328
1322 nWriteBlocks = 0
1329 nWriteBlocks = 0
1323
1330
1324 fileDate = None
1331 fileDate = None
1325
1332
1326 def __init__(self, dataOut=None):
1333 def __init__(self, dataOut=None):
1327 raise ValueError, "Not implemented"
1334 raise ValueError, "Not implemented"
1328
1335
1329
1336
1330 def hasAllDataInBuffer(self):
1337 def hasAllDataInBuffer(self):
1331 raise ValueError, "Not implemented"
1338 raise ValueError, "Not implemented"
1332
1339
1333
1340
1334 def setBlockDimension(self):
1341 def setBlockDimension(self):
1335 raise ValueError, "Not implemented"
1342 raise ValueError, "Not implemented"
1336
1343
1337
1344
1338 def writeBlock(self):
1345 def writeBlock(self):
1339 raise ValueError, "No implemented"
1346 raise ValueError, "No implemented"
1340
1347
1341
1348
1342 def putData(self):
1349 def putData(self):
1343 raise ValueError, "No implemented"
1350 raise ValueError, "No implemented"
1344
1351
1345
1352
1346 def getProcessFlags(self):
1353 def getProcessFlags(self):
1347
1354
1348 processFlags = 0
1355 processFlags = 0
1349
1356
1350 dtype_index = get_dtype_index(self.dtype)
1357 dtype_index = get_dtype_index(self.dtype)
1351 procflag_dtype = get_procflag_dtype(dtype_index)
1358 procflag_dtype = get_procflag_dtype(dtype_index)
1352
1359
1353 processFlags += procflag_dtype
1360 processFlags += procflag_dtype
1354
1361
1355 if self.dataOut.flagDecodeData:
1362 if self.dataOut.flagDecodeData:
1356 processFlags += PROCFLAG.DECODE_DATA
1363 processFlags += PROCFLAG.DECODE_DATA
1357
1364
1358 if self.dataOut.flagDeflipData:
1365 if self.dataOut.flagDeflipData:
1359 processFlags += PROCFLAG.DEFLIP_DATA
1366 processFlags += PROCFLAG.DEFLIP_DATA
1360
1367
1361 if self.dataOut.code is not None:
1368 if self.dataOut.code is not None:
1362 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1369 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1363
1370
1364 if self.dataOut.nCohInt > 1:
1371 if self.dataOut.nCohInt > 1:
1365 processFlags += PROCFLAG.COHERENT_INTEGRATION
1372 processFlags += PROCFLAG.COHERENT_INTEGRATION
1366
1373
1367 if self.dataOut.type == "Spectra":
1374 if self.dataOut.type == "Spectra":
1368 if self.dataOut.nIncohInt > 1:
1375 if self.dataOut.nIncohInt > 1:
1369 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1376 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1370
1377
1371 if self.dataOut.data_dc is not None:
1378 if self.dataOut.data_dc is not None:
1372 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1379 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1373
1380
1374 if self.dataOut.flagShiftFFT:
1381 if self.dataOut.flagShiftFFT:
1375 processFlags += PROCFLAG.SHIFT_FFT_DATA
1382 processFlags += PROCFLAG.SHIFT_FFT_DATA
1376
1383
1377 return processFlags
1384 return processFlags
1378
1385
1379 def setBasicHeader(self):
1386 def setBasicHeader(self):
1380
1387
1381 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1388 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1382 self.basicHeaderObj.version = self.versionFile
1389 self.basicHeaderObj.version = self.versionFile
1383 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1390 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1384
1391
1385 utc = numpy.floor(self.dataOut.utctime)
1392 utc = numpy.floor(self.dataOut.utctime)
1386 milisecond = (self.dataOut.utctime - utc)* 1000.0
1393 milisecond = (self.dataOut.utctime - utc)* 1000.0
1387
1394
1388 self.basicHeaderObj.utc = utc
1395 self.basicHeaderObj.utc = utc
1389 self.basicHeaderObj.miliSecond = milisecond
1396 self.basicHeaderObj.miliSecond = milisecond
1390 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1397 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1391 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1398 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1392 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1399 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1393
1400
1394 def setFirstHeader(self):
1401 def setFirstHeader(self):
1395 """
1402 """
1396 Obtiene una copia del First Header
1403 Obtiene una copia del First Header
1397
1404
1398 Affected:
1405 Affected:
1399
1406
1400 self.basicHeaderObj
1407 self.basicHeaderObj
1401 self.systemHeaderObj
1408 self.systemHeaderObj
1402 self.radarControllerHeaderObj
1409 self.radarControllerHeaderObj
1403 self.processingHeaderObj self.
1410 self.processingHeaderObj self.
1404
1411
1405 Return:
1412 Return:
1406 None
1413 None
1407 """
1414 """
1408
1415
1409 raise ValueError, "No implemented"
1416 raise ValueError, "No implemented"
1410
1417
1411 def __writeFirstHeader(self):
1418 def __writeFirstHeader(self):
1412 """
1419 """
1413 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1420 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1414
1421
1415 Affected:
1422 Affected:
1416 __dataType
1423 __dataType
1417
1424
1418 Return:
1425 Return:
1419 None
1426 None
1420 """
1427 """
1421
1428
1422 # CALCULAR PARAMETROS
1429 # CALCULAR PARAMETROS
1423
1430
1424 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1431 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1425 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1432 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1426
1433
1427 self.basicHeaderObj.write(self.fp)
1434 self.basicHeaderObj.write(self.fp)
1428 self.systemHeaderObj.write(self.fp)
1435 self.systemHeaderObj.write(self.fp)
1429 self.radarControllerHeaderObj.write(self.fp)
1436 self.radarControllerHeaderObj.write(self.fp)
1430 self.processingHeaderObj.write(self.fp)
1437 self.processingHeaderObj.write(self.fp)
1431
1438
1432 def __setNewBlock(self):
1439 def __setNewBlock(self):
1433 """
1440 """
1434 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1441 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1435
1442
1436 Return:
1443 Return:
1437 0 : si no pudo escribir nada
1444 0 : si no pudo escribir nada
1438 1 : Si escribio el Basic el First Header
1445 1 : Si escribio el Basic el First Header
1439 """
1446 """
1440 if self.fp == None:
1447 if self.fp == None:
1441 self.setNextFile()
1448 self.setNextFile()
1442
1449
1443 if self.flagIsNewFile:
1450 if self.flagIsNewFile:
1444 return 1
1451 return 1
1445
1452
1446 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1453 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1447 self.basicHeaderObj.write(self.fp)
1454 self.basicHeaderObj.write(self.fp)
1448 return 1
1455 return 1
1449
1456
1450 if not( self.setNextFile() ):
1457 if not( self.setNextFile() ):
1451 return 0
1458 return 0
1452
1459
1453 return 1
1460 return 1
1454
1461
1455
1462
1456 def writeNextBlock(self):
1463 def writeNextBlock(self):
1457 """
1464 """
1458 Selecciona el bloque siguiente de datos y los escribe en un file
1465 Selecciona el bloque siguiente de datos y los escribe en un file
1459
1466
1460 Return:
1467 Return:
1461 0 : Si no hizo pudo escribir el bloque de datos
1468 0 : Si no hizo pudo escribir el bloque de datos
1462 1 : Si no pudo escribir el bloque de datos
1469 1 : Si no pudo escribir el bloque de datos
1463 """
1470 """
1464 if not( self.__setNewBlock() ):
1471 if not( self.__setNewBlock() ):
1465 return 0
1472 return 0
1466
1473
1467 self.writeBlock()
1474 self.writeBlock()
1468
1475
1469 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1476 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1470 self.processingHeaderObj.dataBlocksPerFile)
1477 self.processingHeaderObj.dataBlocksPerFile)
1471
1478
1472 return 1
1479 return 1
1473
1480
1474 def setNextFile(self):
1481 def setNextFile(self):
1475 """
1482 """
1476 Determina el siguiente file que sera escrito
1483 Determina el siguiente file que sera escrito
1477
1484
1478 Affected:
1485 Affected:
1479 self.filename
1486 self.filename
1480 self.subfolder
1487 self.subfolder
1481 self.fp
1488 self.fp
1482 self.setFile
1489 self.setFile
1483 self.flagIsNewFile
1490 self.flagIsNewFile
1484
1491
1485 Return:
1492 Return:
1486 0 : Si el archivo no puede ser escrito
1493 0 : Si el archivo no puede ser escrito
1487 1 : Si el archivo esta listo para ser escrito
1494 1 : Si el archivo esta listo para ser escrito
1488 """
1495 """
1489 ext = self.ext
1496 ext = self.ext
1490 path = self.path
1497 path = self.path
1491
1498
1492 if self.fp != None:
1499 if self.fp != None:
1493 self.fp.close()
1500 self.fp.close()
1494
1501
1495 timeTuple = time.localtime( self.dataOut.utctime)
1502 timeTuple = time.localtime( self.dataOut.utctime)
1496 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1503 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1497
1504
1498 fullpath = os.path.join( path, subfolder )
1505 fullpath = os.path.join( path, subfolder )
1499 setFile = self.setFile
1506 setFile = self.setFile
1500
1507
1501 if not( os.path.exists(fullpath) ):
1508 if not( os.path.exists(fullpath) ):
1502 os.mkdir(fullpath)
1509 os.mkdir(fullpath)
1503 setFile = -1 #inicializo mi contador de seteo
1510 setFile = -1 #inicializo mi contador de seteo
1504 else:
1511 else:
1505 filesList = os.listdir( fullpath )
1512 filesList = os.listdir( fullpath )
1506 if len( filesList ) > 0:
1513 if len( filesList ) > 0:
1507 filesList = sorted( filesList, key=str.lower )
1514 filesList = sorted( filesList, key=str.lower )
1508 filen = filesList[-1]
1515 filen = filesList[-1]
1509 # el filename debera tener el siguiente formato
1516 # el filename debera tener el siguiente formato
1510 # 0 1234 567 89A BCDE (hex)
1517 # 0 1234 567 89A BCDE (hex)
1511 # x YYYY DDD SSS .ext
1518 # x YYYY DDD SSS .ext
1512 if isNumber( filen[8:11] ):
1519 if isNumber( filen[8:11] ):
1513 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1520 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1514 else:
1521 else:
1515 setFile = -1
1522 setFile = -1
1516 else:
1523 else:
1517 setFile = -1 #inicializo mi contador de seteo
1524 setFile = -1 #inicializo mi contador de seteo
1518
1525
1519 setFile += 1
1526 setFile += 1
1520
1527
1521 #If this is a new day it resets some values
1528 #If this is a new day it resets some values
1522 if self.dataOut.datatime.date() > self.fileDate:
1529 if self.dataOut.datatime.date() > self.fileDate:
1523 setFile = 0
1530 setFile = 0
1524 self.nTotalBlocks = 0
1531 self.nTotalBlocks = 0
1525
1532
1526 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1533 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1527
1534
1528 filename = os.path.join( path, subfolder, filen )
1535 filename = os.path.join( path, subfolder, filen )
1529
1536
1530 fp = open( filename,'wb' )
1537 fp = open( filename,'wb' )
1531
1538
1532 self.blockIndex = 0
1539 self.blockIndex = 0
1533
1540
1534 #guardando atributos
1541 #guardando atributos
1535 self.filename = filename
1542 self.filename = filename
1536 self.subfolder = subfolder
1543 self.subfolder = subfolder
1537 self.fp = fp
1544 self.fp = fp
1538 self.setFile = setFile
1545 self.setFile = setFile
1539 self.flagIsNewFile = 1
1546 self.flagIsNewFile = 1
1540 self.fileDate = self.dataOut.datatime.date()
1547 self.fileDate = self.dataOut.datatime.date()
1541
1548
1542 self.setFirstHeader()
1549 self.setFirstHeader()
1543
1550
1544 print '[Writing] Opening file: %s'%self.filename
1551 print '[Writing] Opening file: %s'%self.filename
1545
1552
1546 self.__writeFirstHeader()
1553 self.__writeFirstHeader()
1547
1554
1548 return 1
1555 return 1
1549
1556
1550 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1557 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1551 """
1558 """
1552 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1559 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1553
1560
1554 Inputs:
1561 Inputs:
1555 path : directory where data will be saved
1562 path : directory where data will be saved
1556 profilesPerBlock : number of profiles per block
1563 profilesPerBlock : number of profiles per block
1557 set : initial file set
1564 set : initial file set
1558 datatype : An integer number that defines data type:
1565 datatype : An integer number that defines data type:
1559 0 : int8 (1 byte)
1566 0 : int8 (1 byte)
1560 1 : int16 (2 bytes)
1567 1 : int16 (2 bytes)
1561 2 : int32 (4 bytes)
1568 2 : int32 (4 bytes)
1562 3 : int64 (8 bytes)
1569 3 : int64 (8 bytes)
1563 4 : float32 (4 bytes)
1570 4 : float32 (4 bytes)
1564 5 : double64 (8 bytes)
1571 5 : double64 (8 bytes)
1565
1572
1566 Return:
1573 Return:
1567 0 : Si no realizo un buen seteo
1574 0 : Si no realizo un buen seteo
1568 1 : Si realizo un buen seteo
1575 1 : Si realizo un buen seteo
1569 """
1576 """
1570
1577
1571 if ext == None:
1578 if ext == None:
1572 ext = self.ext
1579 ext = self.ext
1573
1580
1574 self.ext = ext.lower()
1581 self.ext = ext.lower()
1575
1582
1576 self.path = path
1583 self.path = path
1577
1584
1578 if set is None:
1585 if set is None:
1579 self.setFile = -1
1586 self.setFile = -1
1580 else:
1587 else:
1581 self.setFile = set - 1
1588 self.setFile = set - 1
1582
1589
1583 self.blocksPerFile = blocksPerFile
1590 self.blocksPerFile = blocksPerFile
1584
1591
1585 self.profilesPerBlock = profilesPerBlock
1592 self.profilesPerBlock = profilesPerBlock
1586
1593
1587 self.dataOut = dataOut
1594 self.dataOut = dataOut
1588 self.fileDate = self.dataOut.datatime.date()
1595 self.fileDate = self.dataOut.datatime.date()
1589 #By default
1596 #By default
1590 self.dtype = self.dataOut.dtype
1597 self.dtype = self.dataOut.dtype
1591
1598
1592 if datatype is not None:
1599 if datatype is not None:
1593 self.dtype = get_numpy_dtype(datatype)
1600 self.dtype = get_numpy_dtype(datatype)
1594
1601
1595 if not(self.setNextFile()):
1602 if not(self.setNextFile()):
1596 print "[Writing] There isn't a next file"
1603 print "[Writing] There isn't a next file"
1597 return 0
1604 return 0
1598
1605
1599 self.setBlockDimension()
1606 self.setBlockDimension()
1600
1607
1601 return 1
1608 return 1
1602
1609
1603 def run(self, dataOut, **kwargs):
1610 def run(self, dataOut, **kwargs):
1604
1611
1605 if not(self.isConfig):
1612 if not(self.isConfig):
1606
1613
1607 self.setup(dataOut, **kwargs)
1614 self.setup(dataOut, **kwargs)
1608 self.isConfig = True
1615 self.isConfig = True
1609
1616
1610 self.putData()
1617 self.putData()
1611
1618
General Comments 0
You need to be logged in to leave comments. Login now