##// END OF EJS Templates
schain
RSS
jroIO_base.py: Compare dates when startTime > endTime
Miguel Valdez -
r652:303180c98d9d
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1585 +1,1596
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 traceback.print_exc()
71 traceback.print_exc()
72 raise IOError, "The file %s can't be opened" %(filename)
72 raise IOError, "The file %s can't be opened" %(filename)
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isFileInTimeRange(filename, startTime, endTime):
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
90 Inputs:
90 Inputs:
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92
92
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94
95 endDate : fecha final del rango seleccionado en formato datetime.date
96
93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94
98
95 endTime : tiempo final del rango seleccionado en formato datetime.time
99 endTime : tiempo final del rango seleccionado en formato datetime.time
96
100
97 Return:
101 Return:
98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 fecha especificado, de lo contrario retorna False.
103 fecha especificado, de lo contrario retorna False.
100
104
101 Excepciones:
105 Excepciones:
102 Si el archivo no existe o no puede ser abierto
106 Si el archivo no existe o no puede ser abierto
103 Si la cabecera no puede ser leida.
107 Si la cabecera no puede ser leida.
104
108
105 """
109 """
106
110
107
111
108 try:
112 try:
109 fp = open(filename,'rb')
113 fp = open(filename,'rb')
110 except IOError:
114 except IOError:
111 traceback.print_exc()
115 traceback.print_exc()
112 raise IOError, "The file %s can't be opened" %(filename)
116 raise IOError, "The file %s can't be opened" %(filename)
113
117
114 basicHeaderObj = BasicHeader(LOCALTIME)
118 basicHeaderObj = BasicHeader(LOCALTIME)
115 sts = basicHeaderObj.read(fp)
119 sts = basicHeaderObj.read(fp)
116 fp.close()
120 fp.close()
117
121
118 thisDatetime = basicHeaderObj.datatime
122 thisDatetime = basicHeaderObj.datatime
123 thisDate = thisDatetime.date()
119 thisTime = thisDatetime.time()
124 thisTime = thisDatetime.time()
120
125
121 if not(sts):
126 if not(sts):
122 print "Skipping the file %s because it has not a valid header" %(filename)
127 print "Skipping the file %s because it has not a valid header" %(filename)
123 return None
128 return None
124
129
125 #General case
130 #General case
126 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
127 #-----------o----------------------------o-----------
132 #-----------o----------------------------o-----------
128 # startTime endTime
133 # startTime endTime
129
134
130 if endTime >= startTime:
135 if endTime >= startTime:
131 if (thisTime < startTime) or (thisTime > endTime):
136 if (thisTime < startTime) or (thisTime > endTime):
132 return None
137 return None
133
138
134 return thisDatetime
139 return thisDatetime
135
140
136 #If endTime < startTime then endTime belongs to the next day
141 #If endTime < startTime then endTime belongs to the next day
137
142
138
143
139 #<<<<<<<<<<<o o>>>>>>>>>>>
144 #<<<<<<<<<<<o o>>>>>>>>>>>
140 #-----------o----------------------------o-----------
145 #-----------o----------------------------o-----------
141 # endTime startTime
146 # endTime startTime
142
147
148 if (thisDate == startDate) and (thisTime < startTime):
149 return None
150
151 if (thisDate == endDate) and (thisTime > endTime):
152 return None
153
143 if (thisTime < startTime) and (thisTime > endTime):
154 if (thisTime < startTime) and (thisTime > endTime):
144 return None
155 return None
145
156
146 return thisDatetime
157 return thisDatetime
147
158
148 def isFolderInDateRange(folder, startDate=None, endDate=None):
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
149 """
160 """
150 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
151
162
152 Inputs:
163 Inputs:
153 folder : nombre completo del directorio.
164 folder : nombre completo del directorio.
154 Su formato deberia ser "/path_root/?YYYYDDD"
165 Su formato deberia ser "/path_root/?YYYYDDD"
155
166
156 siendo:
167 siendo:
157 YYYY : Anio (ejemplo 2015)
168 YYYY : Anio (ejemplo 2015)
158 DDD : Dia del anio (ejemplo 305)
169 DDD : Dia del anio (ejemplo 305)
159
170
160 startDate : fecha inicial del rango seleccionado en formato datetime.date
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
161
172
162 endDate : fecha final del rango seleccionado en formato datetime.date
173 endDate : fecha final del rango seleccionado en formato datetime.date
163
174
164 Return:
175 Return:
165 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
166 fecha especificado, de lo contrario retorna False.
177 fecha especificado, de lo contrario retorna False.
167 Excepciones:
178 Excepciones:
168 Si el directorio no tiene el formato adecuado
179 Si el directorio no tiene el formato adecuado
169 """
180 """
170
181
171 basename = os.path.basename(folder)
182 basename = os.path.basename(folder)
172
183
173 if not isRadarFolder(basename):
184 if not isRadarFolder(basename):
174 raise IOError, "The folder %s has not the rigth format" %folder
185 raise IOError, "The folder %s has not the rigth format" %folder
175
186
176 if startDate and endDate:
187 if startDate and endDate:
177 thisDate = getDateFromRadarFolder(basename)
188 thisDate = getDateFromRadarFolder(basename)
178
189
179 if thisDate < startDate:
190 if thisDate < startDate:
180 return 0
191 return 0
181
192
182 if thisDate > endDate:
193 if thisDate > endDate:
183 return 0
194 return 0
184
195
185 return 1
196 return 1
186
197
187 def isFileInDateRange(filename, startDate=None, endDate=None):
198 def isFileInDateRange(filename, startDate=None, endDate=None):
188 """
199 """
189 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
200 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
190
201
191 Inputs:
202 Inputs:
192 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
203 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
193
204
194 Su formato deberia ser "?YYYYDDDsss"
205 Su formato deberia ser "?YYYYDDDsss"
195
206
196 siendo:
207 siendo:
197 YYYY : Anio (ejemplo 2015)
208 YYYY : Anio (ejemplo 2015)
198 DDD : Dia del anio (ejemplo 305)
209 DDD : Dia del anio (ejemplo 305)
199 sss : set
210 sss : set
200
211
201 startDate : fecha inicial del rango seleccionado en formato datetime.date
212 startDate : fecha inicial del rango seleccionado en formato datetime.date
202
213
203 endDate : fecha final del rango seleccionado en formato datetime.date
214 endDate : fecha final del rango seleccionado en formato datetime.date
204
215
205 Return:
216 Return:
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
217 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 fecha especificado, de lo contrario retorna False.
218 fecha especificado, de lo contrario retorna False.
208 Excepciones:
219 Excepciones:
209 Si el archivo no tiene el formato adecuado
220 Si el archivo no tiene el formato adecuado
210 """
221 """
211
222
212 basename = os.path.basename(filename)
223 basename = os.path.basename(filename)
213
224
214 if not isRadarFile(basename):
225 if not isRadarFile(basename):
215 raise IOError, "The filename %s has not the rigth format" %filename
226 raise IOError, "The filename %s has not the rigth format" %filename
216
227
217 if startDate and endDate:
228 if startDate and endDate:
218 thisDate = getDateFromRadarFile(basename)
229 thisDate = getDateFromRadarFile(basename)
219
230
220 if thisDate < startDate:
231 if thisDate < startDate:
221 return 0
232 return 0
222
233
223 if thisDate > endDate:
234 if thisDate > endDate:
224 return 0
235 return 0
225
236
226 return 1
237 return 1
227
238
228 def getFileFromSet(path, ext, set):
239 def getFileFromSet(path, ext, set):
229 validFilelist = []
240 validFilelist = []
230 fileList = os.listdir(path)
241 fileList = os.listdir(path)
231
242
232 # 0 1234 567 89A BCDE
243 # 0 1234 567 89A BCDE
233 # H YYYY DDD SSS .ext
244 # H YYYY DDD SSS .ext
234
245
235 for thisFile in fileList:
246 for thisFile in fileList:
236 try:
247 try:
237 year = int(thisFile[1:5])
248 year = int(thisFile[1:5])
238 doy = int(thisFile[5:8])
249 doy = int(thisFile[5:8])
239 except:
250 except:
240 continue
251 continue
241
252
242 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
253 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
243 continue
254 continue
244
255
245 validFilelist.append(thisFile)
256 validFilelist.append(thisFile)
246
257
247 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
258 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
248
259
249 if len(myfile)!= 0:
260 if len(myfile)!= 0:
250 return myfile[0]
261 return myfile[0]
251 else:
262 else:
252 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
263 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
253 print 'the filename %s does not exist'%filename
264 print 'the filename %s does not exist'%filename
254 print '...going to the last file: '
265 print '...going to the last file: '
255
266
256 if validFilelist:
267 if validFilelist:
257 validFilelist = sorted( validFilelist, key=str.lower )
268 validFilelist = sorted( validFilelist, key=str.lower )
258 return validFilelist[-1]
269 return validFilelist[-1]
259
270
260 return None
271 return None
261
272
262 def getlastFileFromPath(path, ext):
273 def getlastFileFromPath(path, ext):
263 """
274 """
264 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
275 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
265 al final de la depuracion devuelve el ultimo file de la lista que quedo.
276 al final de la depuracion devuelve el ultimo file de la lista que quedo.
266
277
267 Input:
278 Input:
268 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
279 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
269 ext : extension de los files contenidos en una carpeta
280 ext : extension de los files contenidos en una carpeta
270
281
271 Return:
282 Return:
272 El ultimo file de una determinada carpeta, no se considera el path.
283 El ultimo file de una determinada carpeta, no se considera el path.
273 """
284 """
274 validFilelist = []
285 validFilelist = []
275 fileList = os.listdir(path)
286 fileList = os.listdir(path)
276
287
277 # 0 1234 567 89A BCDE
288 # 0 1234 567 89A BCDE
278 # H YYYY DDD SSS .ext
289 # H YYYY DDD SSS .ext
279
290
280 for thisFile in fileList:
291 for thisFile in fileList:
281
292
282 year = thisFile[1:5]
293 year = thisFile[1:5]
283 if not isNumber(year):
294 if not isNumber(year):
284 continue
295 continue
285
296
286 doy = thisFile[5:8]
297 doy = thisFile[5:8]
287 if not isNumber(doy):
298 if not isNumber(doy):
288 continue
299 continue
289
300
290 year = int(year)
301 year = int(year)
291 doy = int(doy)
302 doy = int(doy)
292
303
293 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
294 continue
305 continue
295
306
296 validFilelist.append(thisFile)
307 validFilelist.append(thisFile)
297
308
298 if validFilelist:
309 if validFilelist:
299 validFilelist = sorted( validFilelist, key=str.lower )
310 validFilelist = sorted( validFilelist, key=str.lower )
300 return validFilelist[-1]
311 return validFilelist[-1]
301
312
302 return None
313 return None
303
314
304 def checkForRealPath(path, foldercounter, year, doy, set, ext):
315 def checkForRealPath(path, foldercounter, year, doy, set, ext):
305 """
316 """
306 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
317 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
307 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
318 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
308 el path exacto de un determinado file.
319 el path exacto de un determinado file.
309
320
310 Example :
321 Example :
311 nombre correcto del file es .../.../D2009307/P2009307367.ext
322 nombre correcto del file es .../.../D2009307/P2009307367.ext
312
323
313 Entonces la funcion prueba con las siguientes combinaciones
324 Entonces la funcion prueba con las siguientes combinaciones
314 .../.../y2009307367.ext
325 .../.../y2009307367.ext
315 .../.../Y2009307367.ext
326 .../.../Y2009307367.ext
316 .../.../x2009307/y2009307367.ext
327 .../.../x2009307/y2009307367.ext
317 .../.../x2009307/Y2009307367.ext
328 .../.../x2009307/Y2009307367.ext
318 .../.../X2009307/y2009307367.ext
329 .../.../X2009307/y2009307367.ext
319 .../.../X2009307/Y2009307367.ext
330 .../.../X2009307/Y2009307367.ext
320 siendo para este caso, la ultima combinacion de letras, identica al file buscado
331 siendo para este caso, la ultima combinacion de letras, identica al file buscado
321
332
322 Return:
333 Return:
323 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
334 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
324 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
335 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
325 para el filename
336 para el filename
326 """
337 """
327 fullfilename = None
338 fullfilename = None
328 find_flag = False
339 find_flag = False
329 filename = None
340 filename = None
330
341
331 prefixDirList = [None,'d','D']
342 prefixDirList = [None,'d','D']
332 if ext.lower() == ".r": #voltage
343 if ext.lower() == ".r": #voltage
333 prefixFileList = ['d','D']
344 prefixFileList = ['d','D']
334 elif ext.lower() == ".pdata": #spectra
345 elif ext.lower() == ".pdata": #spectra
335 prefixFileList = ['p','P']
346 prefixFileList = ['p','P']
336 else:
347 else:
337 return None, filename
348 return None, filename
338
349
339 #barrido por las combinaciones posibles
350 #barrido por las combinaciones posibles
340 for prefixDir in prefixDirList:
351 for prefixDir in prefixDirList:
341 thispath = path
352 thispath = path
342 if prefixDir != None:
353 if prefixDir != None:
343 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
354 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
344 if foldercounter == 0:
355 if foldercounter == 0:
345 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
356 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
346 else:
357 else:
347 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
358 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
348 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
359 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
349 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
360 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
350 fullfilename = os.path.join( thispath, filename ) #formo el path completo
361 fullfilename = os.path.join( thispath, filename ) #formo el path completo
351
362
352 if os.path.exists( fullfilename ): #verifico que exista
363 if os.path.exists( fullfilename ): #verifico que exista
353 find_flag = True
364 find_flag = True
354 break
365 break
355 if find_flag:
366 if find_flag:
356 break
367 break
357
368
358 if not(find_flag):
369 if not(find_flag):
359 return None, filename
370 return None, filename
360
371
361 return fullfilename, filename
372 return fullfilename, filename
362
373
363 def isRadarFolder(folder):
374 def isRadarFolder(folder):
364 try:
375 try:
365 year = int(folder[1:5])
376 year = int(folder[1:5])
366 doy = int(folder[5:8])
377 doy = int(folder[5:8])
367 except:
378 except:
368 return 0
379 return 0
369
380
370 return 1
381 return 1
371
382
372 def isRadarFile(file):
383 def isRadarFile(file):
373 try:
384 try:
374 year = int(file[1:5])
385 year = int(file[1:5])
375 doy = int(file[5:8])
386 doy = int(file[5:8])
376 set = int(file[8:11])
387 set = int(file[8:11])
377 except:
388 except:
378 return 0
389 return 0
379
390
380 return 1
391 return 1
381
392
382 def getDateFromRadarFile(file):
393 def getDateFromRadarFile(file):
383 try:
394 try:
384 year = int(file[1:5])
395 year = int(file[1:5])
385 doy = int(file[5:8])
396 doy = int(file[5:8])
386 set = int(file[8:11])
397 set = int(file[8:11])
387 except:
398 except:
388 return None
399 return None
389
400
390 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
401 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
391 return thisDate
402 return thisDate
392
403
393 def getDateFromRadarFolder(folder):
404 def getDateFromRadarFolder(folder):
394 try:
405 try:
395 year = int(folder[1:5])
406 year = int(folder[1:5])
396 doy = int(folder[5:8])
407 doy = int(folder[5:8])
397 except:
408 except:
398 return None
409 return None
399
410
400 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
411 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
401 return thisDate
412 return thisDate
402
413
403 class JRODataIO:
414 class JRODataIO:
404
415
405 c = 3E8
416 c = 3E8
406
417
407 isConfig = False
418 isConfig = False
408
419
409 basicHeaderObj = None
420 basicHeaderObj = None
410
421
411 systemHeaderObj = None
422 systemHeaderObj = None
412
423
413 radarControllerHeaderObj = None
424 radarControllerHeaderObj = None
414
425
415 processingHeaderObj = None
426 processingHeaderObj = None
416
427
417 online = 0
428 online = 0
418
429
419 dtype = None
430 dtype = None
420
431
421 pathList = []
432 pathList = []
422
433
423 filenameList = []
434 filenameList = []
424
435
425 filename = None
436 filename = None
426
437
427 ext = None
438 ext = None
428
439
429 flagIsNewFile = 1
440 flagIsNewFile = 1
430
441
431 flagDiscontinuousBlock = 0
442 flagDiscontinuousBlock = 0
432
443
433 flagIsNewBlock = 0
444 flagIsNewBlock = 0
434
445
435 fp = None
446 fp = None
436
447
437 firstHeaderSize = 0
448 firstHeaderSize = 0
438
449
439 basicHeaderSize = 24
450 basicHeaderSize = 24
440
451
441 versionFile = 1103
452 versionFile = 1103
442
453
443 fileSize = None
454 fileSize = None
444
455
445 # ippSeconds = None
456 # ippSeconds = None
446
457
447 fileSizeByHeader = None
458 fileSizeByHeader = None
448
459
449 fileIndex = None
460 fileIndex = None
450
461
451 profileIndex = None
462 profileIndex = None
452
463
453 blockIndex = None
464 blockIndex = None
454
465
455 nTotalBlocks = None
466 nTotalBlocks = None
456
467
457 maxTimeStep = 30
468 maxTimeStep = 30
458
469
459 lastUTTime = None
470 lastUTTime = None
460
471
461 datablock = None
472 datablock = None
462
473
463 dataOut = None
474 dataOut = None
464
475
465 blocksize = None
476 blocksize = None
466
477
467 getByBlock = False
478 getByBlock = False
468
479
469 def __init__(self):
480 def __init__(self):
470
481
471 raise ValueError, "Not implemented"
482 raise ValueError, "Not implemented"
472
483
473 def run(self):
484 def run(self):
474
485
475 raise ValueError, "Not implemented"
486 raise ValueError, "Not implemented"
476
487
477 def getDtypeWidth(self):
488 def getDtypeWidth(self):
478
489
479 dtype_index = get_dtype_index(self.dtype)
490 dtype_index = get_dtype_index(self.dtype)
480 dtype_width = get_dtype_width(dtype_index)
491 dtype_width = get_dtype_width(dtype_index)
481
492
482 return dtype_width
493 return dtype_width
483
494
484 class JRODataReader(JRODataIO):
495 class JRODataReader(JRODataIO):
485
496
486 nReadBlocks = 0
497 nReadBlocks = 0
487
498
488 delay = 10 #number of seconds waiting a new file
499 delay = 10 #number of seconds waiting a new file
489
500
490 nTries = 3 #quantity tries
501 nTries = 3 #quantity tries
491
502
492 nFiles = 3 #number of files for searching
503 nFiles = 3 #number of files for searching
493
504
494 path = None
505 path = None
495
506
496 foldercounter = 0
507 foldercounter = 0
497
508
498 flagNoMoreFiles = 0
509 flagNoMoreFiles = 0
499
510
500 datetimeList = []
511 datetimeList = []
501
512
502 __isFirstTimeOnline = 1
513 __isFirstTimeOnline = 1
503
514
504 __printInfo = True
515 __printInfo = True
505
516
506 profileIndex = None
517 profileIndex = None
507
518
508 nTxs = 1
519 nTxs = 1
509
520
510 txIndex = None
521 txIndex = None
511
522
512 def __init__(self):
523 def __init__(self):
513
524
514 """
525 """
515
526
516 """
527 """
517
528
518 # raise NotImplementedError, "This method has not been implemented"
529 # raise NotImplementedError, "This method has not been implemented"
519
530
520
531
521 def createObjByDefault(self):
532 def createObjByDefault(self):
522 """
533 """
523
534
524 """
535 """
525 raise NotImplementedError, "This method has not been implemented"
536 raise NotImplementedError, "This method has not been implemented"
526
537
527 def getBlockDimension(self):
538 def getBlockDimension(self):
528
539
529 raise NotImplementedError, "No implemented"
540 raise NotImplementedError, "No implemented"
530
541
531 def __searchFilesOffLine(self,
542 def __searchFilesOffLine(self,
532 path,
543 path,
533 startDate=None,
544 startDate=None,
534 endDate=None,
545 endDate=None,
535 startTime=datetime.time(0,0,0),
546 startTime=datetime.time(0,0,0),
536 endTime=datetime.time(23,59,59),
547 endTime=datetime.time(23,59,59),
537 set=None,
548 set=None,
538 expLabel='',
549 expLabel='',
539 ext='.r',
550 ext='.r',
540 walk=True):
551 walk=True):
541
552
542 self.filenameList = []
553 self.filenameList = []
543 self.datetimeList = []
554 self.datetimeList = []
544
555
545 pathList = []
556 pathList = []
546
557
547 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
558 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
548
559
549 if dateList == []:
560 if dateList == []:
550 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
561 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
551 datetime.datetime.combine(startDate,startTime).ctime(),
562 datetime.datetime.combine(startDate,startTime).ctime(),
552 datetime.datetime.combine(endDate,endTime).ctime())
563 datetime.datetime.combine(endDate,endTime).ctime())
553
564
554 return None, None
565 return None, None
555
566
556 if len(dateList) > 1:
567 if len(dateList) > 1:
557 print "[Reading] %d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
568 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
558 else:
569 else:
559 print "[Reading] data was found for the date %s" %(dateList[0])
570 print "[Reading] data was found for the date %s" %(dateList[0])
560
571
561 filenameList = []
572 filenameList = []
562 datetimeList = []
573 datetimeList = []
563
574
564 for thisPath in pathList:
575 for thisPath in pathList:
565 # thisPath = pathList[pathDict[file]]
576 # thisPath = pathList[pathDict[file]]
566
577
567 fileList = glob.glob1(thisPath, "*%s" %ext)
578 fileList = glob.glob1(thisPath, "*%s" %ext)
568 fileList.sort()
579 fileList.sort()
569
580
570 for file in fileList:
581 for file in fileList:
571
582
572 filename = os.path.join(thisPath,file)
583 filename = os.path.join(thisPath,file)
573
584
574 if not isFileInDateRange(filename, startDate, endDate):
585 if not isFileInDateRange(filename, startDate, endDate):
575 continue
586 continue
576
587
577 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
588 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
578
589
579 if not(thisDatetime):
590 if not(thisDatetime):
580 continue
591 continue
581
592
582 filenameList.append(filename)
593 filenameList.append(filename)
583 datetimeList.append(thisDatetime)
594 datetimeList.append(thisDatetime)
584
595
585 if not(filenameList):
596 if not(filenameList):
586 print "[Reading] Any file was found int time range %s - %s" %(startTime.ctime(), endTime.ctime())
597 print "[Reading] Any file was found int time range %s - %s" %(startTime.ctime(), endTime.ctime())
587 return None, None
598 return None, None
588
599
589 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
600 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
590 print
601 print
591
602
592 for i in range(len(filenameList)):
603 for i in range(len(filenameList)):
593 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
604 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
594
605
595 self.filenameList = filenameList
606 self.filenameList = filenameList
596 self.datetimeList = datetimeList
607 self.datetimeList = datetimeList
597
608
598 return pathList, filenameList
609 return pathList, filenameList
599
610
600 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
611 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
601
612
602 """
613 """
603 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
614 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
604 devuelve el archivo encontrado ademas de otros datos.
615 devuelve el archivo encontrado ademas de otros datos.
605
616
606 Input:
617 Input:
607 path : carpeta donde estan contenidos los files que contiene data
618 path : carpeta donde estan contenidos los files que contiene data
608
619
609 expLabel : Nombre del subexperimento (subfolder)
620 expLabel : Nombre del subexperimento (subfolder)
610
621
611 ext : extension de los files
622 ext : extension de los files
612
623
613 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
624 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
614
625
615 Return:
626 Return:
616 directory : eL directorio donde esta el file encontrado
627 directory : eL directorio donde esta el file encontrado
617 filename : el ultimo file de una determinada carpeta
628 filename : el ultimo file de una determinada carpeta
618 year : el anho
629 year : el anho
619 doy : el numero de dia del anho
630 doy : el numero de dia del anho
620 set : el set del archivo
631 set : el set del archivo
621
632
622
633
623 """
634 """
624 dirList = []
635 dirList = []
625
636
626 if not walk:
637 if not walk:
627 fullpath = path
638 fullpath = path
628 foldercounter = 0
639 foldercounter = 0
629 else:
640 else:
630 #Filtra solo los directorios
641 #Filtra solo los directorios
631 for thisPath in os.listdir(path):
642 for thisPath in os.listdir(path):
632 if not os.path.isdir(os.path.join(path,thisPath)):
643 if not os.path.isdir(os.path.join(path,thisPath)):
633 continue
644 continue
634 if not isRadarFolder(thisPath):
645 if not isRadarFolder(thisPath):
635 continue
646 continue
636
647
637 dirList.append(thisPath)
648 dirList.append(thisPath)
638
649
639 if not(dirList):
650 if not(dirList):
640 return None, None, None, None, None, None
651 return None, None, None, None, None, None
641
652
642 dirList = sorted( dirList, key=str.lower )
653 dirList = sorted( dirList, key=str.lower )
643
654
644 doypath = dirList[-1]
655 doypath = dirList[-1]
645 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
656 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
646 fullpath = os.path.join(path, doypath, expLabel)
657 fullpath = os.path.join(path, doypath, expLabel)
647
658
648
659
649 print "[Reading] %s folder was found: " %(fullpath )
660 print "[Reading] %s folder was found: " %(fullpath )
650
661
651 if set == None:
662 if set == None:
652 filename = getlastFileFromPath(fullpath, ext)
663 filename = getlastFileFromPath(fullpath, ext)
653 else:
664 else:
654 filename = getFileFromSet(fullpath, ext, set)
665 filename = getFileFromSet(fullpath, ext, set)
655
666
656 if not(filename):
667 if not(filename):
657 return None, None, None, None, None, None
668 return None, None, None, None, None, None
658
669
659 print "[Reading] %s file was found" %(filename)
670 print "[Reading] %s file was found" %(filename)
660
671
661 if not(self.__verifyFile(os.path.join(fullpath, filename))):
672 if not(self.__verifyFile(os.path.join(fullpath, filename))):
662 return None, None, None, None, None, None
673 return None, None, None, None, None, None
663
674
664 year = int( filename[1:5] )
675 year = int( filename[1:5] )
665 doy = int( filename[5:8] )
676 doy = int( filename[5:8] )
666 set = int( filename[8:11] )
677 set = int( filename[8:11] )
667
678
668 return fullpath, foldercounter, filename, year, doy, set
679 return fullpath, foldercounter, filename, year, doy, set
669
680
670 def __setNextFileOffline(self):
681 def __setNextFileOffline(self):
671
682
672 idFile = self.fileIndex
683 idFile = self.fileIndex
673
684
674 while (True):
685 while (True):
675 idFile += 1
686 idFile += 1
676 if not(idFile < len(self.filenameList)):
687 if not(idFile < len(self.filenameList)):
677 self.flagNoMoreFiles = 1
688 self.flagNoMoreFiles = 1
678 # print "[Reading] No more Files"
689 # print "[Reading] No more Files"
679 return 0
690 return 0
680
691
681 filename = self.filenameList[idFile]
692 filename = self.filenameList[idFile]
682
693
683 if not(self.__verifyFile(filename)):
694 if not(self.__verifyFile(filename)):
684 continue
695 continue
685
696
686 fileSize = os.path.getsize(filename)
697 fileSize = os.path.getsize(filename)
687 fp = open(filename,'rb')
698 fp = open(filename,'rb')
688 break
699 break
689
700
690 self.flagIsNewFile = 1
701 self.flagIsNewFile = 1
691 self.fileIndex = idFile
702 self.fileIndex = idFile
692 self.filename = filename
703 self.filename = filename
693 self.fileSize = fileSize
704 self.fileSize = fileSize
694 self.fp = fp
705 self.fp = fp
695
706
696 # print "[Reading] Setting the file: %s"%self.filename
707 # print "[Reading] Setting the file: %s"%self.filename
697
708
698 return 1
709 return 1
699
710
700 def __setNextFileOnline(self):
711 def __setNextFileOnline(self):
701 """
712 """
702 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
713 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
703 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
714 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
704 siguientes.
715 siguientes.
705
716
706 Affected:
717 Affected:
707 self.flagIsNewFile
718 self.flagIsNewFile
708 self.filename
719 self.filename
709 self.fileSize
720 self.fileSize
710 self.fp
721 self.fp
711 self.set
722 self.set
712 self.flagNoMoreFiles
723 self.flagNoMoreFiles
713
724
714 Return:
725 Return:
715 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
726 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
716 1 : si el file fue abierto con exito y esta listo a ser leido
727 1 : si el file fue abierto con exito y esta listo a ser leido
717
728
718 Excepciones:
729 Excepciones:
719 Si un determinado file no puede ser abierto
730 Si un determinado file no puede ser abierto
720 """
731 """
721 nFiles = 0
732 nFiles = 0
722 fileOk_flag = False
733 fileOk_flag = False
723 firstTime_flag = True
734 firstTime_flag = True
724
735
725 self.set += 1
736 self.set += 1
726
737
727 if self.set > 999:
738 if self.set > 999:
728 self.set = 0
739 self.set = 0
729 self.foldercounter += 1
740 self.foldercounter += 1
730
741
731 #busca el 1er file disponible
742 #busca el 1er file disponible
732 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
743 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
733 if fullfilename:
744 if fullfilename:
734 if self.__verifyFile(fullfilename, False):
745 if self.__verifyFile(fullfilename, False):
735 fileOk_flag = True
746 fileOk_flag = True
736
747
737 #si no encuentra un file entonces espera y vuelve a buscar
748 #si no encuentra un file entonces espera y vuelve a buscar
738 if not(fileOk_flag):
749 if not(fileOk_flag):
739 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
750 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
740
751
741 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
752 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
742 tries = self.nTries
753 tries = self.nTries
743 else:
754 else:
744 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
755 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
745
756
746 for nTries in range( tries ):
757 for nTries in range( tries ):
747 if firstTime_flag:
758 if firstTime_flag:
748 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
759 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
749 sleep( self.delay )
760 sleep( self.delay )
750 else:
761 else:
751 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
762 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
752
763
753 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
764 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
754 if fullfilename:
765 if fullfilename:
755 if self.__verifyFile(fullfilename):
766 if self.__verifyFile(fullfilename):
756 fileOk_flag = True
767 fileOk_flag = True
757 break
768 break
758
769
759 if fileOk_flag:
770 if fileOk_flag:
760 break
771 break
761
772
762 firstTime_flag = False
773 firstTime_flag = False
763
774
764 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
775 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
765 self.set += 1
776 self.set += 1
766
777
767 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
778 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
768 self.set = 0
779 self.set = 0
769 self.doy += 1
780 self.doy += 1
770 self.foldercounter = 0
781 self.foldercounter = 0
771
782
772 if fileOk_flag:
783 if fileOk_flag:
773 self.fileSize = os.path.getsize( fullfilename )
784 self.fileSize = os.path.getsize( fullfilename )
774 self.filename = fullfilename
785 self.filename = fullfilename
775 self.flagIsNewFile = 1
786 self.flagIsNewFile = 1
776 if self.fp != None: self.fp.close()
787 if self.fp != None: self.fp.close()
777 self.fp = open(fullfilename, 'rb')
788 self.fp = open(fullfilename, 'rb')
778 self.flagNoMoreFiles = 0
789 self.flagNoMoreFiles = 0
779 # print '[Reading] Setting the file: %s' % fullfilename
790 # print '[Reading] Setting the file: %s' % fullfilename
780 else:
791 else:
781 self.fileSize = 0
792 self.fileSize = 0
782 self.filename = None
793 self.filename = None
783 self.flagIsNewFile = 0
794 self.flagIsNewFile = 0
784 self.fp = None
795 self.fp = None
785 self.flagNoMoreFiles = 1
796 self.flagNoMoreFiles = 1
786 # print '[Reading] No more files to read'
797 # print '[Reading] No more files to read'
787
798
788 return fileOk_flag
799 return fileOk_flag
789
800
790 def setNextFile(self):
801 def setNextFile(self):
791 if self.fp != None:
802 if self.fp != None:
792 self.fp.close()
803 self.fp.close()
793
804
794 if self.online:
805 if self.online:
795 newFile = self.__setNextFileOnline()
806 newFile = self.__setNextFileOnline()
796 else:
807 else:
797 newFile = self.__setNextFileOffline()
808 newFile = self.__setNextFileOffline()
798
809
799 if not(newFile):
810 if not(newFile):
800 print '[Reading] No more files to read'
811 print '[Reading] No more files to read'
801 return 0
812 return 0
802
813
803 print '[Reading] Setting the file: %s' % self.filename
814 print '[Reading] Setting the file: %s' % self.filename
804
815
805 self.__readFirstHeader()
816 self.__readFirstHeader()
806 self.nReadBlocks = 0
817 self.nReadBlocks = 0
807 return 1
818 return 1
808
819
809 def __waitNewBlock(self):
820 def __waitNewBlock(self):
810 """
821 """
811 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
822 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
812
823
813 Si el modo de lectura es OffLine siempre retorn 0
824 Si el modo de lectura es OffLine siempre retorn 0
814 """
825 """
815 if not self.online:
826 if not self.online:
816 return 0
827 return 0
817
828
818 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
829 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
819 return 0
830 return 0
820
831
821 currentPointer = self.fp.tell()
832 currentPointer = self.fp.tell()
822
833
823 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
834 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
824
835
825 for nTries in range( self.nTries ):
836 for nTries in range( self.nTries ):
826
837
827 self.fp.close()
838 self.fp.close()
828 self.fp = open( self.filename, 'rb' )
839 self.fp = open( self.filename, 'rb' )
829 self.fp.seek( currentPointer )
840 self.fp.seek( currentPointer )
830
841
831 self.fileSize = os.path.getsize( self.filename )
842 self.fileSize = os.path.getsize( self.filename )
832 currentSize = self.fileSize - currentPointer
843 currentSize = self.fileSize - currentPointer
833
844
834 if ( currentSize >= neededSize ):
845 if ( currentSize >= neededSize ):
835 self.basicHeaderObj.read(self.fp)
846 self.basicHeaderObj.read(self.fp)
836 return 1
847 return 1
837
848
838 if self.fileSize == self.fileSizeByHeader:
849 if self.fileSize == self.fileSizeByHeader:
839 # self.flagEoF = True
850 # self.flagEoF = True
840 return 0
851 return 0
841
852
842 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
853 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
843 sleep( self.delay )
854 sleep( self.delay )
844
855
845
856
846 return 0
857 return 0
847
858
848 def waitDataBlock(self,pointer_location):
859 def waitDataBlock(self,pointer_location):
849
860
850 currentPointer = pointer_location
861 currentPointer = pointer_location
851
862
852 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
863 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
853
864
854 for nTries in range( self.nTries ):
865 for nTries in range( self.nTries ):
855 self.fp.close()
866 self.fp.close()
856 self.fp = open( self.filename, 'rb' )
867 self.fp = open( self.filename, 'rb' )
857 self.fp.seek( currentPointer )
868 self.fp.seek( currentPointer )
858
869
859 self.fileSize = os.path.getsize( self.filename )
870 self.fileSize = os.path.getsize( self.filename )
860 currentSize = self.fileSize - currentPointer
871 currentSize = self.fileSize - currentPointer
861
872
862 if ( currentSize >= neededSize ):
873 if ( currentSize >= neededSize ):
863 return 1
874 return 1
864
875
865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
876 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
866 sleep( self.delay )
877 sleep( self.delay )
867
878
868 return 0
879 return 0
869
880
870 def __jumpToLastBlock(self):
881 def __jumpToLastBlock(self):
871
882
872 if not(self.__isFirstTimeOnline):
883 if not(self.__isFirstTimeOnline):
873 return
884 return
874
885
875 csize = self.fileSize - self.fp.tell()
886 csize = self.fileSize - self.fp.tell()
876 blocksize = self.processingHeaderObj.blockSize
887 blocksize = self.processingHeaderObj.blockSize
877
888
878 #salta el primer bloque de datos
889 #salta el primer bloque de datos
879 if csize > self.processingHeaderObj.blockSize:
890 if csize > self.processingHeaderObj.blockSize:
880 self.fp.seek(self.fp.tell() + blocksize)
891 self.fp.seek(self.fp.tell() + blocksize)
881 else:
892 else:
882 return
893 return
883
894
884 csize = self.fileSize - self.fp.tell()
895 csize = self.fileSize - self.fp.tell()
885 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
896 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
886 while True:
897 while True:
887
898
888 if self.fp.tell()<self.fileSize:
899 if self.fp.tell()<self.fileSize:
889 self.fp.seek(self.fp.tell() + neededsize)
900 self.fp.seek(self.fp.tell() + neededsize)
890 else:
901 else:
891 self.fp.seek(self.fp.tell() - neededsize)
902 self.fp.seek(self.fp.tell() - neededsize)
892 break
903 break
893
904
894 # csize = self.fileSize - self.fp.tell()
905 # csize = self.fileSize - self.fp.tell()
895 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
906 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
896 # factor = int(csize/neededsize)
907 # factor = int(csize/neededsize)
897 # if factor > 0:
908 # if factor > 0:
898 # self.fp.seek(self.fp.tell() + factor*neededsize)
909 # self.fp.seek(self.fp.tell() + factor*neededsize)
899
910
900 self.flagIsNewFile = 0
911 self.flagIsNewFile = 0
901 self.__isFirstTimeOnline = 0
912 self.__isFirstTimeOnline = 0
902
913
903 def __setNewBlock(self):
914 def __setNewBlock(self):
904
915
905 if self.fp == None:
916 if self.fp == None:
906 return 0
917 return 0
907
918
908 if self.online:
919 if self.online:
909 self.__jumpToLastBlock()
920 self.__jumpToLastBlock()
910
921
911 if self.flagIsNewFile:
922 if self.flagIsNewFile:
912 return 1
923 return 1
913
924
914 self.lastUTTime = self.basicHeaderObj.utc
925 self.lastUTTime = self.basicHeaderObj.utc
915 currentSize = self.fileSize - self.fp.tell()
926 currentSize = self.fileSize - self.fp.tell()
916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
927 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917
928
918 if (currentSize >= neededSize):
929 if (currentSize >= neededSize):
919 self.basicHeaderObj.read(self.fp)
930 self.basicHeaderObj.read(self.fp)
920 return 1
931 return 1
921
932
922 if self.__waitNewBlock():
933 if self.__waitNewBlock():
923 return 1
934 return 1
924
935
925 if not(self.setNextFile()):
936 if not(self.setNextFile()):
926 return 0
937 return 0
927
938
928 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
939 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
929
940
930 self.flagDiscontinuousBlock = 0
941 self.flagDiscontinuousBlock = 0
931
942
932 if deltaTime > self.maxTimeStep:
943 if deltaTime > self.maxTimeStep:
933 self.flagDiscontinuousBlock = 1
944 self.flagDiscontinuousBlock = 1
934
945
935 return 1
946 return 1
936
947
937 def readNextBlock(self):
948 def readNextBlock(self):
938
949
939 if not(self.__setNewBlock()):
950 if not(self.__setNewBlock()):
940 return 0
951 return 0
941
952
942 if not(self.readBlock()):
953 if not(self.readBlock()):
943 return 0
954 return 0
944
955
945 self.getBasicHeader()
956 self.getBasicHeader()
946
957
947 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
958 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
948 self.processingHeaderObj.dataBlocksPerFile,
959 self.processingHeaderObj.dataBlocksPerFile,
949 self.dataOut.datatime.ctime())
960 self.dataOut.datatime.ctime())
950 return 1
961 return 1
951
962
952 def __readFirstHeader(self):
963 def __readFirstHeader(self):
953
964
954 self.basicHeaderObj.read(self.fp)
965 self.basicHeaderObj.read(self.fp)
955 self.systemHeaderObj.read(self.fp)
966 self.systemHeaderObj.read(self.fp)
956 self.radarControllerHeaderObj.read(self.fp)
967 self.radarControllerHeaderObj.read(self.fp)
957 self.processingHeaderObj.read(self.fp)
968 self.processingHeaderObj.read(self.fp)
958
969
959 self.firstHeaderSize = self.basicHeaderObj.size
970 self.firstHeaderSize = self.basicHeaderObj.size
960
971
961 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
972 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
962 if datatype == 0:
973 if datatype == 0:
963 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
974 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
964 elif datatype == 1:
975 elif datatype == 1:
965 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
976 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
966 elif datatype == 2:
977 elif datatype == 2:
967 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
978 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
968 elif datatype == 3:
979 elif datatype == 3:
969 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
980 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
970 elif datatype == 4:
981 elif datatype == 4:
971 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
982 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
972 elif datatype == 5:
983 elif datatype == 5:
973 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
984 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
974 else:
985 else:
975 raise ValueError, 'Data type was not defined'
986 raise ValueError, 'Data type was not defined'
976
987
977 self.dtype = datatype_str
988 self.dtype = datatype_str
978 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
989 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
979 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
990 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
980 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
991 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
981 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
992 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
982 self.getBlockDimension()
993 self.getBlockDimension()
983
994
984 def __verifyFile(self, filename, msgFlag=True):
995 def __verifyFile(self, filename, msgFlag=True):
985 msg = None
996 msg = None
986 try:
997 try:
987 fp = open(filename, 'rb')
998 fp = open(filename, 'rb')
988 currentPosition = fp.tell()
999 currentPosition = fp.tell()
989 except IOError:
1000 except IOError:
990 traceback.print_exc()
1001 traceback.print_exc()
991 if msgFlag:
1002 if msgFlag:
992 print "[Reading] The file %s can't be opened" % (filename)
1003 print "[Reading] The file %s can't be opened" % (filename)
993 return False
1004 return False
994
1005
995 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1006 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
996
1007
997 if neededSize == 0:
1008 if neededSize == 0:
998 basicHeaderObj = BasicHeader(LOCALTIME)
1009 basicHeaderObj = BasicHeader(LOCALTIME)
999 systemHeaderObj = SystemHeader()
1010 systemHeaderObj = SystemHeader()
1000 radarControllerHeaderObj = RadarControllerHeader()
1011 radarControllerHeaderObj = RadarControllerHeader()
1001 processingHeaderObj = ProcessingHeader()
1012 processingHeaderObj = ProcessingHeader()
1002
1013
1003 try:
1014 try:
1004 if not( basicHeaderObj.read(fp) ): raise IOError
1015 if not( basicHeaderObj.read(fp) ): raise IOError
1005 if not( systemHeaderObj.read(fp) ): raise IOError
1016 if not( systemHeaderObj.read(fp) ): raise IOError
1006 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1017 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1007 if not( processingHeaderObj.read(fp) ): raise IOError
1018 if not( processingHeaderObj.read(fp) ): raise IOError
1008 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1019 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1009
1020
1010 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1021 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1011
1022
1012 except IOError:
1023 except IOError:
1013 traceback.print_exc()
1024 traceback.print_exc()
1014 sys.exit(0)
1025 sys.exit(0)
1015
1026
1016 if msgFlag:
1027 if msgFlag:
1017 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1028 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1018
1029
1019 fp.close()
1030 fp.close()
1020 return False
1031 return False
1021 else:
1032 else:
1022 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1033 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1023
1034
1024 fp.close()
1035 fp.close()
1025 fileSize = os.path.getsize(filename)
1036 fileSize = os.path.getsize(filename)
1026 currentSize = fileSize - currentPosition
1037 currentSize = fileSize - currentPosition
1027 if currentSize < neededSize:
1038 if currentSize < neededSize:
1028 if msgFlag and (msg != None):
1039 if msgFlag and (msg != None):
1029 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1040 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1030 return False
1041 return False
1031
1042
1032 return True
1043 return True
1033
1044
1034 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1045 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1035
1046
1036 dateList = []
1047 dateList = []
1037 pathList = []
1048 pathList = []
1038
1049
1039 multi_path = path.split(',')
1050 multi_path = path.split(',')
1040
1051
1041 if not walk:
1052 if not walk:
1042
1053
1043 for single_path in multi_path:
1054 for single_path in multi_path:
1044
1055
1045 if not os.path.isdir(single_path):
1056 if not os.path.isdir(single_path):
1046 continue
1057 continue
1047
1058
1048 fileList = glob.glob1(single_path, "*"+ext)
1059 fileList = glob.glob1(single_path, "*"+ext)
1049
1060
1050 for thisFile in fileList:
1061 for thisFile in fileList:
1051
1062
1052 if not os.path.isfile(os.path.join(single_path, thisFile)):
1063 if not os.path.isfile(os.path.join(single_path, thisFile)):
1053 continue
1064 continue
1054
1065
1055 if not isRadarFile(thisFile):
1066 if not isRadarFile(thisFile):
1056 continue
1067 continue
1057
1068
1058 if not isFileInDateRange(thisFile, startDate, endDate):
1069 if not isFileInDateRange(thisFile, startDate, endDate):
1059 continue
1070 continue
1060
1071
1061 thisDate = getDateFromRadarFile(thisFile)
1072 thisDate = getDateFromRadarFile(thisFile)
1062
1073
1063 if thisDate in dateList:
1074 if thisDate in dateList:
1064 continue
1075 continue
1065
1076
1066 dateList.append(thisDate)
1077 dateList.append(thisDate)
1067 pathList.append(single_path)
1078 pathList.append(single_path)
1068
1079
1069 else:
1080 else:
1070 for single_path in multi_path:
1081 for single_path in multi_path:
1071
1082
1072 if not os.path.isdir(single_path):
1083 if not os.path.isdir(single_path):
1073 continue
1084 continue
1074
1085
1075 dirList = []
1086 dirList = []
1076
1087
1077 for thisPath in os.listdir(single_path):
1088 for thisPath in os.listdir(single_path):
1078
1089
1079 if not os.path.isdir(os.path.join(single_path,thisPath)):
1090 if not os.path.isdir(os.path.join(single_path,thisPath)):
1080 continue
1091 continue
1081
1092
1082 if not isRadarFolder(thisPath):
1093 if not isRadarFolder(thisPath):
1083 continue
1094 continue
1084
1095
1085 if not isFolderInDateRange(thisPath, startDate, endDate):
1096 if not isFolderInDateRange(thisPath, startDate, endDate):
1086 continue
1097 continue
1087
1098
1088 dirList.append(thisPath)
1099 dirList.append(thisPath)
1089
1100
1090 if not dirList:
1101 if not dirList:
1091 continue
1102 continue
1092
1103
1093 for thisDir in dirList:
1104 for thisDir in dirList:
1094
1105
1095 datapath = os.path.join(single_path, thisDir, expLabel)
1106 datapath = os.path.join(single_path, thisDir, expLabel)
1096 fileList = glob.glob1(datapath, "*"+ext)
1107 fileList = glob.glob1(datapath, "*"+ext)
1097
1108
1098 if len(fileList) < 1:
1109 if len(fileList) < 1:
1099 continue
1110 continue
1100
1111
1101 thisDate = getDateFromRadarFolder(thisDir)
1112 thisDate = getDateFromRadarFolder(thisDir)
1102
1113
1103 pathList.append(datapath)
1114 pathList.append(datapath)
1104 dateList.append(thisDate)
1115 dateList.append(thisDate)
1105
1116
1106 dateList.sort()
1117 dateList.sort()
1107
1118
1108 if include_path:
1119 if include_path:
1109 return dateList, pathList
1120 return dateList, pathList
1110
1121
1111 return dateList
1122 return dateList
1112
1123
1113 def setup(self,
1124 def setup(self,
1114 path=None,
1125 path=None,
1115 startDate=None,
1126 startDate=None,
1116 endDate=None,
1127 endDate=None,
1117 startTime=datetime.time(0,0,0),
1128 startTime=datetime.time(0,0,0),
1118 endTime=datetime.time(23,59,59),
1129 endTime=datetime.time(23,59,59),
1119 set=None,
1130 set=None,
1120 expLabel = "",
1131 expLabel = "",
1121 ext = None,
1132 ext = None,
1122 online = False,
1133 online = False,
1123 delay = 60,
1134 delay = 60,
1124 walk = True,
1135 walk = True,
1125 getblock = False,
1136 getblock = False,
1126 nTxs = 1):
1137 nTxs = 1):
1127
1138
1128 if path == None:
1139 if path == None:
1129 raise ValueError, "[Reading] The path is not valid"
1140 raise ValueError, "[Reading] The path is not valid"
1130
1141
1131 if ext == None:
1142 if ext == None:
1132 ext = self.ext
1143 ext = self.ext
1133
1144
1134 if online:
1145 if online:
1135 print "[Reading] Searching files in online mode..."
1146 print "[Reading] Searching files in online mode..."
1136
1147
1137 for nTries in range( self.nTries ):
1148 for nTries in range( self.nTries ):
1138 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1149 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1139
1150
1140 if fullpath:
1151 if fullpath:
1141 break
1152 break
1142
1153
1143 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1154 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1144 sleep( self.delay )
1155 sleep( self.delay )
1145
1156
1146 if not(fullpath):
1157 if not(fullpath):
1147 print "[Reading] There 'isn't any valid file in %s" % path
1158 print "[Reading] There 'isn't any valid file in %s" % path
1148 return None
1159 return None
1149
1160
1150 self.year = year
1161 self.year = year
1151 self.doy = doy
1162 self.doy = doy
1152 self.set = set - 1
1163 self.set = set - 1
1153 self.path = path
1164 self.path = path
1154 self.foldercounter = foldercounter
1165 self.foldercounter = foldercounter
1155 last_set = None
1166 last_set = None
1156
1167
1157 else:
1168 else:
1158 print "[Reading] Searching files in offline mode ..."
1169 print "[Reading] Searching files in offline mode ..."
1159 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1170 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1160 startTime=startTime, endTime=endTime,
1171 startTime=startTime, endTime=endTime,
1161 set=set, expLabel=expLabel, ext=ext,
1172 set=set, expLabel=expLabel, ext=ext,
1162 walk=walk)
1173 walk=walk)
1163
1174
1164 if not(pathList):
1175 if not(pathList):
1165 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1176 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1166 # datetime.datetime.combine(startDate,startTime).ctime(),
1177 # datetime.datetime.combine(startDate,startTime).ctime(),
1167 # datetime.datetime.combine(endDate,endTime).ctime())
1178 # datetime.datetime.combine(endDate,endTime).ctime())
1168 #
1179 #
1169 sys.exit(-1)
1180 sys.exit(-1)
1170
1181
1171
1182
1172 self.fileIndex = -1
1183 self.fileIndex = -1
1173 self.pathList = pathList
1184 self.pathList = pathList
1174 self.filenameList = filenameList
1185 self.filenameList = filenameList
1175 file_name = os.path.basename(filenameList[-1])
1186 file_name = os.path.basename(filenameList[-1])
1176 basename, ext = os.path.splitext(file_name)
1187 basename, ext = os.path.splitext(file_name)
1177 last_set = int(basename[-3:])
1188 last_set = int(basename[-3:])
1178
1189
1179 self.online = online
1190 self.online = online
1180 self.delay = delay
1191 self.delay = delay
1181 ext = ext.lower()
1192 ext = ext.lower()
1182 self.ext = ext
1193 self.ext = ext
1183 self.getByBlock = getblock
1194 self.getByBlock = getblock
1184 self.nTxs = int(nTxs)
1195 self.nTxs = int(nTxs)
1185
1196
1186 if not(self.setNextFile()):
1197 if not(self.setNextFile()):
1187 if (startDate!=None) and (endDate!=None):
1198 if (startDate!=None) and (endDate!=None):
1188 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1199 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1189 elif startDate != None:
1200 elif startDate != None:
1190 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1201 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1191 else:
1202 else:
1192 print "[Reading] No files"
1203 print "[Reading] No files"
1193
1204
1194 sys.exit(-1)
1205 sys.exit(-1)
1195
1206
1196 # self.getBasicHeader()
1207 # self.getBasicHeader()
1197
1208
1198 if last_set != None:
1209 if last_set != None:
1199 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1210 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1200 return
1211 return
1201
1212
1202 def getBasicHeader(self):
1213 def getBasicHeader(self):
1203
1214
1204 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1215 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1205
1216
1206 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1217 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1207
1218
1208 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1219 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1209
1220
1210 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1221 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1211
1222
1212 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1223 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1213
1224
1214 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1225 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1215
1226
1216 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1227 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1217
1228
1218 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1229 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1219
1230
1220
1231
1221 def getFirstHeader(self):
1232 def getFirstHeader(self):
1222
1233
1223 raise ValueError, "This method has not been implemented"
1234 raise ValueError, "This method has not been implemented"
1224
1235
1225 def getData(self):
1236 def getData(self):
1226
1237
1227 raise ValueError, "This method has not been implemented"
1238 raise ValueError, "This method has not been implemented"
1228
1239
1229 def hasNotDataInBuffer(self):
1240 def hasNotDataInBuffer(self):
1230
1241
1231 raise ValueError, "This method has not been implemented"
1242 raise ValueError, "This method has not been implemented"
1232
1243
1233 def readBlock(self):
1244 def readBlock(self):
1234
1245
1235 raise ValueError, "This method has not been implemented"
1246 raise ValueError, "This method has not been implemented"
1236
1247
1237 def isEndProcess(self):
1248 def isEndProcess(self):
1238
1249
1239 return self.flagNoMoreFiles
1250 return self.flagNoMoreFiles
1240
1251
1241 def printReadBlocks(self):
1252 def printReadBlocks(self):
1242
1253
1243 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1254 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1244
1255
1245 def printTotalBlocks(self):
1256 def printTotalBlocks(self):
1246
1257
1247 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1258 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1248
1259
1249 def printNumberOfBlock(self):
1260 def printNumberOfBlock(self):
1250
1261
1251 if self.flagIsNewBlock:
1262 if self.flagIsNewBlock:
1252 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1263 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1253 self.processingHeaderObj.dataBlocksPerFile,
1264 self.processingHeaderObj.dataBlocksPerFile,
1254 self.dataOut.datatime.ctime())
1265 self.dataOut.datatime.ctime())
1255
1266
1256 def printInfo(self):
1267 def printInfo(self):
1257
1268
1258 if self.__printInfo == False:
1269 if self.__printInfo == False:
1259 return
1270 return
1260
1271
1261 self.basicHeaderObj.printInfo()
1272 self.basicHeaderObj.printInfo()
1262 self.systemHeaderObj.printInfo()
1273 self.systemHeaderObj.printInfo()
1263 self.radarControllerHeaderObj.printInfo()
1274 self.radarControllerHeaderObj.printInfo()
1264 self.processingHeaderObj.printInfo()
1275 self.processingHeaderObj.printInfo()
1265
1276
1266 self.__printInfo = False
1277 self.__printInfo = False
1267
1278
1268
1279
1269 def run(self, **kwargs):
1280 def run(self, **kwargs):
1270
1281
1271 if not(self.isConfig):
1282 if not(self.isConfig):
1272
1283
1273 # self.dataOut = dataOut
1284 # self.dataOut = dataOut
1274 self.setup(**kwargs)
1285 self.setup(**kwargs)
1275 self.isConfig = True
1286 self.isConfig = True
1276
1287
1277 self.getData()
1288 self.getData()
1278
1289
1279 class JRODataWriter(JRODataIO):
1290 class JRODataWriter(JRODataIO):
1280
1291
1281 """
1292 """
1282 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1293 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1283 de los datos siempre se realiza por bloques.
1294 de los datos siempre se realiza por bloques.
1284 """
1295 """
1285
1296
1286 blockIndex = 0
1297 blockIndex = 0
1287
1298
1288 path = None
1299 path = None
1289
1300
1290 setFile = None
1301 setFile = None
1291
1302
1292 profilesPerBlock = None
1303 profilesPerBlock = None
1293
1304
1294 blocksPerFile = None
1305 blocksPerFile = None
1295
1306
1296 nWriteBlocks = 0
1307 nWriteBlocks = 0
1297
1308
1298 fileDate = None
1309 fileDate = None
1299
1310
1300 def __init__(self, dataOut=None):
1311 def __init__(self, dataOut=None):
1301 raise ValueError, "Not implemented"
1312 raise ValueError, "Not implemented"
1302
1313
1303
1314
1304 def hasAllDataInBuffer(self):
1315 def hasAllDataInBuffer(self):
1305 raise ValueError, "Not implemented"
1316 raise ValueError, "Not implemented"
1306
1317
1307
1318
1308 def setBlockDimension(self):
1319 def setBlockDimension(self):
1309 raise ValueError, "Not implemented"
1320 raise ValueError, "Not implemented"
1310
1321
1311
1322
1312 def writeBlock(self):
1323 def writeBlock(self):
1313 raise ValueError, "No implemented"
1324 raise ValueError, "No implemented"
1314
1325
1315
1326
1316 def putData(self):
1327 def putData(self):
1317 raise ValueError, "No implemented"
1328 raise ValueError, "No implemented"
1318
1329
1319
1330
1320 def getProcessFlags(self):
1331 def getProcessFlags(self):
1321
1332
1322 processFlags = 0
1333 processFlags = 0
1323
1334
1324 dtype_index = get_dtype_index(self.dtype)
1335 dtype_index = get_dtype_index(self.dtype)
1325 procflag_dtype = get_procflag_dtype(dtype_index)
1336 procflag_dtype = get_procflag_dtype(dtype_index)
1326
1337
1327 processFlags += procflag_dtype
1338 processFlags += procflag_dtype
1328
1339
1329 if self.dataOut.flagDecodeData:
1340 if self.dataOut.flagDecodeData:
1330 processFlags += PROCFLAG.DECODE_DATA
1341 processFlags += PROCFLAG.DECODE_DATA
1331
1342
1332 if self.dataOut.flagDeflipData:
1343 if self.dataOut.flagDeflipData:
1333 processFlags += PROCFLAG.DEFLIP_DATA
1344 processFlags += PROCFLAG.DEFLIP_DATA
1334
1345
1335 if self.dataOut.code is not None:
1346 if self.dataOut.code is not None:
1336 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1347 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1337
1348
1338 if self.dataOut.nCohInt > 1:
1349 if self.dataOut.nCohInt > 1:
1339 processFlags += PROCFLAG.COHERENT_INTEGRATION
1350 processFlags += PROCFLAG.COHERENT_INTEGRATION
1340
1351
1341 if self.dataOut.type == "Spectra":
1352 if self.dataOut.type == "Spectra":
1342 if self.dataOut.nIncohInt > 1:
1353 if self.dataOut.nIncohInt > 1:
1343 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1354 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1344
1355
1345 if self.dataOut.data_dc is not None:
1356 if self.dataOut.data_dc is not None:
1346 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1357 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1347
1358
1348 if self.dataOut.flagShiftFFT:
1359 if self.dataOut.flagShiftFFT:
1349 processFlags += PROCFLAG.SHIFT_FFT_DATA
1360 processFlags += PROCFLAG.SHIFT_FFT_DATA
1350
1361
1351 return processFlags
1362 return processFlags
1352
1363
1353 def setBasicHeader(self):
1364 def setBasicHeader(self):
1354
1365
1355 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1366 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1356 self.basicHeaderObj.version = self.versionFile
1367 self.basicHeaderObj.version = self.versionFile
1357 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1368 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1358
1369
1359 utc = numpy.floor(self.dataOut.utctime)
1370 utc = numpy.floor(self.dataOut.utctime)
1360 milisecond = (self.dataOut.utctime - utc)* 1000.0
1371 milisecond = (self.dataOut.utctime - utc)* 1000.0
1361
1372
1362 self.basicHeaderObj.utc = utc
1373 self.basicHeaderObj.utc = utc
1363 self.basicHeaderObj.miliSecond = milisecond
1374 self.basicHeaderObj.miliSecond = milisecond
1364 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1375 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1365 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1376 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1366 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1377 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1367
1378
1368 def setFirstHeader(self):
1379 def setFirstHeader(self):
1369 """
1380 """
1370 Obtiene una copia del First Header
1381 Obtiene una copia del First Header
1371
1382
1372 Affected:
1383 Affected:
1373
1384
1374 self.basicHeaderObj
1385 self.basicHeaderObj
1375 self.systemHeaderObj
1386 self.systemHeaderObj
1376 self.radarControllerHeaderObj
1387 self.radarControllerHeaderObj
1377 self.processingHeaderObj self.
1388 self.processingHeaderObj self.
1378
1389
1379 Return:
1390 Return:
1380 None
1391 None
1381 """
1392 """
1382
1393
1383 raise ValueError, "No implemented"
1394 raise ValueError, "No implemented"
1384
1395
1385 def __writeFirstHeader(self):
1396 def __writeFirstHeader(self):
1386 """
1397 """
1387 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1398 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1388
1399
1389 Affected:
1400 Affected:
1390 __dataType
1401 __dataType
1391
1402
1392 Return:
1403 Return:
1393 None
1404 None
1394 """
1405 """
1395
1406
1396 # CALCULAR PARAMETROS
1407 # CALCULAR PARAMETROS
1397
1408
1398 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1409 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1399 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1410 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1400
1411
1401 self.basicHeaderObj.write(self.fp)
1412 self.basicHeaderObj.write(self.fp)
1402 self.systemHeaderObj.write(self.fp)
1413 self.systemHeaderObj.write(self.fp)
1403 self.radarControllerHeaderObj.write(self.fp)
1414 self.radarControllerHeaderObj.write(self.fp)
1404 self.processingHeaderObj.write(self.fp)
1415 self.processingHeaderObj.write(self.fp)
1405
1416
1406 def __setNewBlock(self):
1417 def __setNewBlock(self):
1407 """
1418 """
1408 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1419 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1409
1420
1410 Return:
1421 Return:
1411 0 : si no pudo escribir nada
1422 0 : si no pudo escribir nada
1412 1 : Si escribio el Basic el First Header
1423 1 : Si escribio el Basic el First Header
1413 """
1424 """
1414 if self.fp == None:
1425 if self.fp == None:
1415 self.setNextFile()
1426 self.setNextFile()
1416
1427
1417 if self.flagIsNewFile:
1428 if self.flagIsNewFile:
1418 return 1
1429 return 1
1419
1430
1420 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1431 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1421 self.basicHeaderObj.write(self.fp)
1432 self.basicHeaderObj.write(self.fp)
1422 return 1
1433 return 1
1423
1434
1424 if not( self.setNextFile() ):
1435 if not( self.setNextFile() ):
1425 return 0
1436 return 0
1426
1437
1427 return 1
1438 return 1
1428
1439
1429
1440
1430 def writeNextBlock(self):
1441 def writeNextBlock(self):
1431 """
1442 """
1432 Selecciona el bloque siguiente de datos y los escribe en un file
1443 Selecciona el bloque siguiente de datos y los escribe en un file
1433
1444
1434 Return:
1445 Return:
1435 0 : Si no hizo pudo escribir el bloque de datos
1446 0 : Si no hizo pudo escribir el bloque de datos
1436 1 : Si no pudo escribir el bloque de datos
1447 1 : Si no pudo escribir el bloque de datos
1437 """
1448 """
1438 if not( self.__setNewBlock() ):
1449 if not( self.__setNewBlock() ):
1439 return 0
1450 return 0
1440
1451
1441 self.writeBlock()
1452 self.writeBlock()
1442
1453
1443 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1454 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1444 self.processingHeaderObj.dataBlocksPerFile)
1455 self.processingHeaderObj.dataBlocksPerFile)
1445
1456
1446 return 1
1457 return 1
1447
1458
1448 def setNextFile(self):
1459 def setNextFile(self):
1449 """
1460 """
1450 Determina el siguiente file que sera escrito
1461 Determina el siguiente file que sera escrito
1451
1462
1452 Affected:
1463 Affected:
1453 self.filename
1464 self.filename
1454 self.subfolder
1465 self.subfolder
1455 self.fp
1466 self.fp
1456 self.setFile
1467 self.setFile
1457 self.flagIsNewFile
1468 self.flagIsNewFile
1458
1469
1459 Return:
1470 Return:
1460 0 : Si el archivo no puede ser escrito
1471 0 : Si el archivo no puede ser escrito
1461 1 : Si el archivo esta listo para ser escrito
1472 1 : Si el archivo esta listo para ser escrito
1462 """
1473 """
1463 ext = self.ext
1474 ext = self.ext
1464 path = self.path
1475 path = self.path
1465
1476
1466 if self.fp != None:
1477 if self.fp != None:
1467 self.fp.close()
1478 self.fp.close()
1468
1479
1469 timeTuple = time.localtime( self.dataOut.utctime)
1480 timeTuple = time.localtime( self.dataOut.utctime)
1470 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1481 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1471
1482
1472 fullpath = os.path.join( path, subfolder )
1483 fullpath = os.path.join( path, subfolder )
1473 setFile = self.setFile
1484 setFile = self.setFile
1474
1485
1475 if not( os.path.exists(fullpath) ):
1486 if not( os.path.exists(fullpath) ):
1476 os.mkdir(fullpath)
1487 os.mkdir(fullpath)
1477 setFile = -1 #inicializo mi contador de seteo
1488 setFile = -1 #inicializo mi contador de seteo
1478 else:
1489 else:
1479 filesList = os.listdir( fullpath )
1490 filesList = os.listdir( fullpath )
1480 if len( filesList ) > 0:
1491 if len( filesList ) > 0:
1481 filesList = sorted( filesList, key=str.lower )
1492 filesList = sorted( filesList, key=str.lower )
1482 filen = filesList[-1]
1493 filen = filesList[-1]
1483 # el filename debera tener el siguiente formato
1494 # el filename debera tener el siguiente formato
1484 # 0 1234 567 89A BCDE (hex)
1495 # 0 1234 567 89A BCDE (hex)
1485 # x YYYY DDD SSS .ext
1496 # x YYYY DDD SSS .ext
1486 if isNumber( filen[8:11] ):
1497 if isNumber( filen[8:11] ):
1487 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1498 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1488 else:
1499 else:
1489 setFile = -1
1500 setFile = -1
1490 else:
1501 else:
1491 setFile = -1 #inicializo mi contador de seteo
1502 setFile = -1 #inicializo mi contador de seteo
1492
1503
1493 setFile += 1
1504 setFile += 1
1494
1505
1495 #If this is a new day it resets some values
1506 #If this is a new day it resets some values
1496 if self.dataOut.datatime.date() > self.fileDate:
1507 if self.dataOut.datatime.date() > self.fileDate:
1497 setFile = 0
1508 setFile = 0
1498 self.nTotalBlocks = 0
1509 self.nTotalBlocks = 0
1499
1510
1500 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1511 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1501
1512
1502 filename = os.path.join( path, subfolder, filen )
1513 filename = os.path.join( path, subfolder, filen )
1503
1514
1504 fp = open( filename,'wb' )
1515 fp = open( filename,'wb' )
1505
1516
1506 self.blockIndex = 0
1517 self.blockIndex = 0
1507
1518
1508 #guardando atributos
1519 #guardando atributos
1509 self.filename = filename
1520 self.filename = filename
1510 self.subfolder = subfolder
1521 self.subfolder = subfolder
1511 self.fp = fp
1522 self.fp = fp
1512 self.setFile = setFile
1523 self.setFile = setFile
1513 self.flagIsNewFile = 1
1524 self.flagIsNewFile = 1
1514 self.fileDate = self.dataOut.datatime.date()
1525 self.fileDate = self.dataOut.datatime.date()
1515
1526
1516 self.setFirstHeader()
1527 self.setFirstHeader()
1517
1528
1518 print '[Writing] Opening file: %s'%self.filename
1529 print '[Writing] Opening file: %s'%self.filename
1519
1530
1520 self.__writeFirstHeader()
1531 self.__writeFirstHeader()
1521
1532
1522 return 1
1533 return 1
1523
1534
1524 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1535 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1525 """
1536 """
1526 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1537 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1527
1538
1528 Inputs:
1539 Inputs:
1529 path : directory where data will be saved
1540 path : directory where data will be saved
1530 profilesPerBlock : number of profiles per block
1541 profilesPerBlock : number of profiles per block
1531 set : initial file set
1542 set : initial file set
1532 datatype : An integer number that defines data type:
1543 datatype : An integer number that defines data type:
1533 0 : int8 (1 byte)
1544 0 : int8 (1 byte)
1534 1 : int16 (2 bytes)
1545 1 : int16 (2 bytes)
1535 2 : int32 (4 bytes)
1546 2 : int32 (4 bytes)
1536 3 : int64 (8 bytes)
1547 3 : int64 (8 bytes)
1537 4 : float32 (4 bytes)
1548 4 : float32 (4 bytes)
1538 5 : double64 (8 bytes)
1549 5 : double64 (8 bytes)
1539
1550
1540 Return:
1551 Return:
1541 0 : Si no realizo un buen seteo
1552 0 : Si no realizo un buen seteo
1542 1 : Si realizo un buen seteo
1553 1 : Si realizo un buen seteo
1543 """
1554 """
1544
1555
1545 if ext == None:
1556 if ext == None:
1546 ext = self.ext
1557 ext = self.ext
1547
1558
1548 self.ext = ext.lower()
1559 self.ext = ext.lower()
1549
1560
1550 self.path = path
1561 self.path = path
1551
1562
1552 if set is None:
1563 if set is None:
1553 self.setFile = -1
1564 self.setFile = -1
1554 else:
1565 else:
1555 self.setFile = set - 1
1566 self.setFile = set - 1
1556
1567
1557 self.blocksPerFile = blocksPerFile
1568 self.blocksPerFile = blocksPerFile
1558
1569
1559 self.profilesPerBlock = profilesPerBlock
1570 self.profilesPerBlock = profilesPerBlock
1560
1571
1561 self.dataOut = dataOut
1572 self.dataOut = dataOut
1562 self.fileDate = self.dataOut.datatime.date()
1573 self.fileDate = self.dataOut.datatime.date()
1563 #By default
1574 #By default
1564 self.dtype = self.dataOut.dtype
1575 self.dtype = self.dataOut.dtype
1565
1576
1566 if datatype is not None:
1577 if datatype is not None:
1567 self.dtype = get_numpy_dtype(datatype)
1578 self.dtype = get_numpy_dtype(datatype)
1568
1579
1569 if not(self.setNextFile()):
1580 if not(self.setNextFile()):
1570 print "[Writing] There isn't a next file"
1581 print "[Writing] There isn't a next file"
1571 return 0
1582 return 0
1572
1583
1573 self.setBlockDimension()
1584 self.setBlockDimension()
1574
1585
1575 return 1
1586 return 1
1576
1587
1577 def run(self, dataOut, **kwargs):
1588 def run(self, dataOut, **kwargs):
1578
1589
1579 if not(self.isConfig):
1590 if not(self.isConfig):
1580
1591
1581 self.setup(dataOut, **kwargs)
1592 self.setup(dataOut, **kwargs)
1582 self.isConfig = True
1593 self.isConfig = True
1583
1594
1584 self.putData()
1595 self.putData()
1585
1596
General Comments 0
You need to be logged in to leave comments. Login now