##// END OF EJS Templates
schain-jc
RSS
isFileinTimeRange: startTime can be greater than endTime
Miguel Valdez -
r648:aa41cfeddebd
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1629 +1,1585
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 traceback.print_exc()
71 traceback.print_exc()
72 raise IOError, "The file %s can't be opened" %(filename)
72 raise IOError, "The file %s can't be opened" %(filename)
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isFileInTimeRange(filename, startTime, endTime):
86 def isFileInTimeRange(filename, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
90 Inputs:
90 Inputs:
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92
92
93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94
94
95 endTime : tiempo final del rango seleccionado en formato datetime.time
95 endTime : tiempo final del rango seleccionado en formato datetime.time
96
96
97 Return:
97 Return:
98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 fecha especificado, de lo contrario retorna False.
99 fecha especificado, de lo contrario retorna False.
100
100
101 Excepciones:
101 Excepciones:
102 Si el archivo no existe o no puede ser abierto
102 Si el archivo no existe o no puede ser abierto
103 Si la cabecera no puede ser leida.
103 Si la cabecera no puede ser leida.
104
104
105 """
105 """
106
106
107
107
108 try:
108 try:
109 fp = open(filename,'rb')
109 fp = open(filename,'rb')
110 except IOError:
110 except IOError:
111 traceback.print_exc()
111 traceback.print_exc()
112 raise IOError, "The file %s can't be opened" %(filename)
112 raise IOError, "The file %s can't be opened" %(filename)
113
113
114 basicHeaderObj = BasicHeader(LOCALTIME)
114 basicHeaderObj = BasicHeader(LOCALTIME)
115 sts = basicHeaderObj.read(fp)
115 sts = basicHeaderObj.read(fp)
116 fp.close()
116 fp.close()
117
117
118 thisDatetime = basicHeaderObj.datatime
118 thisDatetime = basicHeaderObj.datatime
119 thisTime = thisDatetime.time()
119 thisTime = thisDatetime.time()
120
120
121 if not(sts):
121 if not(sts):
122 print "Skipping the file %s because it has not a valid header" %(filename)
122 print "Skipping the file %s because it has not a valid header" %(filename)
123 return None
123 return None
124
124
125 if not ((startTime <= thisTime) and (endTime > thisTime)):
125 #General case
126 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
127 #-----------o----------------------------o-----------
128 # startTime endTime
129
130 if endTime >= startTime:
131 if (thisTime < startTime) or (thisTime > endTime):
132 return None
133
134 return thisDatetime
135
136 #If endTime < startTime then endTime belongs to the next day
137
138
139 #<<<<<<<<<<<o o>>>>>>>>>>>
140 #-----------o----------------------------o-----------
141 # endTime startTime
142
143 if (thisTime < startTime) and (thisTime > endTime):
126 return None
144 return None
127
145
128 return thisDatetime
146 return thisDatetime
129
147
130 def isFolderInDateRange(folder, startDate=None, endDate=None):
148 def isFolderInDateRange(folder, startDate=None, endDate=None):
131 """
149 """
132 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
150 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
133
151
134 Inputs:
152 Inputs:
135 folder : nombre completo del directorio.
153 folder : nombre completo del directorio.
136 Su formato deberia ser "/path_root/?YYYYDDD"
154 Su formato deberia ser "/path_root/?YYYYDDD"
137
155
138 siendo:
156 siendo:
139 YYYY : Anio (ejemplo 2015)
157 YYYY : Anio (ejemplo 2015)
140 DDD : Dia del anio (ejemplo 305)
158 DDD : Dia del anio (ejemplo 305)
141
159
142 startDate : fecha inicial del rango seleccionado en formato datetime.date
160 startDate : fecha inicial del rango seleccionado en formato datetime.date
143
161
144 endDate : fecha final del rango seleccionado en formato datetime.date
162 endDate : fecha final del rango seleccionado en formato datetime.date
145
163
146 Return:
164 Return:
147 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
165 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
148 fecha especificado, de lo contrario retorna False.
166 fecha especificado, de lo contrario retorna False.
149 Excepciones:
167 Excepciones:
150 Si el directorio no tiene el formato adecuado
168 Si el directorio no tiene el formato adecuado
151 """
169 """
152
170
153 basename = os.path.basename(folder)
171 basename = os.path.basename(folder)
154
172
155 if not isRadarFolder(basename):
173 if not isRadarFolder(basename):
156 raise IOError, "The folder %s has not the rigth format" %folder
174 raise IOError, "The folder %s has not the rigth format" %folder
157
175
158 if startDate and endDate:
176 if startDate and endDate:
159 thisDate = getDateFromRadarFolder(basename)
177 thisDate = getDateFromRadarFolder(basename)
160
178
161 if thisDate < startDate:
179 if thisDate < startDate:
162 return 0
180 return 0
163
181
164 if thisDate > endDate:
182 if thisDate > endDate:
165 return 0
183 return 0
166
184
167 return 1
185 return 1
168
186
169 def isFileInDateRange(filename, startDate=None, endDate=None):
187 def isFileInDateRange(filename, startDate=None, endDate=None):
170 """
188 """
171 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
189 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
172
190
173 Inputs:
191 Inputs:
174 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
192 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
175
193
176 Su formato deberia ser "?YYYYDDDsss"
194 Su formato deberia ser "?YYYYDDDsss"
177
195
178 siendo:
196 siendo:
179 YYYY : Anio (ejemplo 2015)
197 YYYY : Anio (ejemplo 2015)
180 DDD : Dia del anio (ejemplo 305)
198 DDD : Dia del anio (ejemplo 305)
181 sss : set
199 sss : set
182
200
183 startDate : fecha inicial del rango seleccionado en formato datetime.date
201 startDate : fecha inicial del rango seleccionado en formato datetime.date
184
202
185 endDate : fecha final del rango seleccionado en formato datetime.date
203 endDate : fecha final del rango seleccionado en formato datetime.date
186
204
187 Return:
205 Return:
188 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
189 fecha especificado, de lo contrario retorna False.
207 fecha especificado, de lo contrario retorna False.
190 Excepciones:
208 Excepciones:
191 Si el archivo no tiene el formato adecuado
209 Si el archivo no tiene el formato adecuado
192 """
210 """
193
211
194 basename = os.path.basename(filename)
212 basename = os.path.basename(filename)
195
213
196 if not isRadarFile(basename):
214 if not isRadarFile(basename):
197 raise IOError, "The filename %s has not the rigth format" %filename
215 raise IOError, "The filename %s has not the rigth format" %filename
198
216
199 if startDate and endDate:
217 if startDate and endDate:
200 thisDate = getDateFromRadarFile(basename)
218 thisDate = getDateFromRadarFile(basename)
201
219
202 if thisDate < startDate:
220 if thisDate < startDate:
203 return 0
221 return 0
204
222
205 if thisDate > endDate:
223 if thisDate > endDate:
206 return 0
224 return 0
207
225
208 return 1
226 return 1
209
227
210 def getFileFromSet(path, ext, set):
228 def getFileFromSet(path, ext, set):
211 validFilelist = []
229 validFilelist = []
212 fileList = os.listdir(path)
230 fileList = os.listdir(path)
213
231
214 # 0 1234 567 89A BCDE
232 # 0 1234 567 89A BCDE
215 # H YYYY DDD SSS .ext
233 # H YYYY DDD SSS .ext
216
234
217 for thisFile in fileList:
235 for thisFile in fileList:
218 try:
236 try:
219 year = int(thisFile[1:5])
237 year = int(thisFile[1:5])
220 doy = int(thisFile[5:8])
238 doy = int(thisFile[5:8])
221 except:
239 except:
222 continue
240 continue
223
241
224 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
242 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
225 continue
243 continue
226
244
227 validFilelist.append(thisFile)
245 validFilelist.append(thisFile)
228
246
229 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
247 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
230
248
231 if len(myfile)!= 0:
249 if len(myfile)!= 0:
232 return myfile[0]
250 return myfile[0]
233 else:
251 else:
234 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
252 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
235 print 'the filename %s does not exist'%filename
253 print 'the filename %s does not exist'%filename
236 print '...going to the last file: '
254 print '...going to the last file: '
237
255
238 if validFilelist:
256 if validFilelist:
239 validFilelist = sorted( validFilelist, key=str.lower )
257 validFilelist = sorted( validFilelist, key=str.lower )
240 return validFilelist[-1]
258 return validFilelist[-1]
241
259
242 return None
260 return None
243
261
244 def getlastFileFromPath(path, ext):
262 def getlastFileFromPath(path, ext):
245 """
263 """
246 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
264 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
247 al final de la depuracion devuelve el ultimo file de la lista que quedo.
265 al final de la depuracion devuelve el ultimo file de la lista que quedo.
248
266
249 Input:
267 Input:
250 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
268 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
251 ext : extension de los files contenidos en una carpeta
269 ext : extension de los files contenidos en una carpeta
252
270
253 Return:
271 Return:
254 El ultimo file de una determinada carpeta, no se considera el path.
272 El ultimo file de una determinada carpeta, no se considera el path.
255 """
273 """
256 validFilelist = []
274 validFilelist = []
257 fileList = os.listdir(path)
275 fileList = os.listdir(path)
258
276
259 # 0 1234 567 89A BCDE
277 # 0 1234 567 89A BCDE
260 # H YYYY DDD SSS .ext
278 # H YYYY DDD SSS .ext
261
279
262 for thisFile in fileList:
280 for thisFile in fileList:
263
281
264 year = thisFile[1:5]
282 year = thisFile[1:5]
265 if not isNumber(year):
283 if not isNumber(year):
266 continue
284 continue
267
285
268 doy = thisFile[5:8]
286 doy = thisFile[5:8]
269 if not isNumber(doy):
287 if not isNumber(doy):
270 continue
288 continue
271
289
272 year = int(year)
290 year = int(year)
273 doy = int(doy)
291 doy = int(doy)
274
292
275 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
293 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
276 continue
294 continue
277
295
278 validFilelist.append(thisFile)
296 validFilelist.append(thisFile)
279
297
280 if validFilelist:
298 if validFilelist:
281 validFilelist = sorted( validFilelist, key=str.lower )
299 validFilelist = sorted( validFilelist, key=str.lower )
282 return validFilelist[-1]
300 return validFilelist[-1]
283
301
284 return None
302 return None
285
303
286 def checkForRealPath(path, foldercounter, year, doy, set, ext):
304 def checkForRealPath(path, foldercounter, year, doy, set, ext):
287 """
305 """
288 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
306 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
289 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
307 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
290 el path exacto de un determinado file.
308 el path exacto de un determinado file.
291
309
292 Example :
310 Example :
293 nombre correcto del file es .../.../D2009307/P2009307367.ext
311 nombre correcto del file es .../.../D2009307/P2009307367.ext
294
312
295 Entonces la funcion prueba con las siguientes combinaciones
313 Entonces la funcion prueba con las siguientes combinaciones
296 .../.../y2009307367.ext
314 .../.../y2009307367.ext
297 .../.../Y2009307367.ext
315 .../.../Y2009307367.ext
298 .../.../x2009307/y2009307367.ext
316 .../.../x2009307/y2009307367.ext
299 .../.../x2009307/Y2009307367.ext
317 .../.../x2009307/Y2009307367.ext
300 .../.../X2009307/y2009307367.ext
318 .../.../X2009307/y2009307367.ext
301 .../.../X2009307/Y2009307367.ext
319 .../.../X2009307/Y2009307367.ext
302 siendo para este caso, la ultima combinacion de letras, identica al file buscado
320 siendo para este caso, la ultima combinacion de letras, identica al file buscado
303
321
304 Return:
322 Return:
305 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
323 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
306 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
324 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
307 para el filename
325 para el filename
308 """
326 """
309 fullfilename = None
327 fullfilename = None
310 find_flag = False
328 find_flag = False
311 filename = None
329 filename = None
312
330
313 prefixDirList = [None,'d','D']
331 prefixDirList = [None,'d','D']
314 if ext.lower() == ".r": #voltage
332 if ext.lower() == ".r": #voltage
315 prefixFileList = ['d','D']
333 prefixFileList = ['d','D']
316 elif ext.lower() == ".pdata": #spectra
334 elif ext.lower() == ".pdata": #spectra
317 prefixFileList = ['p','P']
335 prefixFileList = ['p','P']
318 else:
336 else:
319 return None, filename
337 return None, filename
320
338
321 #barrido por las combinaciones posibles
339 #barrido por las combinaciones posibles
322 for prefixDir in prefixDirList:
340 for prefixDir in prefixDirList:
323 thispath = path
341 thispath = path
324 if prefixDir != None:
342 if prefixDir != None:
325 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
343 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
326 if foldercounter == 0:
344 if foldercounter == 0:
327 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
345 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
328 else:
346 else:
329 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
347 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
330 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
348 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
331 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
349 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
332 fullfilename = os.path.join( thispath, filename ) #formo el path completo
350 fullfilename = os.path.join( thispath, filename ) #formo el path completo
333
351
334 if os.path.exists( fullfilename ): #verifico que exista
352 if os.path.exists( fullfilename ): #verifico que exista
335 find_flag = True
353 find_flag = True
336 break
354 break
337 if find_flag:
355 if find_flag:
338 break
356 break
339
357
340 if not(find_flag):
358 if not(find_flag):
341 return None, filename
359 return None, filename
342
360
343 return fullfilename, filename
361 return fullfilename, filename
344
362
345 def isRadarFolder(folder):
363 def isRadarFolder(folder):
346 try:
364 try:
347 year = int(folder[1:5])
365 year = int(folder[1:5])
348 doy = int(folder[5:8])
366 doy = int(folder[5:8])
349 except:
367 except:
350 return 0
368 return 0
351
369
352 return 1
370 return 1
353
371
354 def isRadarFile(file):
372 def isRadarFile(file):
355 try:
373 try:
356 year = int(file[1:5])
374 year = int(file[1:5])
357 doy = int(file[5:8])
375 doy = int(file[5:8])
358 set = int(file[8:11])
376 set = int(file[8:11])
359 except:
377 except:
360 return 0
378 return 0
361
379
362 return 1
380 return 1
363
381
364 def getDateFromRadarFile(file):
382 def getDateFromRadarFile(file):
365 try:
383 try:
366 year = int(file[1:5])
384 year = int(file[1:5])
367 doy = int(file[5:8])
385 doy = int(file[5:8])
368 set = int(file[8:11])
386 set = int(file[8:11])
369 except:
387 except:
370 return None
388 return None
371
389
372 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
390 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
373 return thisDate
391 return thisDate
374
392
375 def getDateFromRadarFolder(folder):
393 def getDateFromRadarFolder(folder):
376 try:
394 try:
377 year = int(folder[1:5])
395 year = int(folder[1:5])
378 doy = int(folder[5:8])
396 doy = int(folder[5:8])
379 except:
397 except:
380 return None
398 return None
381
399
382 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
400 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
383 return thisDate
401 return thisDate
384
402
385 class JRODataIO:
403 class JRODataIO:
386
404
387 c = 3E8
405 c = 3E8
388
406
389 isConfig = False
407 isConfig = False
390
408
391 basicHeaderObj = None
409 basicHeaderObj = None
392
410
393 systemHeaderObj = None
411 systemHeaderObj = None
394
412
395 radarControllerHeaderObj = None
413 radarControllerHeaderObj = None
396
414
397 processingHeaderObj = None
415 processingHeaderObj = None
398
416
399 online = 0
417 online = 0
400
418
401 dtype = None
419 dtype = None
402
420
403 pathList = []
421 pathList = []
404
422
405 filenameList = []
423 filenameList = []
406
424
407 filename = None
425 filename = None
408
426
409 ext = None
427 ext = None
410
428
411 flagIsNewFile = 1
429 flagIsNewFile = 1
412
430
413 flagDiscontinuousBlock = 0
431 flagDiscontinuousBlock = 0
414
432
415 flagIsNewBlock = 0
433 flagIsNewBlock = 0
416
434
417 fp = None
435 fp = None
418
436
419 firstHeaderSize = 0
437 firstHeaderSize = 0
420
438
421 basicHeaderSize = 24
439 basicHeaderSize = 24
422
440
423 versionFile = 1103
441 versionFile = 1103
424
442
425 fileSize = None
443 fileSize = None
426
444
427 # ippSeconds = None
445 # ippSeconds = None
428
446
429 fileSizeByHeader = None
447 fileSizeByHeader = None
430
448
431 fileIndex = None
449 fileIndex = None
432
450
433 profileIndex = None
451 profileIndex = None
434
452
435 blockIndex = None
453 blockIndex = None
436
454
437 nTotalBlocks = None
455 nTotalBlocks = None
438
456
439 maxTimeStep = 30
457 maxTimeStep = 30
440
458
441 lastUTTime = None
459 lastUTTime = None
442
460
443 datablock = None
461 datablock = None
444
462
445 dataOut = None
463 dataOut = None
446
464
447 blocksize = None
465 blocksize = None
448
466
449 getByBlock = False
467 getByBlock = False
450
468
451 def __init__(self):
469 def __init__(self):
452
470
453 raise ValueError, "Not implemented"
471 raise ValueError, "Not implemented"
454
472
455 def run(self):
473 def run(self):
456
474
457 raise ValueError, "Not implemented"
475 raise ValueError, "Not implemented"
458
476
459 def getDtypeWidth(self):
477 def getDtypeWidth(self):
460
478
461 dtype_index = get_dtype_index(self.dtype)
479 dtype_index = get_dtype_index(self.dtype)
462 dtype_width = get_dtype_width(dtype_index)
480 dtype_width = get_dtype_width(dtype_index)
463
481
464 return dtype_width
482 return dtype_width
465
483
466 class JRODataReader(JRODataIO):
484 class JRODataReader(JRODataIO):
467
485
468 nReadBlocks = 0
486 nReadBlocks = 0
469
487
470 delay = 10 #number of seconds waiting a new file
488 delay = 10 #number of seconds waiting a new file
471
489
472 nTries = 3 #quantity tries
490 nTries = 3 #quantity tries
473
491
474 nFiles = 3 #number of files for searching
492 nFiles = 3 #number of files for searching
475
493
476 path = None
494 path = None
477
495
478 foldercounter = 0
496 foldercounter = 0
479
497
480 flagNoMoreFiles = 0
498 flagNoMoreFiles = 0
481
499
482 datetimeList = []
500 datetimeList = []
483
501
484 __isFirstTimeOnline = 1
502 __isFirstTimeOnline = 1
485
503
486 __printInfo = True
504 __printInfo = True
487
505
488 profileIndex = None
506 profileIndex = None
489
507
490 nTxs = 1
508 nTxs = 1
491
509
492 txIndex = None
510 txIndex = None
493
511
494 def __init__(self):
512 def __init__(self):
495
513
496 """
514 """
497
515
498 """
516 """
499
517
500 # raise NotImplementedError, "This method has not been implemented"
518 # raise NotImplementedError, "This method has not been implemented"
501
519
502
520
503 def createObjByDefault(self):
521 def createObjByDefault(self):
504 """
522 """
505
523
506 """
524 """
507 raise NotImplementedError, "This method has not been implemented"
525 raise NotImplementedError, "This method has not been implemented"
508
526
509 def getBlockDimension(self):
527 def getBlockDimension(self):
510
528
511 raise NotImplementedError, "No implemented"
529 raise NotImplementedError, "No implemented"
512
530
513 def __searchFilesOffLine(self,
531 def __searchFilesOffLine(self,
514 path,
532 path,
515 startDate=None,
533 startDate=None,
516 endDate=None,
534 endDate=None,
517 startTime=datetime.time(0,0,0),
535 startTime=datetime.time(0,0,0),
518 endTime=datetime.time(23,59,59),
536 endTime=datetime.time(23,59,59),
519 set=None,
537 set=None,
520 expLabel='',
538 expLabel='',
521 ext='.r',
539 ext='.r',
522 walk=True):
540 walk=True):
523
541
524 self.filenameList = []
542 self.filenameList = []
525 self.datetimeList = []
543 self.datetimeList = []
526
544
527 pathList = []
545 pathList = []
528
546
529 # if not walk:
530 # #pathList.append(path)
531 # multi_path = path.split(',')
532 # for single_path in multi_path:
533 #
534 # if not os.path.isdir(single_path):
535 # continue
536 #
537 # pathList.append(single_path)
538 #
539 # else:
540 # #dirList = []
541 # multi_path = path.split(',')
542 # for single_path in multi_path:
543 #
544 # if not os.path.isdir(single_path):
545 # continue
546 #
547 # dirList = []
548 # for thisPath in os.listdir(single_path):
549 # if not os.path.isdir(os.path.join(single_path,thisPath)):
550 # continue
551 # if not isRadarFolder(thisPath):
552 # continue
553 #
554 # dirList.append(thisPath)
555 #
556 # if not(dirList):
557 # return None, None
558 #
559 # if startDate and endDate:
560 # thisDate = startDate
561 #
562 # while(thisDate <= endDate):
563 # year = thisDate.timetuple().tm_year
564 # doy = thisDate.timetuple().tm_yday
565 #
566 # matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
567 # if len(matchlist) == 0:
568 # thisDate += datetime.timedelta(1)
569 # continue
570 # for match in matchlist:
571 # pathList.append(os.path.join(single_path,match,expLabel))
572 #
573 # thisDate += datetime.timedelta(1)
574 # else:
575 # for thiDir in dirList:
576 # pathList.append(os.path.join(single_path,thiDir,expLabel))
577
578 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
547 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
579
548
580 if dateList == []:
549 if dateList == []:
581 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
550 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
551 datetime.datetime.combine(startDate,startTime).ctime(),
552 datetime.datetime.combine(endDate,endTime).ctime())
553
582 return None, None
554 return None, None
583
555
584 if len(dateList) > 1:
556 if len(dateList) > 1:
585 print "%d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
557 print "[Reading] %d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
586 else:
558 else:
587 print "data was found for the date %s" %(dateList[0])
559 print "[Reading] data was found for the date %s" %(dateList[0])
588
560
589 filenameList = []
561 filenameList = []
590 datetimeList = []
562 datetimeList = []
591 # pathDict = {}
592 # filenameList_to_sort = []
593 #
594 # for i in range(len(pathList)):
595 #
596 # thisPath = pathList[i]
597 #
598 # fileList = glob.glob1(thisPath, "*%s" %ext)
599 # if len(fileList) < 1:
600 # continue
601 # fileList.sort()
602 # pathDict.setdefault(fileList[0])
603 # pathDict[fileList[0]] = i
604 # filenameList_to_sort.append(fileList[0])
605 #
606 # filenameList_to_sort.sort()
607
563
608 for thisPath in pathList:
564 for thisPath in pathList:
609 # thisPath = pathList[pathDict[file]]
565 # thisPath = pathList[pathDict[file]]
610
566
611 fileList = glob.glob1(thisPath, "*%s" %ext)
567 fileList = glob.glob1(thisPath, "*%s" %ext)
612 fileList.sort()
568 fileList.sort()
613
569
614 for file in fileList:
570 for file in fileList:
615
571
616 filename = os.path.join(thisPath,file)
572 filename = os.path.join(thisPath,file)
617
573
618 if not isFileInDateRange(filename, startDate, endDate):
574 if not isFileInDateRange(filename, startDate, endDate):
619 continue
575 continue
620
576
621 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
577 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
622
578
623 if not(thisDatetime):
579 if not(thisDatetime):
624 continue
580 continue
625
581
626 filenameList.append(filename)
582 filenameList.append(filename)
627 datetimeList.append(thisDatetime)
583 datetimeList.append(thisDatetime)
628
584
629 if not(filenameList):
585 if not(filenameList):
630 print "Any file was found for the time range %s - %s" %(startTime, endTime)
586 print "[Reading] Any file was found int time range %s - %s" %(startTime.ctime(), endTime.ctime())
631 return None, None
587 return None, None
632
588
633 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
589 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
634 print
590 print
635
591
636 for i in range(len(filenameList)):
592 for i in range(len(filenameList)):
637 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
593 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
638
594
639 self.filenameList = filenameList
595 self.filenameList = filenameList
640 self.datetimeList = datetimeList
596 self.datetimeList = datetimeList
641
597
642 return pathList, filenameList
598 return pathList, filenameList
643
599
644 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
600 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
645
601
646 """
602 """
647 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
603 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
648 devuelve el archivo encontrado ademas de otros datos.
604 devuelve el archivo encontrado ademas de otros datos.
649
605
650 Input:
606 Input:
651 path : carpeta donde estan contenidos los files que contiene data
607 path : carpeta donde estan contenidos los files que contiene data
652
608
653 expLabel : Nombre del subexperimento (subfolder)
609 expLabel : Nombre del subexperimento (subfolder)
654
610
655 ext : extension de los files
611 ext : extension de los files
656
612
657 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
613 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
658
614
659 Return:
615 Return:
660 directory : eL directorio donde esta el file encontrado
616 directory : eL directorio donde esta el file encontrado
661 filename : el ultimo file de una determinada carpeta
617 filename : el ultimo file de una determinada carpeta
662 year : el anho
618 year : el anho
663 doy : el numero de dia del anho
619 doy : el numero de dia del anho
664 set : el set del archivo
620 set : el set del archivo
665
621
666
622
667 """
623 """
668 dirList = []
624 dirList = []
669
625
670 if not walk:
626 if not walk:
671 fullpath = path
627 fullpath = path
672 foldercounter = 0
628 foldercounter = 0
673 else:
629 else:
674 #Filtra solo los directorios
630 #Filtra solo los directorios
675 for thisPath in os.listdir(path):
631 for thisPath in os.listdir(path):
676 if not os.path.isdir(os.path.join(path,thisPath)):
632 if not os.path.isdir(os.path.join(path,thisPath)):
677 continue
633 continue
678 if not isRadarFolder(thisPath):
634 if not isRadarFolder(thisPath):
679 continue
635 continue
680
636
681 dirList.append(thisPath)
637 dirList.append(thisPath)
682
638
683 if not(dirList):
639 if not(dirList):
684 return None, None, None, None, None, None
640 return None, None, None, None, None, None
685
641
686 dirList = sorted( dirList, key=str.lower )
642 dirList = sorted( dirList, key=str.lower )
687
643
688 doypath = dirList[-1]
644 doypath = dirList[-1]
689 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
645 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
690 fullpath = os.path.join(path, doypath, expLabel)
646 fullpath = os.path.join(path, doypath, expLabel)
691
647
692
648
693 print "%s folder was found: " %(fullpath )
649 print "[Reading] %s folder was found: " %(fullpath )
694
650
695 if set == None:
651 if set == None:
696 filename = getlastFileFromPath(fullpath, ext)
652 filename = getlastFileFromPath(fullpath, ext)
697 else:
653 else:
698 filename = getFileFromSet(fullpath, ext, set)
654 filename = getFileFromSet(fullpath, ext, set)
699
655
700 if not(filename):
656 if not(filename):
701 return None, None, None, None, None, None
657 return None, None, None, None, None, None
702
658
703 print "%s file was found" %(filename)
659 print "[Reading] %s file was found" %(filename)
704
660
705 if not(self.__verifyFile(os.path.join(fullpath, filename))):
661 if not(self.__verifyFile(os.path.join(fullpath, filename))):
706 return None, None, None, None, None, None
662 return None, None, None, None, None, None
707
663
708 year = int( filename[1:5] )
664 year = int( filename[1:5] )
709 doy = int( filename[5:8] )
665 doy = int( filename[5:8] )
710 set = int( filename[8:11] )
666 set = int( filename[8:11] )
711
667
712 return fullpath, foldercounter, filename, year, doy, set
668 return fullpath, foldercounter, filename, year, doy, set
713
669
714 def __setNextFileOffline(self):
670 def __setNextFileOffline(self):
715
671
716 idFile = self.fileIndex
672 idFile = self.fileIndex
717
673
718 while (True):
674 while (True):
719 idFile += 1
675 idFile += 1
720 if not(idFile < len(self.filenameList)):
676 if not(idFile < len(self.filenameList)):
721 self.flagNoMoreFiles = 1
677 self.flagNoMoreFiles = 1
722 # print "[Reading] No more Files"
678 # print "[Reading] No more Files"
723 return 0
679 return 0
724
680
725 filename = self.filenameList[idFile]
681 filename = self.filenameList[idFile]
726
682
727 if not(self.__verifyFile(filename)):
683 if not(self.__verifyFile(filename)):
728 continue
684 continue
729
685
730 fileSize = os.path.getsize(filename)
686 fileSize = os.path.getsize(filename)
731 fp = open(filename,'rb')
687 fp = open(filename,'rb')
732 break
688 break
733
689
734 self.flagIsNewFile = 1
690 self.flagIsNewFile = 1
735 self.fileIndex = idFile
691 self.fileIndex = idFile
736 self.filename = filename
692 self.filename = filename
737 self.fileSize = fileSize
693 self.fileSize = fileSize
738 self.fp = fp
694 self.fp = fp
739
695
740 # print "[Reading] Setting the file: %s"%self.filename
696 # print "[Reading] Setting the file: %s"%self.filename
741
697
742 return 1
698 return 1
743
699
744 def __setNextFileOnline(self):
700 def __setNextFileOnline(self):
745 """
701 """
746 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
702 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
747 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
703 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
748 siguientes.
704 siguientes.
749
705
750 Affected:
706 Affected:
751 self.flagIsNewFile
707 self.flagIsNewFile
752 self.filename
708 self.filename
753 self.fileSize
709 self.fileSize
754 self.fp
710 self.fp
755 self.set
711 self.set
756 self.flagNoMoreFiles
712 self.flagNoMoreFiles
757
713
758 Return:
714 Return:
759 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
715 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
760 1 : si el file fue abierto con exito y esta listo a ser leido
716 1 : si el file fue abierto con exito y esta listo a ser leido
761
717
762 Excepciones:
718 Excepciones:
763 Si un determinado file no puede ser abierto
719 Si un determinado file no puede ser abierto
764 """
720 """
765 nFiles = 0
721 nFiles = 0
766 fileOk_flag = False
722 fileOk_flag = False
767 firstTime_flag = True
723 firstTime_flag = True
768
724
769 self.set += 1
725 self.set += 1
770
726
771 if self.set > 999:
727 if self.set > 999:
772 self.set = 0
728 self.set = 0
773 self.foldercounter += 1
729 self.foldercounter += 1
774
730
775 #busca el 1er file disponible
731 #busca el 1er file disponible
776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
732 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
777 if fullfilename:
733 if fullfilename:
778 if self.__verifyFile(fullfilename, False):
734 if self.__verifyFile(fullfilename, False):
779 fileOk_flag = True
735 fileOk_flag = True
780
736
781 #si no encuentra un file entonces espera y vuelve a buscar
737 #si no encuentra un file entonces espera y vuelve a buscar
782 if not(fileOk_flag):
738 if not(fileOk_flag):
783 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
739 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
784
740
785 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
741 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
786 tries = self.nTries
742 tries = self.nTries
787 else:
743 else:
788 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
744 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
789
745
790 for nTries in range( tries ):
746 for nTries in range( tries ):
791 if firstTime_flag:
747 if firstTime_flag:
792 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
748 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
793 sleep( self.delay )
749 sleep( self.delay )
794 else:
750 else:
795 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
751 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
796
752
797 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
753 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
798 if fullfilename:
754 if fullfilename:
799 if self.__verifyFile(fullfilename):
755 if self.__verifyFile(fullfilename):
800 fileOk_flag = True
756 fileOk_flag = True
801 break
757 break
802
758
803 if fileOk_flag:
759 if fileOk_flag:
804 break
760 break
805
761
806 firstTime_flag = False
762 firstTime_flag = False
807
763
808 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
764 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
809 self.set += 1
765 self.set += 1
810
766
811 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
767 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
812 self.set = 0
768 self.set = 0
813 self.doy += 1
769 self.doy += 1
814 self.foldercounter = 0
770 self.foldercounter = 0
815
771
816 if fileOk_flag:
772 if fileOk_flag:
817 self.fileSize = os.path.getsize( fullfilename )
773 self.fileSize = os.path.getsize( fullfilename )
818 self.filename = fullfilename
774 self.filename = fullfilename
819 self.flagIsNewFile = 1
775 self.flagIsNewFile = 1
820 if self.fp != None: self.fp.close()
776 if self.fp != None: self.fp.close()
821 self.fp = open(fullfilename, 'rb')
777 self.fp = open(fullfilename, 'rb')
822 self.flagNoMoreFiles = 0
778 self.flagNoMoreFiles = 0
823 # print '[Reading] Setting the file: %s' % fullfilename
779 # print '[Reading] Setting the file: %s' % fullfilename
824 else:
780 else:
825 self.fileSize = 0
781 self.fileSize = 0
826 self.filename = None
782 self.filename = None
827 self.flagIsNewFile = 0
783 self.flagIsNewFile = 0
828 self.fp = None
784 self.fp = None
829 self.flagNoMoreFiles = 1
785 self.flagNoMoreFiles = 1
830 # print '[Reading] No more files to read'
786 # print '[Reading] No more files to read'
831
787
832 return fileOk_flag
788 return fileOk_flag
833
789
834 def setNextFile(self):
790 def setNextFile(self):
835 if self.fp != None:
791 if self.fp != None:
836 self.fp.close()
792 self.fp.close()
837
793
838 if self.online:
794 if self.online:
839 newFile = self.__setNextFileOnline()
795 newFile = self.__setNextFileOnline()
840 else:
796 else:
841 newFile = self.__setNextFileOffline()
797 newFile = self.__setNextFileOffline()
842
798
843 if not(newFile):
799 if not(newFile):
844 print '[Reading] No more files to read'
800 print '[Reading] No more files to read'
845 return 0
801 return 0
846
802
847 print '[Reading] Setting the file: %s' % self.filename
803 print '[Reading] Setting the file: %s' % self.filename
848
804
849 self.__readFirstHeader()
805 self.__readFirstHeader()
850 self.nReadBlocks = 0
806 self.nReadBlocks = 0
851 return 1
807 return 1
852
808
853 def __waitNewBlock(self):
809 def __waitNewBlock(self):
854 """
810 """
855 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
811 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
856
812
857 Si el modo de lectura es OffLine siempre retorn 0
813 Si el modo de lectura es OffLine siempre retorn 0
858 """
814 """
859 if not self.online:
815 if not self.online:
860 return 0
816 return 0
861
817
862 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
818 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
863 return 0
819 return 0
864
820
865 currentPointer = self.fp.tell()
821 currentPointer = self.fp.tell()
866
822
867 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
823 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
868
824
869 for nTries in range( self.nTries ):
825 for nTries in range( self.nTries ):
870
826
871 self.fp.close()
827 self.fp.close()
872 self.fp = open( self.filename, 'rb' )
828 self.fp = open( self.filename, 'rb' )
873 self.fp.seek( currentPointer )
829 self.fp.seek( currentPointer )
874
830
875 self.fileSize = os.path.getsize( self.filename )
831 self.fileSize = os.path.getsize( self.filename )
876 currentSize = self.fileSize - currentPointer
832 currentSize = self.fileSize - currentPointer
877
833
878 if ( currentSize >= neededSize ):
834 if ( currentSize >= neededSize ):
879 self.basicHeaderObj.read(self.fp)
835 self.basicHeaderObj.read(self.fp)
880 return 1
836 return 1
881
837
882 if self.fileSize == self.fileSizeByHeader:
838 if self.fileSize == self.fileSizeByHeader:
883 # self.flagEoF = True
839 # self.flagEoF = True
884 return 0
840 return 0
885
841
886 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
842 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
887 sleep( self.delay )
843 sleep( self.delay )
888
844
889
845
890 return 0
846 return 0
891
847
892 def waitDataBlock(self,pointer_location):
848 def waitDataBlock(self,pointer_location):
893
849
894 currentPointer = pointer_location
850 currentPointer = pointer_location
895
851
896 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
852 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
897
853
898 for nTries in range( self.nTries ):
854 for nTries in range( self.nTries ):
899 self.fp.close()
855 self.fp.close()
900 self.fp = open( self.filename, 'rb' )
856 self.fp = open( self.filename, 'rb' )
901 self.fp.seek( currentPointer )
857 self.fp.seek( currentPointer )
902
858
903 self.fileSize = os.path.getsize( self.filename )
859 self.fileSize = os.path.getsize( self.filename )
904 currentSize = self.fileSize - currentPointer
860 currentSize = self.fileSize - currentPointer
905
861
906 if ( currentSize >= neededSize ):
862 if ( currentSize >= neededSize ):
907 return 1
863 return 1
908
864
909 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
910 sleep( self.delay )
866 sleep( self.delay )
911
867
912 return 0
868 return 0
913
869
914 def __jumpToLastBlock(self):
870 def __jumpToLastBlock(self):
915
871
916 if not(self.__isFirstTimeOnline):
872 if not(self.__isFirstTimeOnline):
917 return
873 return
918
874
919 csize = self.fileSize - self.fp.tell()
875 csize = self.fileSize - self.fp.tell()
920 blocksize = self.processingHeaderObj.blockSize
876 blocksize = self.processingHeaderObj.blockSize
921
877
922 #salta el primer bloque de datos
878 #salta el primer bloque de datos
923 if csize > self.processingHeaderObj.blockSize:
879 if csize > self.processingHeaderObj.blockSize:
924 self.fp.seek(self.fp.tell() + blocksize)
880 self.fp.seek(self.fp.tell() + blocksize)
925 else:
881 else:
926 return
882 return
927
883
928 csize = self.fileSize - self.fp.tell()
884 csize = self.fileSize - self.fp.tell()
929 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
885 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
930 while True:
886 while True:
931
887
932 if self.fp.tell()<self.fileSize:
888 if self.fp.tell()<self.fileSize:
933 self.fp.seek(self.fp.tell() + neededsize)
889 self.fp.seek(self.fp.tell() + neededsize)
934 else:
890 else:
935 self.fp.seek(self.fp.tell() - neededsize)
891 self.fp.seek(self.fp.tell() - neededsize)
936 break
892 break
937
893
938 # csize = self.fileSize - self.fp.tell()
894 # csize = self.fileSize - self.fp.tell()
939 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
895 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
940 # factor = int(csize/neededsize)
896 # factor = int(csize/neededsize)
941 # if factor > 0:
897 # if factor > 0:
942 # self.fp.seek(self.fp.tell() + factor*neededsize)
898 # self.fp.seek(self.fp.tell() + factor*neededsize)
943
899
944 self.flagIsNewFile = 0
900 self.flagIsNewFile = 0
945 self.__isFirstTimeOnline = 0
901 self.__isFirstTimeOnline = 0
946
902
947 def __setNewBlock(self):
903 def __setNewBlock(self):
948
904
949 if self.fp == None:
905 if self.fp == None:
950 return 0
906 return 0
951
907
952 if self.online:
908 if self.online:
953 self.__jumpToLastBlock()
909 self.__jumpToLastBlock()
954
910
955 if self.flagIsNewFile:
911 if self.flagIsNewFile:
956 return 1
912 return 1
957
913
958 self.lastUTTime = self.basicHeaderObj.utc
914 self.lastUTTime = self.basicHeaderObj.utc
959 currentSize = self.fileSize - self.fp.tell()
915 currentSize = self.fileSize - self.fp.tell()
960 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
961
917
962 if (currentSize >= neededSize):
918 if (currentSize >= neededSize):
963 self.basicHeaderObj.read(self.fp)
919 self.basicHeaderObj.read(self.fp)
964 return 1
920 return 1
965
921
966 if self.__waitNewBlock():
922 if self.__waitNewBlock():
967 return 1
923 return 1
968
924
969 if not(self.setNextFile()):
925 if not(self.setNextFile()):
970 return 0
926 return 0
971
927
972 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
928 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
973
929
974 self.flagDiscontinuousBlock = 0
930 self.flagDiscontinuousBlock = 0
975
931
976 if deltaTime > self.maxTimeStep:
932 if deltaTime > self.maxTimeStep:
977 self.flagDiscontinuousBlock = 1
933 self.flagDiscontinuousBlock = 1
978
934
979 return 1
935 return 1
980
936
981 def readNextBlock(self):
937 def readNextBlock(self):
982
938
983 if not(self.__setNewBlock()):
939 if not(self.__setNewBlock()):
984 return 0
940 return 0
985
941
986 if not(self.readBlock()):
942 if not(self.readBlock()):
987 return 0
943 return 0
988
944
989 self.getBasicHeader()
945 self.getBasicHeader()
990
946
991 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
947 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
992 self.processingHeaderObj.dataBlocksPerFile,
948 self.processingHeaderObj.dataBlocksPerFile,
993 self.dataOut.datatime.ctime())
949 self.dataOut.datatime.ctime())
994 return 1
950 return 1
995
951
996 def __readFirstHeader(self):
952 def __readFirstHeader(self):
997
953
998 self.basicHeaderObj.read(self.fp)
954 self.basicHeaderObj.read(self.fp)
999 self.systemHeaderObj.read(self.fp)
955 self.systemHeaderObj.read(self.fp)
1000 self.radarControllerHeaderObj.read(self.fp)
956 self.radarControllerHeaderObj.read(self.fp)
1001 self.processingHeaderObj.read(self.fp)
957 self.processingHeaderObj.read(self.fp)
1002
958
1003 self.firstHeaderSize = self.basicHeaderObj.size
959 self.firstHeaderSize = self.basicHeaderObj.size
1004
960
1005 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
961 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1006 if datatype == 0:
962 if datatype == 0:
1007 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
963 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1008 elif datatype == 1:
964 elif datatype == 1:
1009 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
965 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1010 elif datatype == 2:
966 elif datatype == 2:
1011 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
967 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1012 elif datatype == 3:
968 elif datatype == 3:
1013 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
969 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1014 elif datatype == 4:
970 elif datatype == 4:
1015 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
971 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1016 elif datatype == 5:
972 elif datatype == 5:
1017 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
973 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1018 else:
974 else:
1019 raise ValueError, 'Data type was not defined'
975 raise ValueError, 'Data type was not defined'
1020
976
1021 self.dtype = datatype_str
977 self.dtype = datatype_str
1022 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
978 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1023 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
979 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1024 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
980 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1025 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
981 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1026 self.getBlockDimension()
982 self.getBlockDimension()
1027
983
1028 def __verifyFile(self, filename, msgFlag=True):
984 def __verifyFile(self, filename, msgFlag=True):
1029 msg = None
985 msg = None
1030 try:
986 try:
1031 fp = open(filename, 'rb')
987 fp = open(filename, 'rb')
1032 currentPosition = fp.tell()
988 currentPosition = fp.tell()
1033 except IOError:
989 except IOError:
1034 traceback.print_exc()
990 traceback.print_exc()
1035 if msgFlag:
991 if msgFlag:
1036 print "[Reading] The file %s can't be opened" % (filename)
992 print "[Reading] The file %s can't be opened" % (filename)
1037 return False
993 return False
1038
994
1039 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
995 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1040
996
1041 if neededSize == 0:
997 if neededSize == 0:
1042 basicHeaderObj = BasicHeader(LOCALTIME)
998 basicHeaderObj = BasicHeader(LOCALTIME)
1043 systemHeaderObj = SystemHeader()
999 systemHeaderObj = SystemHeader()
1044 radarControllerHeaderObj = RadarControllerHeader()
1000 radarControllerHeaderObj = RadarControllerHeader()
1045 processingHeaderObj = ProcessingHeader()
1001 processingHeaderObj = ProcessingHeader()
1046
1002
1047 try:
1003 try:
1048 if not( basicHeaderObj.read(fp) ): raise IOError
1004 if not( basicHeaderObj.read(fp) ): raise IOError
1049 if not( systemHeaderObj.read(fp) ): raise IOError
1005 if not( systemHeaderObj.read(fp) ): raise IOError
1050 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1006 if not( radarControllerHeaderObj.read(fp) ): raise IOError
1051 if not( processingHeaderObj.read(fp) ): raise IOError
1007 if not( processingHeaderObj.read(fp) ): raise IOError
1052 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1008 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1053
1009
1054 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1010 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1055
1011
1056 except IOError:
1012 except IOError:
1057 traceback.print_exc()
1013 traceback.print_exc()
1058 sys.exit(0)
1014 sys.exit(0)
1059
1015
1060 if msgFlag:
1016 if msgFlag:
1061 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1017 print "[Reading] The file %s is empty or it hasn't enough data" % filename
1062
1018
1063 fp.close()
1019 fp.close()
1064 return False
1020 return False
1065 else:
1021 else:
1066 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1022 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1067
1023
1068 fp.close()
1024 fp.close()
1069 fileSize = os.path.getsize(filename)
1025 fileSize = os.path.getsize(filename)
1070 currentSize = fileSize - currentPosition
1026 currentSize = fileSize - currentPosition
1071 if currentSize < neededSize:
1027 if currentSize < neededSize:
1072 if msgFlag and (msg != None):
1028 if msgFlag and (msg != None):
1073 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1029 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
1074 return False
1030 return False
1075
1031
1076 return True
1032 return True
1077
1033
1078 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1034 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1079
1035
1080 dateList = []
1036 dateList = []
1081 pathList = []
1037 pathList = []
1082
1038
1083 multi_path = path.split(',')
1039 multi_path = path.split(',')
1084
1040
1085 if not walk:
1041 if not walk:
1086
1042
1087 for single_path in multi_path:
1043 for single_path in multi_path:
1088
1044
1089 if not os.path.isdir(single_path):
1045 if not os.path.isdir(single_path):
1090 continue
1046 continue
1091
1047
1092 fileList = glob.glob1(single_path, "*"+ext)
1048 fileList = glob.glob1(single_path, "*"+ext)
1093
1049
1094 for thisFile in fileList:
1050 for thisFile in fileList:
1095
1051
1096 if not os.path.isfile(os.path.join(single_path, thisFile)):
1052 if not os.path.isfile(os.path.join(single_path, thisFile)):
1097 continue
1053 continue
1098
1054
1099 if not isRadarFile(thisFile):
1055 if not isRadarFile(thisFile):
1100 continue
1056 continue
1101
1057
1102 if not isFileInDateRange(thisFile, startDate, endDate):
1058 if not isFileInDateRange(thisFile, startDate, endDate):
1103 continue
1059 continue
1104
1060
1105 thisDate = getDateFromRadarFile(thisFile)
1061 thisDate = getDateFromRadarFile(thisFile)
1106
1062
1107 if thisDate in dateList:
1063 if thisDate in dateList:
1108 continue
1064 continue
1109
1065
1110 dateList.append(thisDate)
1066 dateList.append(thisDate)
1111 pathList.append(single_path)
1067 pathList.append(single_path)
1112
1068
1113 else:
1069 else:
1114 for single_path in multi_path:
1070 for single_path in multi_path:
1115
1071
1116 if not os.path.isdir(single_path):
1072 if not os.path.isdir(single_path):
1117 continue
1073 continue
1118
1074
1119 dirList = []
1075 dirList = []
1120
1076
1121 for thisPath in os.listdir(single_path):
1077 for thisPath in os.listdir(single_path):
1122
1078
1123 if not os.path.isdir(os.path.join(single_path,thisPath)):
1079 if not os.path.isdir(os.path.join(single_path,thisPath)):
1124 continue
1080 continue
1125
1081
1126 if not isRadarFolder(thisPath):
1082 if not isRadarFolder(thisPath):
1127 continue
1083 continue
1128
1084
1129 if not isFolderInDateRange(thisPath, startDate, endDate):
1085 if not isFolderInDateRange(thisPath, startDate, endDate):
1130 continue
1086 continue
1131
1087
1132 dirList.append(thisPath)
1088 dirList.append(thisPath)
1133
1089
1134 if not dirList:
1090 if not dirList:
1135 continue
1091 continue
1136
1092
1137 for thisDir in dirList:
1093 for thisDir in dirList:
1138
1094
1139 datapath = os.path.join(single_path, thisDir, expLabel)
1095 datapath = os.path.join(single_path, thisDir, expLabel)
1140 fileList = glob.glob1(datapath, "*"+ext)
1096 fileList = glob.glob1(datapath, "*"+ext)
1141
1097
1142 if len(fileList) < 1:
1098 if len(fileList) < 1:
1143 continue
1099 continue
1144
1100
1145 thisDate = getDateFromRadarFolder(thisDir)
1101 thisDate = getDateFromRadarFolder(thisDir)
1146
1102
1147 pathList.append(datapath)
1103 pathList.append(datapath)
1148 dateList.append(thisDate)
1104 dateList.append(thisDate)
1149
1105
1150 dateList.sort()
1106 dateList.sort()
1151
1107
1152 if include_path:
1108 if include_path:
1153 return dateList, pathList
1109 return dateList, pathList
1154
1110
1155 return dateList
1111 return dateList
1156
1112
1157 def setup(self,
1113 def setup(self,
1158 path=None,
1114 path=None,
1159 startDate=None,
1115 startDate=None,
1160 endDate=None,
1116 endDate=None,
1161 startTime=datetime.time(0,0,0),
1117 startTime=datetime.time(0,0,0),
1162 endTime=datetime.time(23,59,59),
1118 endTime=datetime.time(23,59,59),
1163 set=None,
1119 set=None,
1164 expLabel = "",
1120 expLabel = "",
1165 ext = None,
1121 ext = None,
1166 online = False,
1122 online = False,
1167 delay = 60,
1123 delay = 60,
1168 walk = True,
1124 walk = True,
1169 getblock = False,
1125 getblock = False,
1170 nTxs = 1):
1126 nTxs = 1):
1171
1127
1172 if path == None:
1128 if path == None:
1173 raise ValueError, "[Reading] The path is not valid"
1129 raise ValueError, "[Reading] The path is not valid"
1174
1130
1175 if ext == None:
1131 if ext == None:
1176 ext = self.ext
1132 ext = self.ext
1177
1133
1178 if online:
1134 if online:
1179 print "[Reading] Searching files in online mode..."
1135 print "[Reading] Searching files in online mode..."
1180
1136
1181 for nTries in range( self.nTries ):
1137 for nTries in range( self.nTries ):
1182 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1138 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1183
1139
1184 if fullpath:
1140 if fullpath:
1185 break
1141 break
1186
1142
1187 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1143 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1188 sleep( self.delay )
1144 sleep( self.delay )
1189
1145
1190 if not(fullpath):
1146 if not(fullpath):
1191 print "[Reading] There 'isn't any valid file in %s" % path
1147 print "[Reading] There 'isn't any valid file in %s" % path
1192 return None
1148 return None
1193
1149
1194 self.year = year
1150 self.year = year
1195 self.doy = doy
1151 self.doy = doy
1196 self.set = set - 1
1152 self.set = set - 1
1197 self.path = path
1153 self.path = path
1198 self.foldercounter = foldercounter
1154 self.foldercounter = foldercounter
1199 last_set = None
1155 last_set = None
1200
1156
1201 else:
1157 else:
1202 print "[Reading] Searching files in offline mode ..."
1158 print "[Reading] Searching files in offline mode ..."
1203 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1159 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1204 startTime=startTime, endTime=endTime,
1160 startTime=startTime, endTime=endTime,
1205 set=set, expLabel=expLabel, ext=ext,
1161 set=set, expLabel=expLabel, ext=ext,
1206 walk=walk)
1162 walk=walk)
1207
1163
1208 if not(pathList):
1164 if not(pathList):
1209 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1165 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1210 datetime.datetime.combine(startDate,startTime).ctime(),
1166 # datetime.datetime.combine(startDate,startTime).ctime(),
1211 datetime.datetime.combine(endDate,endTime).ctime())
1167 # datetime.datetime.combine(endDate,endTime).ctime())
1212
1168 #
1213 sys.exit(-1)
1169 sys.exit(-1)
1214
1170
1215
1171
1216 self.fileIndex = -1
1172 self.fileIndex = -1
1217 self.pathList = pathList
1173 self.pathList = pathList
1218 self.filenameList = filenameList
1174 self.filenameList = filenameList
1219 file_name = os.path.basename(filenameList[-1])
1175 file_name = os.path.basename(filenameList[-1])
1220 basename, ext = os.path.splitext(file_name)
1176 basename, ext = os.path.splitext(file_name)
1221 last_set = int(basename[-3:])
1177 last_set = int(basename[-3:])
1222
1178
1223 self.online = online
1179 self.online = online
1224 self.delay = delay
1180 self.delay = delay
1225 ext = ext.lower()
1181 ext = ext.lower()
1226 self.ext = ext
1182 self.ext = ext
1227 self.getByBlock = getblock
1183 self.getByBlock = getblock
1228 self.nTxs = int(nTxs)
1184 self.nTxs = int(nTxs)
1229
1185
1230 if not(self.setNextFile()):
1186 if not(self.setNextFile()):
1231 if (startDate!=None) and (endDate!=None):
1187 if (startDate!=None) and (endDate!=None):
1232 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1188 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1233 elif startDate != None:
1189 elif startDate != None:
1234 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1190 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1235 else:
1191 else:
1236 print "[Reading] No files"
1192 print "[Reading] No files"
1237
1193
1238 sys.exit(-1)
1194 sys.exit(-1)
1239
1195
1240 # self.getBasicHeader()
1196 # self.getBasicHeader()
1241
1197
1242 if last_set != None:
1198 if last_set != None:
1243 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1199 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1244 return
1200 return
1245
1201
1246 def getBasicHeader(self):
1202 def getBasicHeader(self):
1247
1203
1248 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1204 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1249
1205
1250 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1206 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1251
1207
1252 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1208 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1253
1209
1254 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1210 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1255
1211
1256 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1212 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1257
1213
1258 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1214 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1259
1215
1260 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1216 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1261
1217
1262 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1218 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1263
1219
1264
1220
1265 def getFirstHeader(self):
1221 def getFirstHeader(self):
1266
1222
1267 raise ValueError, "This method has not been implemented"
1223 raise ValueError, "This method has not been implemented"
1268
1224
1269 def getData(self):
1225 def getData(self):
1270
1226
1271 raise ValueError, "This method has not been implemented"
1227 raise ValueError, "This method has not been implemented"
1272
1228
1273 def hasNotDataInBuffer(self):
1229 def hasNotDataInBuffer(self):
1274
1230
1275 raise ValueError, "This method has not been implemented"
1231 raise ValueError, "This method has not been implemented"
1276
1232
1277 def readBlock(self):
1233 def readBlock(self):
1278
1234
1279 raise ValueError, "This method has not been implemented"
1235 raise ValueError, "This method has not been implemented"
1280
1236
1281 def isEndProcess(self):
1237 def isEndProcess(self):
1282
1238
1283 return self.flagNoMoreFiles
1239 return self.flagNoMoreFiles
1284
1240
1285 def printReadBlocks(self):
1241 def printReadBlocks(self):
1286
1242
1287 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1243 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1288
1244
1289 def printTotalBlocks(self):
1245 def printTotalBlocks(self):
1290
1246
1291 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1247 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1292
1248
1293 def printNumberOfBlock(self):
1249 def printNumberOfBlock(self):
1294
1250
1295 if self.flagIsNewBlock:
1251 if self.flagIsNewBlock:
1296 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1252 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1297 self.processingHeaderObj.dataBlocksPerFile,
1253 self.processingHeaderObj.dataBlocksPerFile,
1298 self.dataOut.datatime.ctime())
1254 self.dataOut.datatime.ctime())
1299
1255
1300 def printInfo(self):
1256 def printInfo(self):
1301
1257
1302 if self.__printInfo == False:
1258 if self.__printInfo == False:
1303 return
1259 return
1304
1260
1305 self.basicHeaderObj.printInfo()
1261 self.basicHeaderObj.printInfo()
1306 self.systemHeaderObj.printInfo()
1262 self.systemHeaderObj.printInfo()
1307 self.radarControllerHeaderObj.printInfo()
1263 self.radarControllerHeaderObj.printInfo()
1308 self.processingHeaderObj.printInfo()
1264 self.processingHeaderObj.printInfo()
1309
1265
1310 self.__printInfo = False
1266 self.__printInfo = False
1311
1267
1312
1268
1313 def run(self, **kwargs):
1269 def run(self, **kwargs):
1314
1270
1315 if not(self.isConfig):
1271 if not(self.isConfig):
1316
1272
1317 # self.dataOut = dataOut
1273 # self.dataOut = dataOut
1318 self.setup(**kwargs)
1274 self.setup(**kwargs)
1319 self.isConfig = True
1275 self.isConfig = True
1320
1276
1321 self.getData()
1277 self.getData()
1322
1278
1323 class JRODataWriter(JRODataIO):
1279 class JRODataWriter(JRODataIO):
1324
1280
1325 """
1281 """
1326 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1282 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1327 de los datos siempre se realiza por bloques.
1283 de los datos siempre se realiza por bloques.
1328 """
1284 """
1329
1285
1330 blockIndex = 0
1286 blockIndex = 0
1331
1287
1332 path = None
1288 path = None
1333
1289
1334 setFile = None
1290 setFile = None
1335
1291
1336 profilesPerBlock = None
1292 profilesPerBlock = None
1337
1293
1338 blocksPerFile = None
1294 blocksPerFile = None
1339
1295
1340 nWriteBlocks = 0
1296 nWriteBlocks = 0
1341
1297
1342 fileDate = None
1298 fileDate = None
1343
1299
1344 def __init__(self, dataOut=None):
1300 def __init__(self, dataOut=None):
1345 raise ValueError, "Not implemented"
1301 raise ValueError, "Not implemented"
1346
1302
1347
1303
1348 def hasAllDataInBuffer(self):
1304 def hasAllDataInBuffer(self):
1349 raise ValueError, "Not implemented"
1305 raise ValueError, "Not implemented"
1350
1306
1351
1307
1352 def setBlockDimension(self):
1308 def setBlockDimension(self):
1353 raise ValueError, "Not implemented"
1309 raise ValueError, "Not implemented"
1354
1310
1355
1311
1356 def writeBlock(self):
1312 def writeBlock(self):
1357 raise ValueError, "No implemented"
1313 raise ValueError, "No implemented"
1358
1314
1359
1315
1360 def putData(self):
1316 def putData(self):
1361 raise ValueError, "No implemented"
1317 raise ValueError, "No implemented"
1362
1318
1363
1319
1364 def getProcessFlags(self):
1320 def getProcessFlags(self):
1365
1321
1366 processFlags = 0
1322 processFlags = 0
1367
1323
1368 dtype_index = get_dtype_index(self.dtype)
1324 dtype_index = get_dtype_index(self.dtype)
1369 procflag_dtype = get_procflag_dtype(dtype_index)
1325 procflag_dtype = get_procflag_dtype(dtype_index)
1370
1326
1371 processFlags += procflag_dtype
1327 processFlags += procflag_dtype
1372
1328
1373 if self.dataOut.flagDecodeData:
1329 if self.dataOut.flagDecodeData:
1374 processFlags += PROCFLAG.DECODE_DATA
1330 processFlags += PROCFLAG.DECODE_DATA
1375
1331
1376 if self.dataOut.flagDeflipData:
1332 if self.dataOut.flagDeflipData:
1377 processFlags += PROCFLAG.DEFLIP_DATA
1333 processFlags += PROCFLAG.DEFLIP_DATA
1378
1334
1379 if self.dataOut.code is not None:
1335 if self.dataOut.code is not None:
1380 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1336 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1381
1337
1382 if self.dataOut.nCohInt > 1:
1338 if self.dataOut.nCohInt > 1:
1383 processFlags += PROCFLAG.COHERENT_INTEGRATION
1339 processFlags += PROCFLAG.COHERENT_INTEGRATION
1384
1340
1385 if self.dataOut.type == "Spectra":
1341 if self.dataOut.type == "Spectra":
1386 if self.dataOut.nIncohInt > 1:
1342 if self.dataOut.nIncohInt > 1:
1387 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1343 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1388
1344
1389 if self.dataOut.data_dc is not None:
1345 if self.dataOut.data_dc is not None:
1390 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1346 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1391
1347
1392 if self.dataOut.flagShiftFFT:
1348 if self.dataOut.flagShiftFFT:
1393 processFlags += PROCFLAG.SHIFT_FFT_DATA
1349 processFlags += PROCFLAG.SHIFT_FFT_DATA
1394
1350
1395 return processFlags
1351 return processFlags
1396
1352
1397 def setBasicHeader(self):
1353 def setBasicHeader(self):
1398
1354
1399 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1355 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1400 self.basicHeaderObj.version = self.versionFile
1356 self.basicHeaderObj.version = self.versionFile
1401 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1357 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1402
1358
1403 utc = numpy.floor(self.dataOut.utctime)
1359 utc = numpy.floor(self.dataOut.utctime)
1404 milisecond = (self.dataOut.utctime - utc)* 1000.0
1360 milisecond = (self.dataOut.utctime - utc)* 1000.0
1405
1361
1406 self.basicHeaderObj.utc = utc
1362 self.basicHeaderObj.utc = utc
1407 self.basicHeaderObj.miliSecond = milisecond
1363 self.basicHeaderObj.miliSecond = milisecond
1408 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1364 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1409 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1365 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1410 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1366 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1411
1367
1412 def setFirstHeader(self):
1368 def setFirstHeader(self):
1413 """
1369 """
1414 Obtiene una copia del First Header
1370 Obtiene una copia del First Header
1415
1371
1416 Affected:
1372 Affected:
1417
1373
1418 self.basicHeaderObj
1374 self.basicHeaderObj
1419 self.systemHeaderObj
1375 self.systemHeaderObj
1420 self.radarControllerHeaderObj
1376 self.radarControllerHeaderObj
1421 self.processingHeaderObj self.
1377 self.processingHeaderObj self.
1422
1378
1423 Return:
1379 Return:
1424 None
1380 None
1425 """
1381 """
1426
1382
1427 raise ValueError, "No implemented"
1383 raise ValueError, "No implemented"
1428
1384
1429 def __writeFirstHeader(self):
1385 def __writeFirstHeader(self):
1430 """
1386 """
1431 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1387 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1432
1388
1433 Affected:
1389 Affected:
1434 __dataType
1390 __dataType
1435
1391
1436 Return:
1392 Return:
1437 None
1393 None
1438 """
1394 """
1439
1395
1440 # CALCULAR PARAMETROS
1396 # CALCULAR PARAMETROS
1441
1397
1442 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1398 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1443 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1399 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1444
1400
1445 self.basicHeaderObj.write(self.fp)
1401 self.basicHeaderObj.write(self.fp)
1446 self.systemHeaderObj.write(self.fp)
1402 self.systemHeaderObj.write(self.fp)
1447 self.radarControllerHeaderObj.write(self.fp)
1403 self.radarControllerHeaderObj.write(self.fp)
1448 self.processingHeaderObj.write(self.fp)
1404 self.processingHeaderObj.write(self.fp)
1449
1405
1450 def __setNewBlock(self):
1406 def __setNewBlock(self):
1451 """
1407 """
1452 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1408 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1453
1409
1454 Return:
1410 Return:
1455 0 : si no pudo escribir nada
1411 0 : si no pudo escribir nada
1456 1 : Si escribio el Basic el First Header
1412 1 : Si escribio el Basic el First Header
1457 """
1413 """
1458 if self.fp == None:
1414 if self.fp == None:
1459 self.setNextFile()
1415 self.setNextFile()
1460
1416
1461 if self.flagIsNewFile:
1417 if self.flagIsNewFile:
1462 return 1
1418 return 1
1463
1419
1464 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1420 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1465 self.basicHeaderObj.write(self.fp)
1421 self.basicHeaderObj.write(self.fp)
1466 return 1
1422 return 1
1467
1423
1468 if not( self.setNextFile() ):
1424 if not( self.setNextFile() ):
1469 return 0
1425 return 0
1470
1426
1471 return 1
1427 return 1
1472
1428
1473
1429
1474 def writeNextBlock(self):
1430 def writeNextBlock(self):
1475 """
1431 """
1476 Selecciona el bloque siguiente de datos y los escribe en un file
1432 Selecciona el bloque siguiente de datos y los escribe en un file
1477
1433
1478 Return:
1434 Return:
1479 0 : Si no hizo pudo escribir el bloque de datos
1435 0 : Si no hizo pudo escribir el bloque de datos
1480 1 : Si no pudo escribir el bloque de datos
1436 1 : Si no pudo escribir el bloque de datos
1481 """
1437 """
1482 if not( self.__setNewBlock() ):
1438 if not( self.__setNewBlock() ):
1483 return 0
1439 return 0
1484
1440
1485 self.writeBlock()
1441 self.writeBlock()
1486
1442
1487 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1443 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1488 self.processingHeaderObj.dataBlocksPerFile)
1444 self.processingHeaderObj.dataBlocksPerFile)
1489
1445
1490 return 1
1446 return 1
1491
1447
1492 def setNextFile(self):
1448 def setNextFile(self):
1493 """
1449 """
1494 Determina el siguiente file que sera escrito
1450 Determina el siguiente file que sera escrito
1495
1451
1496 Affected:
1452 Affected:
1497 self.filename
1453 self.filename
1498 self.subfolder
1454 self.subfolder
1499 self.fp
1455 self.fp
1500 self.setFile
1456 self.setFile
1501 self.flagIsNewFile
1457 self.flagIsNewFile
1502
1458
1503 Return:
1459 Return:
1504 0 : Si el archivo no puede ser escrito
1460 0 : Si el archivo no puede ser escrito
1505 1 : Si el archivo esta listo para ser escrito
1461 1 : Si el archivo esta listo para ser escrito
1506 """
1462 """
1507 ext = self.ext
1463 ext = self.ext
1508 path = self.path
1464 path = self.path
1509
1465
1510 if self.fp != None:
1466 if self.fp != None:
1511 self.fp.close()
1467 self.fp.close()
1512
1468
1513 timeTuple = time.localtime( self.dataOut.utctime)
1469 timeTuple = time.localtime( self.dataOut.utctime)
1514 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1470 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1515
1471
1516 fullpath = os.path.join( path, subfolder )
1472 fullpath = os.path.join( path, subfolder )
1517 setFile = self.setFile
1473 setFile = self.setFile
1518
1474
1519 if not( os.path.exists(fullpath) ):
1475 if not( os.path.exists(fullpath) ):
1520 os.mkdir(fullpath)
1476 os.mkdir(fullpath)
1521 setFile = -1 #inicializo mi contador de seteo
1477 setFile = -1 #inicializo mi contador de seteo
1522 else:
1478 else:
1523 filesList = os.listdir( fullpath )
1479 filesList = os.listdir( fullpath )
1524 if len( filesList ) > 0:
1480 if len( filesList ) > 0:
1525 filesList = sorted( filesList, key=str.lower )
1481 filesList = sorted( filesList, key=str.lower )
1526 filen = filesList[-1]
1482 filen = filesList[-1]
1527 # el filename debera tener el siguiente formato
1483 # el filename debera tener el siguiente formato
1528 # 0 1234 567 89A BCDE (hex)
1484 # 0 1234 567 89A BCDE (hex)
1529 # x YYYY DDD SSS .ext
1485 # x YYYY DDD SSS .ext
1530 if isNumber( filen[8:11] ):
1486 if isNumber( filen[8:11] ):
1531 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1487 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1532 else:
1488 else:
1533 setFile = -1
1489 setFile = -1
1534 else:
1490 else:
1535 setFile = -1 #inicializo mi contador de seteo
1491 setFile = -1 #inicializo mi contador de seteo
1536
1492
1537 setFile += 1
1493 setFile += 1
1538
1494
1539 #If this is a new day it resets some values
1495 #If this is a new day it resets some values
1540 if self.dataOut.datatime.date() > self.fileDate:
1496 if self.dataOut.datatime.date() > self.fileDate:
1541 setFile = 0
1497 setFile = 0
1542 self.nTotalBlocks = 0
1498 self.nTotalBlocks = 0
1543
1499
1544 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1500 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1545
1501
1546 filename = os.path.join( path, subfolder, filen )
1502 filename = os.path.join( path, subfolder, filen )
1547
1503
1548 fp = open( filename,'wb' )
1504 fp = open( filename,'wb' )
1549
1505
1550 self.blockIndex = 0
1506 self.blockIndex = 0
1551
1507
1552 #guardando atributos
1508 #guardando atributos
1553 self.filename = filename
1509 self.filename = filename
1554 self.subfolder = subfolder
1510 self.subfolder = subfolder
1555 self.fp = fp
1511 self.fp = fp
1556 self.setFile = setFile
1512 self.setFile = setFile
1557 self.flagIsNewFile = 1
1513 self.flagIsNewFile = 1
1558 self.fileDate = self.dataOut.datatime.date()
1514 self.fileDate = self.dataOut.datatime.date()
1559
1515
1560 self.setFirstHeader()
1516 self.setFirstHeader()
1561
1517
1562 print '[Writing] Opening file: %s'%self.filename
1518 print '[Writing] Opening file: %s'%self.filename
1563
1519
1564 self.__writeFirstHeader()
1520 self.__writeFirstHeader()
1565
1521
1566 return 1
1522 return 1
1567
1523
1568 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1524 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1569 """
1525 """
1570 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1526 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1571
1527
1572 Inputs:
1528 Inputs:
1573 path : directory where data will be saved
1529 path : directory where data will be saved
1574 profilesPerBlock : number of profiles per block
1530 profilesPerBlock : number of profiles per block
1575 set : initial file set
1531 set : initial file set
1576 datatype : An integer number that defines data type:
1532 datatype : An integer number that defines data type:
1577 0 : int8 (1 byte)
1533 0 : int8 (1 byte)
1578 1 : int16 (2 bytes)
1534 1 : int16 (2 bytes)
1579 2 : int32 (4 bytes)
1535 2 : int32 (4 bytes)
1580 3 : int64 (8 bytes)
1536 3 : int64 (8 bytes)
1581 4 : float32 (4 bytes)
1537 4 : float32 (4 bytes)
1582 5 : double64 (8 bytes)
1538 5 : double64 (8 bytes)
1583
1539
1584 Return:
1540 Return:
1585 0 : Si no realizo un buen seteo
1541 0 : Si no realizo un buen seteo
1586 1 : Si realizo un buen seteo
1542 1 : Si realizo un buen seteo
1587 """
1543 """
1588
1544
1589 if ext == None:
1545 if ext == None:
1590 ext = self.ext
1546 ext = self.ext
1591
1547
1592 self.ext = ext.lower()
1548 self.ext = ext.lower()
1593
1549
1594 self.path = path
1550 self.path = path
1595
1551
1596 if set is None:
1552 if set is None:
1597 self.setFile = -1
1553 self.setFile = -1
1598 else:
1554 else:
1599 self.setFile = set - 1
1555 self.setFile = set - 1
1600
1556
1601 self.blocksPerFile = blocksPerFile
1557 self.blocksPerFile = blocksPerFile
1602
1558
1603 self.profilesPerBlock = profilesPerBlock
1559 self.profilesPerBlock = profilesPerBlock
1604
1560
1605 self.dataOut = dataOut
1561 self.dataOut = dataOut
1606 self.fileDate = self.dataOut.datatime.date()
1562 self.fileDate = self.dataOut.datatime.date()
1607 #By default
1563 #By default
1608 self.dtype = self.dataOut.dtype
1564 self.dtype = self.dataOut.dtype
1609
1565
1610 if datatype is not None:
1566 if datatype is not None:
1611 self.dtype = get_numpy_dtype(datatype)
1567 self.dtype = get_numpy_dtype(datatype)
1612
1568
1613 if not(self.setNextFile()):
1569 if not(self.setNextFile()):
1614 print "[Writing] There isn't a next file"
1570 print "[Writing] There isn't a next file"
1615 return 0
1571 return 0
1616
1572
1617 self.setBlockDimension()
1573 self.setBlockDimension()
1618
1574
1619 return 1
1575 return 1
1620
1576
1621 def run(self, dataOut, **kwargs):
1577 def run(self, dataOut, **kwargs):
1622
1578
1623 if not(self.isConfig):
1579 if not(self.isConfig):
1624
1580
1625 self.setup(dataOut, **kwargs)
1581 self.setup(dataOut, **kwargs)
1626 self.isConfig = True
1582 self.isConfig = True
1627
1583
1628 self.putData()
1584 self.putData()
1629
1585
General Comments 0
You need to be logged in to leave comments. Login now