##// END OF EJS Templates
schain
RSS
Miguel Valdez -
r700:5ddbdeccaf1c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1636 +1,1639
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 print "The file %s can't be opened" %(filename)
71 print "The file %s can't be opened" %(filename)
72 return 0
72 return 0
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
90 Inputs:
90 Inputs:
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92
92
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94
94
95 endDate : fecha final del rango seleccionado en formato datetime.date
95 endDate : fecha final del rango seleccionado en formato datetime.date
96
96
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98
98
99 endTime : tiempo final del rango seleccionado en formato datetime.time
99 endTime : tiempo final del rango seleccionado en formato datetime.time
100
100
101 Return:
101 Return:
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 fecha especificado, de lo contrario retorna False.
103 fecha especificado, de lo contrario retorna False.
104
104
105 Excepciones:
105 Excepciones:
106 Si el archivo no existe o no puede ser abierto
106 Si el archivo no existe o no puede ser abierto
107 Si la cabecera no puede ser leida.
107 Si la cabecera no puede ser leida.
108
108
109 """
109 """
110
110
111
111
112 try:
112 try:
113 fp = open(filename,'rb')
113 fp = open(filename,'rb')
114 except IOError:
114 except IOError:
115 print "The file %s can't be opened" %(filename)
115 print "The file %s can't be opened" %(filename)
116 return None
116 return None
117
117
118 basicHeaderObj = BasicHeader(LOCALTIME)
118 basicHeaderObj = BasicHeader(LOCALTIME)
119 sts = basicHeaderObj.read(fp)
119 sts = basicHeaderObj.read(fp)
120 fp.close()
120 fp.close()
121
121
122 thisDatetime = basicHeaderObj.datatime
122 thisDatetime = basicHeaderObj.datatime
123 thisDate = thisDatetime.date()
123 thisDate = thisDatetime.date()
124 thisTime = thisDatetime.time()
124 thisTime = thisDatetime.time()
125
125
126 if not(sts):
126 if not(sts):
127 print "Skipping the file %s because it has not a valid header" %(filename)
127 print "Skipping the file %s because it has not a valid header" %(filename)
128 return None
128 return None
129
129
130 #General case
130 #General case
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 #-----------o----------------------------o-----------
132 #-----------o----------------------------o-----------
133 # startTime endTime
133 # startTime endTime
134
134
135 if endTime >= startTime:
135 if endTime >= startTime:
136 if (thisTime < startTime) or (thisTime > endTime):
136 if (thisTime < startTime) or (thisTime > endTime):
137 return None
137 return None
138
138
139 return thisDatetime
139 return thisDatetime
140
140
141 #If endTime < startTime then endTime belongs to the next day
141 #If endTime < startTime then endTime belongs to the next day
142
142
143
143
144 #<<<<<<<<<<<o o>>>>>>>>>>>
144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 #-----------o----------------------------o-----------
145 #-----------o----------------------------o-----------
146 # endTime startTime
146 # endTime startTime
147
147
148 if (thisDate == startDate) and (thisTime < startTime):
148 if (thisDate == startDate) and (thisTime < startTime):
149 return None
149 return None
150
150
151 if (thisDate == endDate) and (thisTime > endTime):
151 if (thisDate == endDate) and (thisTime > endTime):
152 return None
152 return None
153
153
154 if (thisTime < startTime) and (thisTime > endTime):
154 if (thisTime < startTime) and (thisTime > endTime):
155 return None
155 return None
156
156
157 return thisDatetime
157 return thisDatetime
158
158
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 """
160 """
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162
162
163 Inputs:
163 Inputs:
164 folder : nombre completo del directorio.
164 folder : nombre completo del directorio.
165 Su formato deberia ser "/path_root/?YYYYDDD"
165 Su formato deberia ser "/path_root/?YYYYDDD"
166
166
167 siendo:
167 siendo:
168 YYYY : Anio (ejemplo 2015)
168 YYYY : Anio (ejemplo 2015)
169 DDD : Dia del anio (ejemplo 305)
169 DDD : Dia del anio (ejemplo 305)
170
170
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172
172
173 endDate : fecha final del rango seleccionado en formato datetime.date
173 endDate : fecha final del rango seleccionado en formato datetime.date
174
174
175 Return:
175 Return:
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 fecha especificado, de lo contrario retorna False.
177 fecha especificado, de lo contrario retorna False.
178 Excepciones:
178 Excepciones:
179 Si el directorio no tiene el formato adecuado
179 Si el directorio no tiene el formato adecuado
180 """
180 """
181
181
182 basename = os.path.basename(folder)
182 basename = os.path.basename(folder)
183
183
184 if not isRadarFolder(basename):
184 if not isRadarFolder(basename):
185 print "The folder %s has not the rigth format" %folder
185 print "The folder %s has not the rigth format" %folder
186 return 0
186 return 0
187
187
188 if startDate and endDate:
188 if startDate and endDate:
189 thisDate = getDateFromRadarFolder(basename)
189 thisDate = getDateFromRadarFolder(basename)
190
190
191 if thisDate < startDate:
191 if thisDate < startDate:
192 return 0
192 return 0
193
193
194 if thisDate > endDate:
194 if thisDate > endDate:
195 return 0
195 return 0
196
196
197 return 1
197 return 1
198
198
199 def isFileInDateRange(filename, startDate=None, endDate=None):
199 def isFileInDateRange(filename, startDate=None, endDate=None):
200 """
200 """
201 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
201 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
202
202
203 Inputs:
203 Inputs:
204 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
204 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
205
205
206 Su formato deberia ser "?YYYYDDDsss"
206 Su formato deberia ser "?YYYYDDDsss"
207
207
208 siendo:
208 siendo:
209 YYYY : Anio (ejemplo 2015)
209 YYYY : Anio (ejemplo 2015)
210 DDD : Dia del anio (ejemplo 305)
210 DDD : Dia del anio (ejemplo 305)
211 sss : set
211 sss : set
212
212
213 startDate : fecha inicial del rango seleccionado en formato datetime.date
213 startDate : fecha inicial del rango seleccionado en formato datetime.date
214
214
215 endDate : fecha final del rango seleccionado en formato datetime.date
215 endDate : fecha final del rango seleccionado en formato datetime.date
216
216
217 Return:
217 Return:
218 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
218 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
219 fecha especificado, de lo contrario retorna False.
219 fecha especificado, de lo contrario retorna False.
220 Excepciones:
220 Excepciones:
221 Si el archivo no tiene el formato adecuado
221 Si el archivo no tiene el formato adecuado
222 """
222 """
223
223
224 basename = os.path.basename(filename)
224 basename = os.path.basename(filename)
225
225
226 if not isRadarFile(basename):
226 if not isRadarFile(basename):
227 print "The filename %s has not the rigth format" %filename
227 print "The filename %s has not the rigth format" %filename
228 return 0
228 return 0
229
229
230 if startDate and endDate:
230 if startDate and endDate:
231 thisDate = getDateFromRadarFile(basename)
231 thisDate = getDateFromRadarFile(basename)
232
232
233 if thisDate < startDate:
233 if thisDate < startDate:
234 return 0
234 return 0
235
235
236 if thisDate > endDate:
236 if thisDate > endDate:
237 return 0
237 return 0
238
238
239 return 1
239 return 1
240
240
241 def getFileFromSet(path, ext, set):
241 def getFileFromSet(path, ext, set):
242 validFilelist = []
242 validFilelist = []
243 fileList = os.listdir(path)
243 fileList = os.listdir(path)
244
244
245 # 0 1234 567 89A BCDE
245 # 0 1234 567 89A BCDE
246 # H YYYY DDD SSS .ext
246 # H YYYY DDD SSS .ext
247
247
248 for thisFile in fileList:
248 for thisFile in fileList:
249 try:
249 try:
250 year = int(thisFile[1:5])
250 year = int(thisFile[1:5])
251 doy = int(thisFile[5:8])
251 doy = int(thisFile[5:8])
252 except:
252 except:
253 continue
253 continue
254
254
255 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
255 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
256 continue
256 continue
257
257
258 validFilelist.append(thisFile)
258 validFilelist.append(thisFile)
259
259
260 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
260 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
261
261
262 if len(myfile)!= 0:
262 if len(myfile)!= 0:
263 return myfile[0]
263 return myfile[0]
264 else:
264 else:
265 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
265 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
266 print 'the filename %s does not exist'%filename
266 print 'the filename %s does not exist'%filename
267 print '...going to the last file: '
267 print '...going to the last file: '
268
268
269 if validFilelist:
269 if validFilelist:
270 validFilelist = sorted( validFilelist, key=str.lower )
270 validFilelist = sorted( validFilelist, key=str.lower )
271 return validFilelist[-1]
271 return validFilelist[-1]
272
272
273 return None
273 return None
274
274
275 def getlastFileFromPath(path, ext):
275 def getlastFileFromPath(path, ext):
276 """
276 """
277 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
277 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
278 al final de la depuracion devuelve el ultimo file de la lista que quedo.
278 al final de la depuracion devuelve el ultimo file de la lista que quedo.
279
279
280 Input:
280 Input:
281 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
281 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
282 ext : extension de los files contenidos en una carpeta
282 ext : extension de los files contenidos en una carpeta
283
283
284 Return:
284 Return:
285 El ultimo file de una determinada carpeta, no se considera el path.
285 El ultimo file de una determinada carpeta, no se considera el path.
286 """
286 """
287 validFilelist = []
287 validFilelist = []
288 fileList = os.listdir(path)
288 fileList = os.listdir(path)
289
289
290 # 0 1234 567 89A BCDE
290 # 0 1234 567 89A BCDE
291 # H YYYY DDD SSS .ext
291 # H YYYY DDD SSS .ext
292
292
293 for thisFile in fileList:
293 for thisFile in fileList:
294
294
295 year = thisFile[1:5]
295 year = thisFile[1:5]
296 if not isNumber(year):
296 if not isNumber(year):
297 continue
297 continue
298
298
299 doy = thisFile[5:8]
299 doy = thisFile[5:8]
300 if not isNumber(doy):
300 if not isNumber(doy):
301 continue
301 continue
302
302
303 year = int(year)
303 year = int(year)
304 doy = int(doy)
304 doy = int(doy)
305
305
306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 continue
307 continue
308
308
309 validFilelist.append(thisFile)
309 validFilelist.append(thisFile)
310
310
311 if validFilelist:
311 if validFilelist:
312 validFilelist = sorted( validFilelist, key=str.lower )
312 validFilelist = sorted( validFilelist, key=str.lower )
313 return validFilelist[-1]
313 return validFilelist[-1]
314
314
315 return None
315 return None
316
316
317 def checkForRealPath(path, foldercounter, year, doy, set, ext):
317 def checkForRealPath(path, foldercounter, year, doy, set, ext):
318 """
318 """
319 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
319 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
320 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
320 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
321 el path exacto de un determinado file.
321 el path exacto de un determinado file.
322
322
323 Example :
323 Example :
324 nombre correcto del file es .../.../D2009307/P2009307367.ext
324 nombre correcto del file es .../.../D2009307/P2009307367.ext
325
325
326 Entonces la funcion prueba con las siguientes combinaciones
326 Entonces la funcion prueba con las siguientes combinaciones
327 .../.../y2009307367.ext
327 .../.../y2009307367.ext
328 .../.../Y2009307367.ext
328 .../.../Y2009307367.ext
329 .../.../x2009307/y2009307367.ext
329 .../.../x2009307/y2009307367.ext
330 .../.../x2009307/Y2009307367.ext
330 .../.../x2009307/Y2009307367.ext
331 .../.../X2009307/y2009307367.ext
331 .../.../X2009307/y2009307367.ext
332 .../.../X2009307/Y2009307367.ext
332 .../.../X2009307/Y2009307367.ext
333 siendo para este caso, la ultima combinacion de letras, identica al file buscado
333 siendo para este caso, la ultima combinacion de letras, identica al file buscado
334
334
335 Return:
335 Return:
336 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
336 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
337 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
337 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
338 para el filename
338 para el filename
339 """
339 """
340 fullfilename = None
340 fullfilename = None
341 find_flag = False
341 find_flag = False
342 filename = None
342 filename = None
343
343
344 prefixDirList = [None,'d','D']
344 prefixDirList = [None,'d','D']
345 if ext.lower() == ".r": #voltage
345 if ext.lower() == ".r": #voltage
346 prefixFileList = ['d','D']
346 prefixFileList = ['d','D']
347 elif ext.lower() == ".pdata": #spectra
347 elif ext.lower() == ".pdata": #spectra
348 prefixFileList = ['p','P']
348 prefixFileList = ['p','P']
349 else:
349 else:
350 return None, filename
350 return None, filename
351
351
352 #barrido por las combinaciones posibles
352 #barrido por las combinaciones posibles
353 for prefixDir in prefixDirList:
353 for prefixDir in prefixDirList:
354 thispath = path
354 thispath = path
355 if prefixDir != None:
355 if prefixDir != None:
356 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
356 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
357 if foldercounter == 0:
357 if foldercounter == 0:
358 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
358 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
359 else:
359 else:
360 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
360 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
361 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
361 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
362 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
362 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
363 fullfilename = os.path.join( thispath, filename ) #formo el path completo
363 fullfilename = os.path.join( thispath, filename ) #formo el path completo
364
364
365 if os.path.exists( fullfilename ): #verifico que exista
365 if os.path.exists( fullfilename ): #verifico que exista
366 find_flag = True
366 find_flag = True
367 break
367 break
368 if find_flag:
368 if find_flag:
369 break
369 break
370
370
371 if not(find_flag):
371 if not(find_flag):
372 return None, filename
372 return None, filename
373
373
374 return fullfilename, filename
374 return fullfilename, filename
375
375
376 def isRadarFolder(folder):
376 def isRadarFolder(folder):
377 try:
377 try:
378 year = int(folder[1:5])
378 year = int(folder[1:5])
379 doy = int(folder[5:8])
379 doy = int(folder[5:8])
380 except:
380 except:
381 return 0
381 return 0
382
382
383 return 1
383 return 1
384
384
385 def isRadarFile(file):
385 def isRadarFile(file):
386 try:
386 try:
387 year = int(file[1:5])
387 year = int(file[1:5])
388 doy = int(file[5:8])
388 doy = int(file[5:8])
389 set = int(file[8:11])
389 set = int(file[8:11])
390 except:
390 except:
391 return 0
391 return 0
392
392
393 return 1
393 return 1
394
394
395 def getDateFromRadarFile(file):
395 def getDateFromRadarFile(file):
396 try:
396 try:
397 year = int(file[1:5])
397 year = int(file[1:5])
398 doy = int(file[5:8])
398 doy = int(file[5:8])
399 set = int(file[8:11])
399 set = int(file[8:11])
400 except:
400 except:
401 return None
401 return None
402
402
403 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
403 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
404 return thisDate
404 return thisDate
405
405
406 def getDateFromRadarFolder(folder):
406 def getDateFromRadarFolder(folder):
407 try:
407 try:
408 year = int(folder[1:5])
408 year = int(folder[1:5])
409 doy = int(folder[5:8])
409 doy = int(folder[5:8])
410 except:
410 except:
411 return None
411 return None
412
412
413 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
413 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
414 return thisDate
414 return thisDate
415
415
416 class JRODataIO:
416 class JRODataIO:
417
417
418 c = 3E8
418 c = 3E8
419
419
420 isConfig = False
420 isConfig = False
421
421
422 basicHeaderObj = None
422 basicHeaderObj = None
423
423
424 systemHeaderObj = None
424 systemHeaderObj = None
425
425
426 radarControllerHeaderObj = None
426 radarControllerHeaderObj = None
427
427
428 processingHeaderObj = None
428 processingHeaderObj = None
429
429
430 dtype = None
430 dtype = None
431
431
432 pathList = []
432 pathList = []
433
433
434 filenameList = []
434 filenameList = []
435
435
436 filename = None
436 filename = None
437
437
438 ext = None
438 ext = None
439
439
440 flagIsNewFile = 1
440 flagIsNewFile = 1
441
441
442 flagDiscontinuousBlock = 0
442 flagDiscontinuousBlock = 0
443
443
444 flagIsNewBlock = 0
444 flagIsNewBlock = 0
445
445
446 fp = None
446 fp = None
447
447
448 firstHeaderSize = 0
448 firstHeaderSize = 0
449
449
450 basicHeaderSize = 24
450 basicHeaderSize = 24
451
451
452 versionFile = 1103
452 versionFile = 1103
453
453
454 fileSize = None
454 fileSize = None
455
455
456 # ippSeconds = None
456 # ippSeconds = None
457
457
458 fileSizeByHeader = None
458 fileSizeByHeader = None
459
459
460 fileIndex = None
460 fileIndex = None
461
461
462 profileIndex = None
462 profileIndex = None
463
463
464 blockIndex = None
464 blockIndex = None
465
465
466 nTotalBlocks = None
466 nTotalBlocks = None
467
467
468 maxTimeStep = 30
468 maxTimeStep = 30
469
469
470 lastUTTime = None
470 lastUTTime = None
471
471
472 datablock = None
472 datablock = None
473
473
474 dataOut = None
474 dataOut = None
475
475
476 blocksize = None
476 blocksize = None
477
477
478 getByBlock = False
478 getByBlock = False
479
479
480 def __init__(self):
480 def __init__(self):
481
481
482 raise NotImplementedError
482 raise NotImplementedError
483
483
484 def run(self):
484 def run(self):
485
485
486 raise NotImplementedError
486 raise NotImplementedError
487
487
488 def getDtypeWidth(self):
488 def getDtypeWidth(self):
489
489
490 dtype_index = get_dtype_index(self.dtype)
490 dtype_index = get_dtype_index(self.dtype)
491 dtype_width = get_dtype_width(dtype_index)
491 dtype_width = get_dtype_width(dtype_index)
492
492
493 return dtype_width
493 return dtype_width
494
494
495 class JRODataReader(JRODataIO):
495 class JRODataReader(JRODataIO):
496
496
497
497
498 online = 0
498 online = 0
499
499
500 realtime = 0
500 realtime = 0
501
501
502 nReadBlocks = 0
502 nReadBlocks = 0
503
503
504 delay = 10 #number of seconds waiting a new file
504 delay = 10 #number of seconds waiting a new file
505
505
506 nTries = 3 #quantity tries
506 nTries = 3 #quantity tries
507
507
508 nFiles = 3 #number of files for searching
508 nFiles = 3 #number of files for searching
509
509
510 path = None
510 path = None
511
511
512 foldercounter = 0
512 foldercounter = 0
513
513
514 flagNoMoreFiles = 0
514 flagNoMoreFiles = 0
515
515
516 datetimeList = []
516 datetimeList = []
517
517
518 __isFirstTimeOnline = 1
518 __isFirstTimeOnline = 1
519
519
520 __printInfo = True
520 __printInfo = True
521
521
522 profileIndex = None
522 profileIndex = None
523
523
524 nTxs = 1
524 nTxs = 1
525
525
526 txIndex = None
526 txIndex = None
527
527
528 def __init__(self):
528 def __init__(self):
529
529
530 """
530 """
531 This class is used to find data files
531 This class is used to find data files
532
532
533 Example:
533 Example:
534 reader = JRODataReader()
534 reader = JRODataReader()
535 fileList = reader.findDataFiles()
535 fileList = reader.findDataFiles()
536
536
537 """
537 """
538 pass
538 pass
539
539
540
540
541 def createObjByDefault(self):
541 def createObjByDefault(self):
542 """
542 """
543
543
544 """
544 """
545 raise NotImplementedError
545 raise NotImplementedError
546
546
547 def getBlockDimension(self):
547 def getBlockDimension(self):
548
548
549 raise NotImplementedError
549 raise NotImplementedError
550
550
551 def __searchFilesOffLine(self,
551 def __searchFilesOffLine(self,
552 path,
552 path,
553 startDate=None,
553 startDate=None,
554 endDate=None,
554 endDate=None,
555 startTime=datetime.time(0,0,0),
555 startTime=datetime.time(0,0,0),
556 endTime=datetime.time(23,59,59),
556 endTime=datetime.time(23,59,59),
557 set=None,
557 set=None,
558 expLabel='',
558 expLabel='',
559 ext='.r',
559 ext='.r',
560 walk=True):
560 walk=True):
561
561
562 self.filenameList = []
562 self.filenameList = []
563 self.datetimeList = []
563 self.datetimeList = []
564
564
565 pathList = []
565 pathList = []
566
566
567 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
567 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
568
568
569 if dateList == []:
569 if dateList == []:
570 # print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
570 # print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
571 # datetime.datetime.combine(startDate,startTime).ctime(),
571 # datetime.datetime.combine(startDate,startTime).ctime(),
572 # datetime.datetime.combine(endDate,endTime).ctime())
572 # datetime.datetime.combine(endDate,endTime).ctime())
573
573
574 return None, None
574 return None, None
575
575
576 if len(dateList) > 1:
576 if len(dateList) > 1:
577 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
577 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
578 else:
578 else:
579 print "[Reading] data was found for the date %s" %(dateList[0])
579 print "[Reading] data was found for the date %s" %(dateList[0])
580
580
581 filenameList = []
581 filenameList = []
582 datetimeList = []
582 datetimeList = []
583
583
584 for thisPath in pathList:
584 for thisPath in pathList:
585 # thisPath = pathList[pathDict[file]]
585 # thisPath = pathList[pathDict[file]]
586
586
587 fileList = glob.glob1(thisPath, "*%s" %ext)
587 fileList = glob.glob1(thisPath, "*%s" %ext)
588 fileList.sort()
588 fileList.sort()
589
589
590 for file in fileList:
590 for file in fileList:
591
591
592 filename = os.path.join(thisPath,file)
592 filename = os.path.join(thisPath,file)
593
593
594 if not isFileInDateRange(filename, startDate, endDate):
594 if not isFileInDateRange(filename, startDate, endDate):
595 continue
595 continue
596
596
597 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
597 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
598
598
599 if not(thisDatetime):
599 if not(thisDatetime):
600 continue
600 continue
601
601
602 filenameList.append(filename)
602 filenameList.append(filename)
603 datetimeList.append(thisDatetime)
603 datetimeList.append(thisDatetime)
604
604
605 if not(filenameList):
605 if not(filenameList):
606 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
606 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
607 return None, None
607 return None, None
608
608
609 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
609 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
610 print
610 print
611
611
612 for i in range(len(filenameList)):
612 for i in range(len(filenameList)):
613 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
613 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
614
614
615 self.filenameList = filenameList
615 self.filenameList = filenameList
616 self.datetimeList = datetimeList
616 self.datetimeList = datetimeList
617
617
618 return pathList, filenameList
618 return pathList, filenameList
619
619
620 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
620 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
621
621
622 """
622 """
623 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
623 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
624 devuelve el archivo encontrado ademas de otros datos.
624 devuelve el archivo encontrado ademas de otros datos.
625
625
626 Input:
626 Input:
627 path : carpeta donde estan contenidos los files que contiene data
627 path : carpeta donde estan contenidos los files que contiene data
628
628
629 expLabel : Nombre del subexperimento (subfolder)
629 expLabel : Nombre del subexperimento (subfolder)
630
630
631 ext : extension de los files
631 ext : extension de los files
632
632
633 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
633 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
634
634
635 Return:
635 Return:
636 directory : eL directorio donde esta el file encontrado
636 directory : eL directorio donde esta el file encontrado
637 filename : el ultimo file de una determinada carpeta
637 filename : el ultimo file de una determinada carpeta
638 year : el anho
638 year : el anho
639 doy : el numero de dia del anho
639 doy : el numero de dia del anho
640 set : el set del archivo
640 set : el set del archivo
641
641
642
642
643 """
643 """
644 dirList = []
644 if not os.path.isdir(path):
645 return None, None, None, None, None, None
645
646
647 dirList = []
648
646 if not walk:
649 if not walk:
647 fullpath = path
650 fullpath = path
648 foldercounter = 0
651 foldercounter = 0
649 else:
652 else:
650 #Filtra solo los directorios
653 #Filtra solo los directorios
651 for thisPath in os.listdir(path):
654 for thisPath in os.listdir(path):
652 if not os.path.isdir(os.path.join(path,thisPath)):
655 if not os.path.isdir(os.path.join(path,thisPath)):
653 continue
656 continue
654 if not isRadarFolder(thisPath):
657 if not isRadarFolder(thisPath):
655 continue
658 continue
656
659
657 dirList.append(thisPath)
660 dirList.append(thisPath)
658
661
659 if not(dirList):
662 if not(dirList):
660 return None, None, None, None, None, None
663 return None, None, None, None, None, None
661
664
662 dirList = sorted( dirList, key=str.lower )
665 dirList = sorted( dirList, key=str.lower )
663
666
664 doypath = dirList[-1]
667 doypath = dirList[-1]
665 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
668 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
666 fullpath = os.path.join(path, doypath, expLabel)
669 fullpath = os.path.join(path, doypath, expLabel)
667
670
668
671
669 print "[Reading] %s folder was found: " %(fullpath )
672 print "[Reading] %s folder was found: " %(fullpath )
670
673
671 if set == None:
674 if set == None:
672 filename = getlastFileFromPath(fullpath, ext)
675 filename = getlastFileFromPath(fullpath, ext)
673 else:
676 else:
674 filename = getFileFromSet(fullpath, ext, set)
677 filename = getFileFromSet(fullpath, ext, set)
675
678
676 if not(filename):
679 if not(filename):
677 return None, None, None, None, None, None
680 return None, None, None, None, None, None
678
681
679 print "[Reading] %s file was found" %(filename)
682 print "[Reading] %s file was found" %(filename)
680
683
681 if not(self.__verifyFile(os.path.join(fullpath, filename))):
684 if not(self.__verifyFile(os.path.join(fullpath, filename))):
682 return None, None, None, None, None, None
685 return None, None, None, None, None, None
683
686
684 year = int( filename[1:5] )
687 year = int( filename[1:5] )
685 doy = int( filename[5:8] )
688 doy = int( filename[5:8] )
686 set = int( filename[8:11] )
689 set = int( filename[8:11] )
687
690
688 return fullpath, foldercounter, filename, year, doy, set
691 return fullpath, foldercounter, filename, year, doy, set
689
692
690 def __setNextFileOffline(self):
693 def __setNextFileOffline(self):
691
694
692 idFile = self.fileIndex
695 idFile = self.fileIndex
693
696
694 while (True):
697 while (True):
695 idFile += 1
698 idFile += 1
696 if not(idFile < len(self.filenameList)):
699 if not(idFile < len(self.filenameList)):
697 self.flagNoMoreFiles = 1
700 self.flagNoMoreFiles = 1
698 # print "[Reading] No more Files"
701 # print "[Reading] No more Files"
699 return 0
702 return 0
700
703
701 filename = self.filenameList[idFile]
704 filename = self.filenameList[idFile]
702
705
703 if not(self.__verifyFile(filename)):
706 if not(self.__verifyFile(filename)):
704 continue
707 continue
705
708
706 fileSize = os.path.getsize(filename)
709 fileSize = os.path.getsize(filename)
707 fp = open(filename,'rb')
710 fp = open(filename,'rb')
708 break
711 break
709
712
710 self.flagIsNewFile = 1
713 self.flagIsNewFile = 1
711 self.fileIndex = idFile
714 self.fileIndex = idFile
712 self.filename = filename
715 self.filename = filename
713 self.fileSize = fileSize
716 self.fileSize = fileSize
714 self.fp = fp
717 self.fp = fp
715
718
716 # print "[Reading] Setting the file: %s"%self.filename
719 # print "[Reading] Setting the file: %s"%self.filename
717
720
718 return 1
721 return 1
719
722
720 def __setNextFileOnline(self):
723 def __setNextFileOnline(self):
721 """
724 """
722 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
725 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
723 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
726 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
724 siguientes.
727 siguientes.
725
728
726 Affected:
729 Affected:
727 self.flagIsNewFile
730 self.flagIsNewFile
728 self.filename
731 self.filename
729 self.fileSize
732 self.fileSize
730 self.fp
733 self.fp
731 self.set
734 self.set
732 self.flagNoMoreFiles
735 self.flagNoMoreFiles
733
736
734 Return:
737 Return:
735 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
738 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
736 1 : si el file fue abierto con exito y esta listo a ser leido
739 1 : si el file fue abierto con exito y esta listo a ser leido
737
740
738 Excepciones:
741 Excepciones:
739 Si un determinado file no puede ser abierto
742 Si un determinado file no puede ser abierto
740 """
743 """
741 nFiles = 0
744 nFiles = 0
742 fileOk_flag = False
745 fileOk_flag = False
743 firstTime_flag = True
746 firstTime_flag = True
744
747
745 self.set += 1
748 self.set += 1
746
749
747 if self.set > 999:
750 if self.set > 999:
748 self.set = 0
751 self.set = 0
749 self.foldercounter += 1
752 self.foldercounter += 1
750
753
751 #busca el 1er file disponible
754 #busca el 1er file disponible
752 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
755 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
753 if fullfilename:
756 if fullfilename:
754 if self.__verifyFile(fullfilename, False):
757 if self.__verifyFile(fullfilename, False):
755 fileOk_flag = True
758 fileOk_flag = True
756
759
757 #si no encuentra un file entonces espera y vuelve a buscar
760 #si no encuentra un file entonces espera y vuelve a buscar
758 if not(fileOk_flag):
761 if not(fileOk_flag):
759 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
762 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
760
763
761 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
764 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
762 tries = self.nTries
765 tries = self.nTries
763 else:
766 else:
764 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
767 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
765
768
766 for nTries in range( tries ):
769 for nTries in range( tries ):
767 if firstTime_flag:
770 if firstTime_flag:
768 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
771 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
769 sleep( self.delay )
772 sleep( self.delay )
770 else:
773 else:
771 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
774 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
772
775
773 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
774 if fullfilename:
777 if fullfilename:
775 if self.__verifyFile(fullfilename):
778 if self.__verifyFile(fullfilename):
776 fileOk_flag = True
779 fileOk_flag = True
777 break
780 break
778
781
779 if fileOk_flag:
782 if fileOk_flag:
780 break
783 break
781
784
782 firstTime_flag = False
785 firstTime_flag = False
783
786
784 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
787 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
785 self.set += 1
788 self.set += 1
786
789
787 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
790 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
788 self.set = 0
791 self.set = 0
789 self.doy += 1
792 self.doy += 1
790 self.foldercounter = 0
793 self.foldercounter = 0
791
794
792 if fileOk_flag:
795 if fileOk_flag:
793 self.fileSize = os.path.getsize( fullfilename )
796 self.fileSize = os.path.getsize( fullfilename )
794 self.filename = fullfilename
797 self.filename = fullfilename
795 self.flagIsNewFile = 1
798 self.flagIsNewFile = 1
796 if self.fp != None: self.fp.close()
799 if self.fp != None: self.fp.close()
797 self.fp = open(fullfilename, 'rb')
800 self.fp = open(fullfilename, 'rb')
798 self.flagNoMoreFiles = 0
801 self.flagNoMoreFiles = 0
799 # print '[Reading] Setting the file: %s' % fullfilename
802 # print '[Reading] Setting the file: %s' % fullfilename
800 else:
803 else:
801 self.fileSize = 0
804 self.fileSize = 0
802 self.filename = None
805 self.filename = None
803 self.flagIsNewFile = 0
806 self.flagIsNewFile = 0
804 self.fp = None
807 self.fp = None
805 self.flagNoMoreFiles = 1
808 self.flagNoMoreFiles = 1
806 # print '[Reading] No more files to read'
809 # print '[Reading] No more files to read'
807
810
808 return fileOk_flag
811 return fileOk_flag
809
812
810 def setNextFile(self):
813 def setNextFile(self):
811 if self.fp != None:
814 if self.fp != None:
812 self.fp.close()
815 self.fp.close()
813
816
814 if self.online:
817 if self.online:
815 newFile = self.__setNextFileOnline()
818 newFile = self.__setNextFileOnline()
816 else:
819 else:
817 newFile = self.__setNextFileOffline()
820 newFile = self.__setNextFileOffline()
818
821
819 if not(newFile):
822 if not(newFile):
820 print '[Reading] No more files to read'
823 print '[Reading] No more files to read'
821 return 0
824 return 0
822
825
823 print '[Reading] Setting the file: %s' % self.filename
826 print '[Reading] Setting the file: %s' % self.filename
824
827
825 self.__readFirstHeader()
828 self.__readFirstHeader()
826 self.nReadBlocks = 0
829 self.nReadBlocks = 0
827 return 1
830 return 1
828
831
829 def __waitNewBlock(self):
832 def __waitNewBlock(self):
830 """
833 """
831 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
834 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
832
835
833 Si el modo de lectura es OffLine siempre retorn 0
836 Si el modo de lectura es OffLine siempre retorn 0
834 """
837 """
835 if not self.online:
838 if not self.online:
836 return 0
839 return 0
837
840
838 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
841 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
839 return 0
842 return 0
840
843
841 currentPointer = self.fp.tell()
844 currentPointer = self.fp.tell()
842
845
843 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
846 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
844
847
845 for nTries in range( self.nTries ):
848 for nTries in range( self.nTries ):
846
849
847 self.fp.close()
850 self.fp.close()
848 self.fp = open( self.filename, 'rb' )
851 self.fp = open( self.filename, 'rb' )
849 self.fp.seek( currentPointer )
852 self.fp.seek( currentPointer )
850
853
851 self.fileSize = os.path.getsize( self.filename )
854 self.fileSize = os.path.getsize( self.filename )
852 currentSize = self.fileSize - currentPointer
855 currentSize = self.fileSize - currentPointer
853
856
854 if ( currentSize >= neededSize ):
857 if ( currentSize >= neededSize ):
855 self.basicHeaderObj.read(self.fp)
858 self.basicHeaderObj.read(self.fp)
856 return 1
859 return 1
857
860
858 if self.fileSize == self.fileSizeByHeader:
861 if self.fileSize == self.fileSizeByHeader:
859 # self.flagEoF = True
862 # self.flagEoF = True
860 return 0
863 return 0
861
864
862 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
863 sleep( self.delay )
866 sleep( self.delay )
864
867
865
868
866 return 0
869 return 0
867
870
868 def waitDataBlock(self,pointer_location):
871 def waitDataBlock(self,pointer_location):
869
872
870 currentPointer = pointer_location
873 currentPointer = pointer_location
871
874
872 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
875 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
873
876
874 for nTries in range( self.nTries ):
877 for nTries in range( self.nTries ):
875 self.fp.close()
878 self.fp.close()
876 self.fp = open( self.filename, 'rb' )
879 self.fp = open( self.filename, 'rb' )
877 self.fp.seek( currentPointer )
880 self.fp.seek( currentPointer )
878
881
879 self.fileSize = os.path.getsize( self.filename )
882 self.fileSize = os.path.getsize( self.filename )
880 currentSize = self.fileSize - currentPointer
883 currentSize = self.fileSize - currentPointer
881
884
882 if ( currentSize >= neededSize ):
885 if ( currentSize >= neededSize ):
883 return 1
886 return 1
884
887
885 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
888 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
886 sleep( self.delay )
889 sleep( self.delay )
887
890
888 return 0
891 return 0
889
892
890 def __jumpToLastBlock(self):
893 def __jumpToLastBlock(self):
891
894
892 if not(self.__isFirstTimeOnline):
895 if not(self.__isFirstTimeOnline):
893 return
896 return
894
897
895 csize = self.fileSize - self.fp.tell()
898 csize = self.fileSize - self.fp.tell()
896 blocksize = self.processingHeaderObj.blockSize
899 blocksize = self.processingHeaderObj.blockSize
897
900
898 #salta el primer bloque de datos
901 #salta el primer bloque de datos
899 if csize > self.processingHeaderObj.blockSize:
902 if csize > self.processingHeaderObj.blockSize:
900 self.fp.seek(self.fp.tell() + blocksize)
903 self.fp.seek(self.fp.tell() + blocksize)
901 else:
904 else:
902 return
905 return
903
906
904 csize = self.fileSize - self.fp.tell()
907 csize = self.fileSize - self.fp.tell()
905 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
908 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
906 while True:
909 while True:
907
910
908 if self.fp.tell()<self.fileSize:
911 if self.fp.tell()<self.fileSize:
909 self.fp.seek(self.fp.tell() + neededsize)
912 self.fp.seek(self.fp.tell() + neededsize)
910 else:
913 else:
911 self.fp.seek(self.fp.tell() - neededsize)
914 self.fp.seek(self.fp.tell() - neededsize)
912 break
915 break
913
916
914 # csize = self.fileSize - self.fp.tell()
917 # csize = self.fileSize - self.fp.tell()
915 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
918 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
916 # factor = int(csize/neededsize)
919 # factor = int(csize/neededsize)
917 # if factor > 0:
920 # if factor > 0:
918 # self.fp.seek(self.fp.tell() + factor*neededsize)
921 # self.fp.seek(self.fp.tell() + factor*neededsize)
919
922
920 self.flagIsNewFile = 0
923 self.flagIsNewFile = 0
921 self.__isFirstTimeOnline = 0
924 self.__isFirstTimeOnline = 0
922
925
923 def __setNewBlock(self):
926 def __setNewBlock(self):
924
927
925 if self.fp == None:
928 if self.fp == None:
926 return 0
929 return 0
927
930
928 # if self.online:
931 # if self.online:
929 # self.__jumpToLastBlock()
932 # self.__jumpToLastBlock()
930
933
931 if self.flagIsNewFile:
934 if self.flagIsNewFile:
932 self.lastUTTime = self.basicHeaderObj.utc
935 self.lastUTTime = self.basicHeaderObj.utc
933 return 1
936 return 1
934
937
935 if self.realtime:
938 if self.realtime:
936 self.flagDiscontinuousBlock = 1
939 self.flagDiscontinuousBlock = 1
937 if not(self.setNextFile()):
940 if not(self.setNextFile()):
938 return 0
941 return 0
939 else:
942 else:
940 return 1
943 return 1
941
944
942 currentSize = self.fileSize - self.fp.tell()
945 currentSize = self.fileSize - self.fp.tell()
943 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
946 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
944
947
945 if (currentSize >= neededSize):
948 if (currentSize >= neededSize):
946 self.basicHeaderObj.read(self.fp)
949 self.basicHeaderObj.read(self.fp)
947 self.lastUTTime = self.basicHeaderObj.utc
950 self.lastUTTime = self.basicHeaderObj.utc
948 return 1
951 return 1
949
952
950 if self.__waitNewBlock():
953 if self.__waitNewBlock():
951 self.lastUTTime = self.basicHeaderObj.utc
954 self.lastUTTime = self.basicHeaderObj.utc
952 return 1
955 return 1
953
956
954 if not(self.setNextFile()):
957 if not(self.setNextFile()):
955 return 0
958 return 0
956
959
957 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
960 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
958 self.lastUTTime = self.basicHeaderObj.utc
961 self.lastUTTime = self.basicHeaderObj.utc
959
962
960 self.flagDiscontinuousBlock = 0
963 self.flagDiscontinuousBlock = 0
961
964
962 if deltaTime > self.maxTimeStep:
965 if deltaTime > self.maxTimeStep:
963 self.flagDiscontinuousBlock = 1
966 self.flagDiscontinuousBlock = 1
964
967
965 return 1
968 return 1
966
969
967 def readNextBlock(self):
970 def readNextBlock(self):
968
971
969 if not(self.__setNewBlock()):
972 if not(self.__setNewBlock()):
970 return 0
973 return 0
971
974
972 if not(self.readBlock()):
975 if not(self.readBlock()):
973 return 0
976 return 0
974
977
975 self.getBasicHeader()
978 self.getBasicHeader()
976
979
977 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
980 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
978 self.processingHeaderObj.dataBlocksPerFile,
981 self.processingHeaderObj.dataBlocksPerFile,
979 self.dataOut.datatime.ctime())
982 self.dataOut.datatime.ctime())
980 return 1
983 return 1
981
984
982 def __readFirstHeader(self):
985 def __readFirstHeader(self):
983
986
984 self.basicHeaderObj.read(self.fp)
987 self.basicHeaderObj.read(self.fp)
985 self.systemHeaderObj.read(self.fp)
988 self.systemHeaderObj.read(self.fp)
986 self.radarControllerHeaderObj.read(self.fp)
989 self.radarControllerHeaderObj.read(self.fp)
987 self.processingHeaderObj.read(self.fp)
990 self.processingHeaderObj.read(self.fp)
988
991
989 self.firstHeaderSize = self.basicHeaderObj.size
992 self.firstHeaderSize = self.basicHeaderObj.size
990
993
991 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
994 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
992 if datatype == 0:
995 if datatype == 0:
993 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
996 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
994 elif datatype == 1:
997 elif datatype == 1:
995 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
998 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
996 elif datatype == 2:
999 elif datatype == 2:
997 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1000 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
998 elif datatype == 3:
1001 elif datatype == 3:
999 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1002 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1000 elif datatype == 4:
1003 elif datatype == 4:
1001 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1004 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1002 elif datatype == 5:
1005 elif datatype == 5:
1003 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1006 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1004 else:
1007 else:
1005 raise ValueError, 'Data type was not defined'
1008 raise ValueError, 'Data type was not defined'
1006
1009
1007 self.dtype = datatype_str
1010 self.dtype = datatype_str
1008 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1011 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1009 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1012 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1010 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1013 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1011 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1014 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1012 self.getBlockDimension()
1015 self.getBlockDimension()
1013
1016
1014 def __verifyFile(self, filename, msgFlag=True):
1017 def __verifyFile(self, filename, msgFlag=True):
1015
1018
1016 msg = None
1019 msg = None
1017
1020
1018 try:
1021 try:
1019 fp = open(filename, 'rb')
1022 fp = open(filename, 'rb')
1020 except IOError:
1023 except IOError:
1021
1024
1022 if msgFlag:
1025 if msgFlag:
1023 print "[Reading] File %s can't be opened" % (filename)
1026 print "[Reading] File %s can't be opened" % (filename)
1024
1027
1025 return False
1028 return False
1026
1029
1027 currentPosition = fp.tell()
1030 currentPosition = fp.tell()
1028 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1031 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1029
1032
1030 if neededSize == 0:
1033 if neededSize == 0:
1031 basicHeaderObj = BasicHeader(LOCALTIME)
1034 basicHeaderObj = BasicHeader(LOCALTIME)
1032 systemHeaderObj = SystemHeader()
1035 systemHeaderObj = SystemHeader()
1033 radarControllerHeaderObj = RadarControllerHeader()
1036 radarControllerHeaderObj = RadarControllerHeader()
1034 processingHeaderObj = ProcessingHeader()
1037 processingHeaderObj = ProcessingHeader()
1035
1038
1036 if not( basicHeaderObj.read(fp) ):
1039 if not( basicHeaderObj.read(fp) ):
1037 fp.close()
1040 fp.close()
1038 return False
1041 return False
1039
1042
1040 if not( systemHeaderObj.read(fp) ):
1043 if not( systemHeaderObj.read(fp) ):
1041 fp.close()
1044 fp.close()
1042 return False
1045 return False
1043
1046
1044 if not( radarControllerHeaderObj.read(fp) ):
1047 if not( radarControllerHeaderObj.read(fp) ):
1045 fp.close()
1048 fp.close()
1046 return False
1049 return False
1047
1050
1048 if not( processingHeaderObj.read(fp) ):
1051 if not( processingHeaderObj.read(fp) ):
1049 fp.close()
1052 fp.close()
1050 return False
1053 return False
1051
1054
1052 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1055 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1053 else:
1056 else:
1054 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1057 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1055
1058
1056 fp.close()
1059 fp.close()
1057
1060
1058 fileSize = os.path.getsize(filename)
1061 fileSize = os.path.getsize(filename)
1059 currentSize = fileSize - currentPosition
1062 currentSize = fileSize - currentPosition
1060
1063
1061 if currentSize < neededSize:
1064 if currentSize < neededSize:
1062 if msgFlag and (msg != None):
1065 if msgFlag and (msg != None):
1063 print msg
1066 print msg
1064 return False
1067 return False
1065
1068
1066 return True
1069 return True
1067
1070
1068 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1071 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1069
1072
1070 dateList = []
1073 dateList = []
1071 pathList = []
1074 pathList = []
1072
1075
1073 multi_path = path.split(',')
1076 multi_path = path.split(',')
1074
1077
1075 if not walk:
1078 if not walk:
1076
1079
1077 for single_path in multi_path:
1080 for single_path in multi_path:
1078
1081
1079 if not os.path.isdir(single_path):
1082 if not os.path.isdir(single_path):
1080 continue
1083 continue
1081
1084
1082 fileList = glob.glob1(single_path, "*"+ext)
1085 fileList = glob.glob1(single_path, "*"+ext)
1083
1086
1084 for thisFile in fileList:
1087 for thisFile in fileList:
1085
1088
1086 if not os.path.isfile(os.path.join(single_path, thisFile)):
1089 if not os.path.isfile(os.path.join(single_path, thisFile)):
1087 continue
1090 continue
1088
1091
1089 if not isRadarFile(thisFile):
1092 if not isRadarFile(thisFile):
1090 continue
1093 continue
1091
1094
1092 if not isFileInDateRange(thisFile, startDate, endDate):
1095 if not isFileInDateRange(thisFile, startDate, endDate):
1093 continue
1096 continue
1094
1097
1095 thisDate = getDateFromRadarFile(thisFile)
1098 thisDate = getDateFromRadarFile(thisFile)
1096
1099
1097 if thisDate in dateList:
1100 if thisDate in dateList:
1098 continue
1101 continue
1099
1102
1100 dateList.append(thisDate)
1103 dateList.append(thisDate)
1101 pathList.append(single_path)
1104 pathList.append(single_path)
1102
1105
1103 else:
1106 else:
1104 for single_path in multi_path:
1107 for single_path in multi_path:
1105
1108
1106 if not os.path.isdir(single_path):
1109 if not os.path.isdir(single_path):
1107 continue
1110 continue
1108
1111
1109 dirList = []
1112 dirList = []
1110
1113
1111 for thisPath in os.listdir(single_path):
1114 for thisPath in os.listdir(single_path):
1112
1115
1113 if not os.path.isdir(os.path.join(single_path,thisPath)):
1116 if not os.path.isdir(os.path.join(single_path,thisPath)):
1114 continue
1117 continue
1115
1118
1116 if not isRadarFolder(thisPath):
1119 if not isRadarFolder(thisPath):
1117 continue
1120 continue
1118
1121
1119 if not isFolderInDateRange(thisPath, startDate, endDate):
1122 if not isFolderInDateRange(thisPath, startDate, endDate):
1120 continue
1123 continue
1121
1124
1122 dirList.append(thisPath)
1125 dirList.append(thisPath)
1123
1126
1124 if not dirList:
1127 if not dirList:
1125 continue
1128 continue
1126
1129
1127 for thisDir in dirList:
1130 for thisDir in dirList:
1128
1131
1129 datapath = os.path.join(single_path, thisDir, expLabel)
1132 datapath = os.path.join(single_path, thisDir, expLabel)
1130 fileList = glob.glob1(datapath, "*"+ext)
1133 fileList = glob.glob1(datapath, "*"+ext)
1131
1134
1132 if len(fileList) < 1:
1135 if len(fileList) < 1:
1133 continue
1136 continue
1134
1137
1135 thisDate = getDateFromRadarFolder(thisDir)
1138 thisDate = getDateFromRadarFolder(thisDir)
1136
1139
1137 pathList.append(datapath)
1140 pathList.append(datapath)
1138 dateList.append(thisDate)
1141 dateList.append(thisDate)
1139
1142
1140 dateList.sort()
1143 dateList.sort()
1141
1144
1142 if walk:
1145 if walk:
1143 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1146 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1144 else:
1147 else:
1145 pattern_path = multi_path[0]
1148 pattern_path = multi_path[0]
1146
1149
1147 if not dateList:
1150 if not dateList:
1148 print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
1151 print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
1149
1152
1150 if include_path:
1153 if include_path:
1151 return dateList, pathList
1154 return dateList, pathList
1152
1155
1153 return dateList
1156 return dateList
1154
1157
1155 def setup(self,
1158 def setup(self,
1156 path=None,
1159 path=None,
1157 startDate=None,
1160 startDate=None,
1158 endDate=None,
1161 endDate=None,
1159 startTime=datetime.time(0,0,0),
1162 startTime=datetime.time(0,0,0),
1160 endTime=datetime.time(23,59,59),
1163 endTime=datetime.time(23,59,59),
1161 set=None,
1164 set=None,
1162 expLabel = "",
1165 expLabel = "",
1163 ext = None,
1166 ext = None,
1164 online = False,
1167 online = False,
1165 delay = 60,
1168 delay = 60,
1166 walk = True,
1169 walk = True,
1167 getblock = False,
1170 getblock = False,
1168 nTxs = 1,
1171 nTxs = 1,
1169 realtime=False):
1172 realtime=False):
1170
1173
1171 if path == None:
1174 if path == None:
1172 raise ValueError, "[Reading] The path is not valid"
1175 raise ValueError, "[Reading] The path is not valid"
1173
1176
1174 if ext == None:
1177 if ext == None:
1175 ext = self.ext
1178 ext = self.ext
1176
1179
1177 if online:
1180 if online:
1178 print "[Reading] Searching files in online mode..."
1181 print "[Reading] Searching files in online mode..."
1179
1182
1180 for nTries in range( self.nTries ):
1183 for nTries in range( self.nTries ):
1181 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1184 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1182
1185
1183 if fullpath:
1186 if fullpath:
1184 break
1187 break
1185
1188
1186 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1189 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1187 sleep( self.delay )
1190 sleep( self.delay )
1188
1191
1189 if not(fullpath):
1192 if not(fullpath):
1190 print "[Reading] There 'isn't any valid file in %s" % path
1193 print "[Reading] There 'isn't any valid file in %s" % path
1191 return
1194 return
1192
1195
1193 self.year = year
1196 self.year = year
1194 self.doy = doy
1197 self.doy = doy
1195 self.set = set - 1
1198 self.set = set - 1
1196 self.path = path
1199 self.path = path
1197 self.foldercounter = foldercounter
1200 self.foldercounter = foldercounter
1198 last_set = None
1201 last_set = None
1199
1202
1200 else:
1203 else:
1201 print "[Reading] Searching files in offline mode ..."
1204 print "[Reading] Searching files in offline mode ..."
1202 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1205 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1203 startTime=startTime, endTime=endTime,
1206 startTime=startTime, endTime=endTime,
1204 set=set, expLabel=expLabel, ext=ext,
1207 set=set, expLabel=expLabel, ext=ext,
1205 walk=walk)
1208 walk=walk)
1206
1209
1207 if not(pathList):
1210 if not(pathList):
1208 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1211 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1209 # datetime.datetime.combine(startDate,startTime).ctime(),
1212 # datetime.datetime.combine(startDate,startTime).ctime(),
1210 # datetime.datetime.combine(endDate,endTime).ctime())
1213 # datetime.datetime.combine(endDate,endTime).ctime())
1211
1214
1212 # sys.exit(-1)
1215 # sys.exit(-1)
1213
1216
1214 self.fileIndex = -1
1217 self.fileIndex = -1
1215 self.pathList = []
1218 self.pathList = []
1216 self.filenameList = []
1219 self.filenameList = []
1217 return
1220 return
1218
1221
1219 self.fileIndex = -1
1222 self.fileIndex = -1
1220 self.pathList = pathList
1223 self.pathList = pathList
1221 self.filenameList = filenameList
1224 self.filenameList = filenameList
1222 file_name = os.path.basename(filenameList[-1])
1225 file_name = os.path.basename(filenameList[-1])
1223 basename, ext = os.path.splitext(file_name)
1226 basename, ext = os.path.splitext(file_name)
1224 last_set = int(basename[-3:])
1227 last_set = int(basename[-3:])
1225
1228
1226 self.online = online
1229 self.online = online
1227 self.realtime = realtime
1230 self.realtime = realtime
1228 self.delay = delay
1231 self.delay = delay
1229 ext = ext.lower()
1232 ext = ext.lower()
1230 self.ext = ext
1233 self.ext = ext
1231 self.getByBlock = getblock
1234 self.getByBlock = getblock
1232 self.nTxs = int(nTxs)
1235 self.nTxs = int(nTxs)
1233
1236
1234 if not(self.setNextFile()):
1237 if not(self.setNextFile()):
1235 if (startDate!=None) and (endDate!=None):
1238 if (startDate!=None) and (endDate!=None):
1236 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1239 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1237 elif startDate != None:
1240 elif startDate != None:
1238 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1241 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1239 else:
1242 else:
1240 print "[Reading] No files"
1243 print "[Reading] No files"
1241
1244
1242 self.fileIndex = -1
1245 self.fileIndex = -1
1243 self.pathList = []
1246 self.pathList = []
1244 self.filenameList = []
1247 self.filenameList = []
1245 return
1248 return
1246
1249
1247 # self.getBasicHeader()
1250 # self.getBasicHeader()
1248
1251
1249 if last_set != None:
1252 if last_set != None:
1250 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1253 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1251 return
1254 return
1252
1255
1253 def getBasicHeader(self):
1256 def getBasicHeader(self):
1254
1257
1255 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1258 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1256
1259
1257 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1260 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1258
1261
1259 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1262 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1260
1263
1261 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1264 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1262
1265
1263 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1266 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1264
1267
1265 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1268 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1266
1269
1267 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1270 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1268
1271
1269 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1272 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1270
1273
1271
1274
1272 def getFirstHeader(self):
1275 def getFirstHeader(self):
1273
1276
1274 raise NotImplementedError
1277 raise NotImplementedError
1275
1278
1276 def getData(self):
1279 def getData(self):
1277
1280
1278 raise NotImplementedError
1281 raise NotImplementedError
1279
1282
1280 def hasNotDataInBuffer(self):
1283 def hasNotDataInBuffer(self):
1281
1284
1282 raise NotImplementedError
1285 raise NotImplementedError
1283
1286
1284 def readBlock(self):
1287 def readBlock(self):
1285
1288
1286 raise NotImplementedError
1289 raise NotImplementedError
1287
1290
1288 def isEndProcess(self):
1291 def isEndProcess(self):
1289
1292
1290 return self.flagNoMoreFiles
1293 return self.flagNoMoreFiles
1291
1294
1292 def printReadBlocks(self):
1295 def printReadBlocks(self):
1293
1296
1294 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1297 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1295
1298
1296 def printTotalBlocks(self):
1299 def printTotalBlocks(self):
1297
1300
1298 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1301 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1299
1302
1300 def printNumberOfBlock(self):
1303 def printNumberOfBlock(self):
1301
1304
1302 if self.flagIsNewBlock:
1305 if self.flagIsNewBlock:
1303 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1306 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1304 self.processingHeaderObj.dataBlocksPerFile,
1307 self.processingHeaderObj.dataBlocksPerFile,
1305 self.dataOut.datatime.ctime())
1308 self.dataOut.datatime.ctime())
1306
1309
1307 def printInfo(self):
1310 def printInfo(self):
1308
1311
1309 if self.__printInfo == False:
1312 if self.__printInfo == False:
1310 return
1313 return
1311
1314
1312 self.basicHeaderObj.printInfo()
1315 self.basicHeaderObj.printInfo()
1313 self.systemHeaderObj.printInfo()
1316 self.systemHeaderObj.printInfo()
1314 self.radarControllerHeaderObj.printInfo()
1317 self.radarControllerHeaderObj.printInfo()
1315 self.processingHeaderObj.printInfo()
1318 self.processingHeaderObj.printInfo()
1316
1319
1317 self.__printInfo = False
1320 self.__printInfo = False
1318
1321
1319
1322
1320 def run(self, **kwargs):
1323 def run(self, **kwargs):
1321
1324
1322 if not(self.isConfig):
1325 if not(self.isConfig):
1323
1326
1324 # self.dataOut = dataOut
1327 # self.dataOut = dataOut
1325 self.setup(**kwargs)
1328 self.setup(**kwargs)
1326 self.isConfig = True
1329 self.isConfig = True
1327
1330
1328 self.getData()
1331 self.getData()
1329
1332
1330 class JRODataWriter(JRODataIO):
1333 class JRODataWriter(JRODataIO):
1331
1334
1332 """
1335 """
1333 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1336 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1334 de los datos siempre se realiza por bloques.
1337 de los datos siempre se realiza por bloques.
1335 """
1338 """
1336
1339
1337 blockIndex = 0
1340 blockIndex = 0
1338
1341
1339 path = None
1342 path = None
1340
1343
1341 setFile = None
1344 setFile = None
1342
1345
1343 profilesPerBlock = None
1346 profilesPerBlock = None
1344
1347
1345 blocksPerFile = None
1348 blocksPerFile = None
1346
1349
1347 nWriteBlocks = 0
1350 nWriteBlocks = 0
1348
1351
1349 fileDate = None
1352 fileDate = None
1350
1353
1351 def __init__(self, dataOut=None):
1354 def __init__(self, dataOut=None):
1352 raise NotImplementedError
1355 raise NotImplementedError
1353
1356
1354
1357
1355 def hasAllDataInBuffer(self):
1358 def hasAllDataInBuffer(self):
1356 raise NotImplementedError
1359 raise NotImplementedError
1357
1360
1358
1361
1359 def setBlockDimension(self):
1362 def setBlockDimension(self):
1360 raise NotImplementedError
1363 raise NotImplementedError
1361
1364
1362
1365
1363 def writeBlock(self):
1366 def writeBlock(self):
1364 raise NotImplementedError
1367 raise NotImplementedError
1365
1368
1366
1369
1367 def putData(self):
1370 def putData(self):
1368 raise NotImplementedError
1371 raise NotImplementedError
1369
1372
1370
1373
1371 def getProcessFlags(self):
1374 def getProcessFlags(self):
1372
1375
1373 processFlags = 0
1376 processFlags = 0
1374
1377
1375 dtype_index = get_dtype_index(self.dtype)
1378 dtype_index = get_dtype_index(self.dtype)
1376 procflag_dtype = get_procflag_dtype(dtype_index)
1379 procflag_dtype = get_procflag_dtype(dtype_index)
1377
1380
1378 processFlags += procflag_dtype
1381 processFlags += procflag_dtype
1379
1382
1380 if self.dataOut.flagDecodeData:
1383 if self.dataOut.flagDecodeData:
1381 processFlags += PROCFLAG.DECODE_DATA
1384 processFlags += PROCFLAG.DECODE_DATA
1382
1385
1383 if self.dataOut.flagDeflipData:
1386 if self.dataOut.flagDeflipData:
1384 processFlags += PROCFLAG.DEFLIP_DATA
1387 processFlags += PROCFLAG.DEFLIP_DATA
1385
1388
1386 if self.dataOut.code is not None:
1389 if self.dataOut.code is not None:
1387 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1390 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1388
1391
1389 if self.dataOut.nCohInt > 1:
1392 if self.dataOut.nCohInt > 1:
1390 processFlags += PROCFLAG.COHERENT_INTEGRATION
1393 processFlags += PROCFLAG.COHERENT_INTEGRATION
1391
1394
1392 if self.dataOut.type == "Spectra":
1395 if self.dataOut.type == "Spectra":
1393 if self.dataOut.nIncohInt > 1:
1396 if self.dataOut.nIncohInt > 1:
1394 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1397 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1395
1398
1396 if self.dataOut.data_dc is not None:
1399 if self.dataOut.data_dc is not None:
1397 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1400 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1398
1401
1399 if self.dataOut.flagShiftFFT:
1402 if self.dataOut.flagShiftFFT:
1400 processFlags += PROCFLAG.SHIFT_FFT_DATA
1403 processFlags += PROCFLAG.SHIFT_FFT_DATA
1401
1404
1402 return processFlags
1405 return processFlags
1403
1406
1404 def setBasicHeader(self):
1407 def setBasicHeader(self):
1405
1408
1406 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1409 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1407 self.basicHeaderObj.version = self.versionFile
1410 self.basicHeaderObj.version = self.versionFile
1408 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1411 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1409
1412
1410 utc = numpy.floor(self.dataOut.utctime)
1413 utc = numpy.floor(self.dataOut.utctime)
1411 milisecond = (self.dataOut.utctime - utc)* 1000.0
1414 milisecond = (self.dataOut.utctime - utc)* 1000.0
1412
1415
1413 self.basicHeaderObj.utc = utc
1416 self.basicHeaderObj.utc = utc
1414 self.basicHeaderObj.miliSecond = milisecond
1417 self.basicHeaderObj.miliSecond = milisecond
1415 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1418 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1416 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1419 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1417 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1420 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1418
1421
1419 def setFirstHeader(self):
1422 def setFirstHeader(self):
1420 """
1423 """
1421 Obtiene una copia del First Header
1424 Obtiene una copia del First Header
1422
1425
1423 Affected:
1426 Affected:
1424
1427
1425 self.basicHeaderObj
1428 self.basicHeaderObj
1426 self.systemHeaderObj
1429 self.systemHeaderObj
1427 self.radarControllerHeaderObj
1430 self.radarControllerHeaderObj
1428 self.processingHeaderObj self.
1431 self.processingHeaderObj self.
1429
1432
1430 Return:
1433 Return:
1431 None
1434 None
1432 """
1435 """
1433
1436
1434 raise NotImplementedError
1437 raise NotImplementedError
1435
1438
1436 def __writeFirstHeader(self):
1439 def __writeFirstHeader(self):
1437 """
1440 """
1438 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1441 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1439
1442
1440 Affected:
1443 Affected:
1441 __dataType
1444 __dataType
1442
1445
1443 Return:
1446 Return:
1444 None
1447 None
1445 """
1448 """
1446
1449
1447 # CALCULAR PARAMETROS
1450 # CALCULAR PARAMETROS
1448
1451
1449 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1452 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1450 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1453 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1451
1454
1452 self.basicHeaderObj.write(self.fp)
1455 self.basicHeaderObj.write(self.fp)
1453 self.systemHeaderObj.write(self.fp)
1456 self.systemHeaderObj.write(self.fp)
1454 self.radarControllerHeaderObj.write(self.fp)
1457 self.radarControllerHeaderObj.write(self.fp)
1455 self.processingHeaderObj.write(self.fp)
1458 self.processingHeaderObj.write(self.fp)
1456
1459
1457 def __setNewBlock(self):
1460 def __setNewBlock(self):
1458 """
1461 """
1459 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1462 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1460
1463
1461 Return:
1464 Return:
1462 0 : si no pudo escribir nada
1465 0 : si no pudo escribir nada
1463 1 : Si escribio el Basic el First Header
1466 1 : Si escribio el Basic el First Header
1464 """
1467 """
1465 if self.fp == None:
1468 if self.fp == None:
1466 self.setNextFile()
1469 self.setNextFile()
1467
1470
1468 if self.flagIsNewFile:
1471 if self.flagIsNewFile:
1469 return 1
1472 return 1
1470
1473
1471 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1474 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1472 self.basicHeaderObj.write(self.fp)
1475 self.basicHeaderObj.write(self.fp)
1473 return 1
1476 return 1
1474
1477
1475 if not( self.setNextFile() ):
1478 if not( self.setNextFile() ):
1476 return 0
1479 return 0
1477
1480
1478 return 1
1481 return 1
1479
1482
1480
1483
1481 def writeNextBlock(self):
1484 def writeNextBlock(self):
1482 """
1485 """
1483 Selecciona el bloque siguiente de datos y los escribe en un file
1486 Selecciona el bloque siguiente de datos y los escribe en un file
1484
1487
1485 Return:
1488 Return:
1486 0 : Si no hizo pudo escribir el bloque de datos
1489 0 : Si no hizo pudo escribir el bloque de datos
1487 1 : Si no pudo escribir el bloque de datos
1490 1 : Si no pudo escribir el bloque de datos
1488 """
1491 """
1489 if not( self.__setNewBlock() ):
1492 if not( self.__setNewBlock() ):
1490 return 0
1493 return 0
1491
1494
1492 self.writeBlock()
1495 self.writeBlock()
1493
1496
1494 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1497 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1495 self.processingHeaderObj.dataBlocksPerFile)
1498 self.processingHeaderObj.dataBlocksPerFile)
1496
1499
1497 return 1
1500 return 1
1498
1501
1499 def setNextFile(self):
1502 def setNextFile(self):
1500 """
1503 """
1501 Determina el siguiente file que sera escrito
1504 Determina el siguiente file que sera escrito
1502
1505
1503 Affected:
1506 Affected:
1504 self.filename
1507 self.filename
1505 self.subfolder
1508 self.subfolder
1506 self.fp
1509 self.fp
1507 self.setFile
1510 self.setFile
1508 self.flagIsNewFile
1511 self.flagIsNewFile
1509
1512
1510 Return:
1513 Return:
1511 0 : Si el archivo no puede ser escrito
1514 0 : Si el archivo no puede ser escrito
1512 1 : Si el archivo esta listo para ser escrito
1515 1 : Si el archivo esta listo para ser escrito
1513 """
1516 """
1514 ext = self.ext
1517 ext = self.ext
1515 path = self.path
1518 path = self.path
1516
1519
1517 if self.fp != None:
1520 if self.fp != None:
1518 self.fp.close()
1521 self.fp.close()
1519
1522
1520 timeTuple = time.localtime( self.dataOut.utctime)
1523 timeTuple = time.localtime( self.dataOut.utctime)
1521 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1524 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1522
1525
1523 fullpath = os.path.join( path, subfolder )
1526 fullpath = os.path.join( path, subfolder )
1524 setFile = self.setFile
1527 setFile = self.setFile
1525
1528
1526 if not( os.path.exists(fullpath) ):
1529 if not( os.path.exists(fullpath) ):
1527 os.mkdir(fullpath)
1530 os.mkdir(fullpath)
1528 setFile = -1 #inicializo mi contador de seteo
1531 setFile = -1 #inicializo mi contador de seteo
1529 else:
1532 else:
1530 filesList = os.listdir( fullpath )
1533 filesList = os.listdir( fullpath )
1531 if len( filesList ) > 0:
1534 if len( filesList ) > 0:
1532 filesList = sorted( filesList, key=str.lower )
1535 filesList = sorted( filesList, key=str.lower )
1533 filen = filesList[-1]
1536 filen = filesList[-1]
1534 # el filename debera tener el siguiente formato
1537 # el filename debera tener el siguiente formato
1535 # 0 1234 567 89A BCDE (hex)
1538 # 0 1234 567 89A BCDE (hex)
1536 # x YYYY DDD SSS .ext
1539 # x YYYY DDD SSS .ext
1537 if isNumber( filen[8:11] ):
1540 if isNumber( filen[8:11] ):
1538 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1541 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1539 else:
1542 else:
1540 setFile = -1
1543 setFile = -1
1541 else:
1544 else:
1542 setFile = -1 #inicializo mi contador de seteo
1545 setFile = -1 #inicializo mi contador de seteo
1543
1546
1544 setFile += 1
1547 setFile += 1
1545
1548
1546 #If this is a new day it resets some values
1549 #If this is a new day it resets some values
1547 if self.dataOut.datatime.date() > self.fileDate:
1550 if self.dataOut.datatime.date() > self.fileDate:
1548 setFile = 0
1551 setFile = 0
1549 self.nTotalBlocks = 0
1552 self.nTotalBlocks = 0
1550
1553
1551 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1554 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1552
1555
1553 filename = os.path.join( path, subfolder, filen )
1556 filename = os.path.join( path, subfolder, filen )
1554
1557
1555 fp = open( filename,'wb' )
1558 fp = open( filename,'wb' )
1556
1559
1557 self.blockIndex = 0
1560 self.blockIndex = 0
1558
1561
1559 #guardando atributos
1562 #guardando atributos
1560 self.filename = filename
1563 self.filename = filename
1561 self.subfolder = subfolder
1564 self.subfolder = subfolder
1562 self.fp = fp
1565 self.fp = fp
1563 self.setFile = setFile
1566 self.setFile = setFile
1564 self.flagIsNewFile = 1
1567 self.flagIsNewFile = 1
1565 self.fileDate = self.dataOut.datatime.date()
1568 self.fileDate = self.dataOut.datatime.date()
1566
1569
1567 self.setFirstHeader()
1570 self.setFirstHeader()
1568
1571
1569 print '[Writing] Opening file: %s'%self.filename
1572 print '[Writing] Opening file: %s'%self.filename
1570
1573
1571 self.__writeFirstHeader()
1574 self.__writeFirstHeader()
1572
1575
1573 return 1
1576 return 1
1574
1577
1575 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1578 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1576 """
1579 """
1577 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1580 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1578
1581
1579 Inputs:
1582 Inputs:
1580 path : directory where data will be saved
1583 path : directory where data will be saved
1581 profilesPerBlock : number of profiles per block
1584 profilesPerBlock : number of profiles per block
1582 set : initial file set
1585 set : initial file set
1583 datatype : An integer number that defines data type:
1586 datatype : An integer number that defines data type:
1584 0 : int8 (1 byte)
1587 0 : int8 (1 byte)
1585 1 : int16 (2 bytes)
1588 1 : int16 (2 bytes)
1586 2 : int32 (4 bytes)
1589 2 : int32 (4 bytes)
1587 3 : int64 (8 bytes)
1590 3 : int64 (8 bytes)
1588 4 : float32 (4 bytes)
1591 4 : float32 (4 bytes)
1589 5 : double64 (8 bytes)
1592 5 : double64 (8 bytes)
1590
1593
1591 Return:
1594 Return:
1592 0 : Si no realizo un buen seteo
1595 0 : Si no realizo un buen seteo
1593 1 : Si realizo un buen seteo
1596 1 : Si realizo un buen seteo
1594 """
1597 """
1595
1598
1596 if ext == None:
1599 if ext == None:
1597 ext = self.ext
1600 ext = self.ext
1598
1601
1599 self.ext = ext.lower()
1602 self.ext = ext.lower()
1600
1603
1601 self.path = path
1604 self.path = path
1602
1605
1603 if set is None:
1606 if set is None:
1604 self.setFile = -1
1607 self.setFile = -1
1605 else:
1608 else:
1606 self.setFile = set - 1
1609 self.setFile = set - 1
1607
1610
1608 self.blocksPerFile = blocksPerFile
1611 self.blocksPerFile = blocksPerFile
1609
1612
1610 self.profilesPerBlock = profilesPerBlock
1613 self.profilesPerBlock = profilesPerBlock
1611
1614
1612 self.dataOut = dataOut
1615 self.dataOut = dataOut
1613 self.fileDate = self.dataOut.datatime.date()
1616 self.fileDate = self.dataOut.datatime.date()
1614 #By default
1617 #By default
1615 self.dtype = self.dataOut.dtype
1618 self.dtype = self.dataOut.dtype
1616
1619
1617 if datatype is not None:
1620 if datatype is not None:
1618 self.dtype = get_numpy_dtype(datatype)
1621 self.dtype = get_numpy_dtype(datatype)
1619
1622
1620 if not(self.setNextFile()):
1623 if not(self.setNextFile()):
1621 print "[Writing] There isn't a next file"
1624 print "[Writing] There isn't a next file"
1622 return 0
1625 return 0
1623
1626
1624 self.setBlockDimension()
1627 self.setBlockDimension()
1625
1628
1626 return 1
1629 return 1
1627
1630
1628 def run(self, dataOut, **kwargs):
1631 def run(self, dataOut, **kwargs):
1629
1632
1630 if not(self.isConfig):
1633 if not(self.isConfig):
1631
1634
1632 self.setup(dataOut, **kwargs)
1635 self.setup(dataOut, **kwargs)
1633 self.isConfig = True
1636 self.isConfig = True
1634
1637
1635 self.putData()
1638 self.putData()
1636
1639
General Comments 0
You need to be logged in to leave comments. Login now