##// END OF EJS Templates
schain
RSS
Data directories are sorted before reading files
Miguel Valdez -
r726:a766e7f85e72
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1639 +1,1646
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 print "The file %s can't be opened" %(filename)
71 print "The file %s can't be opened" %(filename)
72 return 0
72 return 0
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
86 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
90 Inputs:
90 Inputs:
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92
92
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
93 startDate : fecha inicial del rango seleccionado en formato datetime.date
94
94
95 endDate : fecha final del rango seleccionado en formato datetime.date
95 endDate : fecha final del rango seleccionado en formato datetime.date
96
96
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
97 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98
98
99 endTime : tiempo final del rango seleccionado en formato datetime.time
99 endTime : tiempo final del rango seleccionado en formato datetime.time
100
100
101 Return:
101 Return:
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
102 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 fecha especificado, de lo contrario retorna False.
103 fecha especificado, de lo contrario retorna False.
104
104
105 Excepciones:
105 Excepciones:
106 Si el archivo no existe o no puede ser abierto
106 Si el archivo no existe o no puede ser abierto
107 Si la cabecera no puede ser leida.
107 Si la cabecera no puede ser leida.
108
108
109 """
109 """
110
110
111
111
112 try:
112 try:
113 fp = open(filename,'rb')
113 fp = open(filename,'rb')
114 except IOError:
114 except IOError:
115 print "The file %s can't be opened" %(filename)
115 print "The file %s can't be opened" %(filename)
116 return None
116 return None
117
117
118 basicHeaderObj = BasicHeader(LOCALTIME)
118 basicHeaderObj = BasicHeader(LOCALTIME)
119 sts = basicHeaderObj.read(fp)
119 sts = basicHeaderObj.read(fp)
120 fp.close()
120 fp.close()
121
121
122 thisDatetime = basicHeaderObj.datatime
122 thisDatetime = basicHeaderObj.datatime
123 thisDate = thisDatetime.date()
123 thisDate = thisDatetime.date()
124 thisTime = thisDatetime.time()
124 thisTime = thisDatetime.time()
125
125
126 if not(sts):
126 if not(sts):
127 print "Skipping the file %s because it has not a valid header" %(filename)
127 print "Skipping the file %s because it has not a valid header" %(filename)
128 return None
128 return None
129
129
130 #General case
130 #General case
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
131 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
132 #-----------o----------------------------o-----------
132 #-----------o----------------------------o-----------
133 # startTime endTime
133 # startTime endTime
134
134
135 if endTime >= startTime:
135 if endTime >= startTime:
136 if (thisTime < startTime) or (thisTime > endTime):
136 if (thisTime < startTime) or (thisTime > endTime):
137 return None
137 return None
138
138
139 return thisDatetime
139 return thisDatetime
140
140
141 #If endTime < startTime then endTime belongs to the next day
141 #If endTime < startTime then endTime belongs to the next day
142
142
143
143
144 #<<<<<<<<<<<o o>>>>>>>>>>>
144 #<<<<<<<<<<<o o>>>>>>>>>>>
145 #-----------o----------------------------o-----------
145 #-----------o----------------------------o-----------
146 # endTime startTime
146 # endTime startTime
147
147
148 if (thisDate == startDate) and (thisTime < startTime):
148 if (thisDate == startDate) and (thisTime < startTime):
149 return None
149 return None
150
150
151 if (thisDate == endDate) and (thisTime > endTime):
151 if (thisDate == endDate) and (thisTime > endTime):
152 return None
152 return None
153
153
154 if (thisTime < startTime) and (thisTime > endTime):
154 if (thisTime < startTime) and (thisTime > endTime):
155 return None
155 return None
156
156
157 return thisDatetime
157 return thisDatetime
158
158
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
159 def isFolderInDateRange(folder, startDate=None, endDate=None):
160 """
160 """
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
161 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
162
162
163 Inputs:
163 Inputs:
164 folder : nombre completo del directorio.
164 folder : nombre completo del directorio.
165 Su formato deberia ser "/path_root/?YYYYDDD"
165 Su formato deberia ser "/path_root/?YYYYDDD"
166
166
167 siendo:
167 siendo:
168 YYYY : Anio (ejemplo 2015)
168 YYYY : Anio (ejemplo 2015)
169 DDD : Dia del anio (ejemplo 305)
169 DDD : Dia del anio (ejemplo 305)
170
170
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
171 startDate : fecha inicial del rango seleccionado en formato datetime.date
172
172
173 endDate : fecha final del rango seleccionado en formato datetime.date
173 endDate : fecha final del rango seleccionado en formato datetime.date
174
174
175 Return:
175 Return:
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
176 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
177 fecha especificado, de lo contrario retorna False.
177 fecha especificado, de lo contrario retorna False.
178 Excepciones:
178 Excepciones:
179 Si el directorio no tiene el formato adecuado
179 Si el directorio no tiene el formato adecuado
180 """
180 """
181
181
182 basename = os.path.basename(folder)
182 basename = os.path.basename(folder)
183
183
184 if not isRadarFolder(basename):
184 if not isRadarFolder(basename):
185 print "The folder %s has not the rigth format" %folder
185 print "The folder %s has not the rigth format" %folder
186 return 0
186 return 0
187
187
188 if startDate and endDate:
188 if startDate and endDate:
189 thisDate = getDateFromRadarFolder(basename)
189 thisDate = getDateFromRadarFolder(basename)
190
190
191 if thisDate < startDate:
191 if thisDate < startDate:
192 return 0
192 return 0
193
193
194 if thisDate > endDate:
194 if thisDate > endDate:
195 return 0
195 return 0
196
196
197 return 1
197 return 1
198
198
199 def isFileInDateRange(filename, startDate=None, endDate=None):
199 def isFileInDateRange(filename, startDate=None, endDate=None):
200 """
200 """
201 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
201 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
202
202
203 Inputs:
203 Inputs:
204 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
204 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
205
205
206 Su formato deberia ser "?YYYYDDDsss"
206 Su formato deberia ser "?YYYYDDDsss"
207
207
208 siendo:
208 siendo:
209 YYYY : Anio (ejemplo 2015)
209 YYYY : Anio (ejemplo 2015)
210 DDD : Dia del anio (ejemplo 305)
210 DDD : Dia del anio (ejemplo 305)
211 sss : set
211 sss : set
212
212
213 startDate : fecha inicial del rango seleccionado en formato datetime.date
213 startDate : fecha inicial del rango seleccionado en formato datetime.date
214
214
215 endDate : fecha final del rango seleccionado en formato datetime.date
215 endDate : fecha final del rango seleccionado en formato datetime.date
216
216
217 Return:
217 Return:
218 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
218 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
219 fecha especificado, de lo contrario retorna False.
219 fecha especificado, de lo contrario retorna False.
220 Excepciones:
220 Excepciones:
221 Si el archivo no tiene el formato adecuado
221 Si el archivo no tiene el formato adecuado
222 """
222 """
223
223
224 basename = os.path.basename(filename)
224 basename = os.path.basename(filename)
225
225
226 if not isRadarFile(basename):
226 if not isRadarFile(basename):
227 print "The filename %s has not the rigth format" %filename
227 print "The filename %s has not the rigth format" %filename
228 return 0
228 return 0
229
229
230 if startDate and endDate:
230 if startDate and endDate:
231 thisDate = getDateFromRadarFile(basename)
231 thisDate = getDateFromRadarFile(basename)
232
232
233 if thisDate < startDate:
233 if thisDate < startDate:
234 return 0
234 return 0
235
235
236 if thisDate > endDate:
236 if thisDate > endDate:
237 return 0
237 return 0
238
238
239 return 1
239 return 1
240
240
241 def getFileFromSet(path, ext, set):
241 def getFileFromSet(path, ext, set):
242 validFilelist = []
242 validFilelist = []
243 fileList = os.listdir(path)
243 fileList = os.listdir(path)
244
244
245 # 0 1234 567 89A BCDE
245 # 0 1234 567 89A BCDE
246 # H YYYY DDD SSS .ext
246 # H YYYY DDD SSS .ext
247
247
248 for thisFile in fileList:
248 for thisFile in fileList:
249 try:
249 try:
250 year = int(thisFile[1:5])
250 year = int(thisFile[1:5])
251 doy = int(thisFile[5:8])
251 doy = int(thisFile[5:8])
252 except:
252 except:
253 continue
253 continue
254
254
255 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
255 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
256 continue
256 continue
257
257
258 validFilelist.append(thisFile)
258 validFilelist.append(thisFile)
259
259
260 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
260 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
261
261
262 if len(myfile)!= 0:
262 if len(myfile)!= 0:
263 return myfile[0]
263 return myfile[0]
264 else:
264 else:
265 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
265 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
266 print 'the filename %s does not exist'%filename
266 print 'the filename %s does not exist'%filename
267 print '...going to the last file: '
267 print '...going to the last file: '
268
268
269 if validFilelist:
269 if validFilelist:
270 validFilelist = sorted( validFilelist, key=str.lower )
270 validFilelist = sorted( validFilelist, key=str.lower )
271 return validFilelist[-1]
271 return validFilelist[-1]
272
272
273 return None
273 return None
274
274
275 def getlastFileFromPath(path, ext):
275 def getlastFileFromPath(path, ext):
276 """
276 """
277 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
277 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
278 al final de la depuracion devuelve el ultimo file de la lista que quedo.
278 al final de la depuracion devuelve el ultimo file de la lista que quedo.
279
279
280 Input:
280 Input:
281 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
281 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
282 ext : extension de los files contenidos en una carpeta
282 ext : extension de los files contenidos en una carpeta
283
283
284 Return:
284 Return:
285 El ultimo file de una determinada carpeta, no se considera el path.
285 El ultimo file de una determinada carpeta, no se considera el path.
286 """
286 """
287 validFilelist = []
287 validFilelist = []
288 fileList = os.listdir(path)
288 fileList = os.listdir(path)
289
289
290 # 0 1234 567 89A BCDE
290 # 0 1234 567 89A BCDE
291 # H YYYY DDD SSS .ext
291 # H YYYY DDD SSS .ext
292
292
293 for thisFile in fileList:
293 for thisFile in fileList:
294
294
295 year = thisFile[1:5]
295 year = thisFile[1:5]
296 if not isNumber(year):
296 if not isNumber(year):
297 continue
297 continue
298
298
299 doy = thisFile[5:8]
299 doy = thisFile[5:8]
300 if not isNumber(doy):
300 if not isNumber(doy):
301 continue
301 continue
302
302
303 year = int(year)
303 year = int(year)
304 doy = int(doy)
304 doy = int(doy)
305
305
306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 continue
307 continue
308
308
309 validFilelist.append(thisFile)
309 validFilelist.append(thisFile)
310
310
311 if validFilelist:
311 if validFilelist:
312 validFilelist = sorted( validFilelist, key=str.lower )
312 validFilelist = sorted( validFilelist, key=str.lower )
313 return validFilelist[-1]
313 return validFilelist[-1]
314
314
315 return None
315 return None
316
316
317 def checkForRealPath(path, foldercounter, year, doy, set, ext):
317 def checkForRealPath(path, foldercounter, year, doy, set, ext):
318 """
318 """
319 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
319 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
320 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
320 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
321 el path exacto de un determinado file.
321 el path exacto de un determinado file.
322
322
323 Example :
323 Example :
324 nombre correcto del file es .../.../D2009307/P2009307367.ext
324 nombre correcto del file es .../.../D2009307/P2009307367.ext
325
325
326 Entonces la funcion prueba con las siguientes combinaciones
326 Entonces la funcion prueba con las siguientes combinaciones
327 .../.../y2009307367.ext
327 .../.../y2009307367.ext
328 .../.../Y2009307367.ext
328 .../.../Y2009307367.ext
329 .../.../x2009307/y2009307367.ext
329 .../.../x2009307/y2009307367.ext
330 .../.../x2009307/Y2009307367.ext
330 .../.../x2009307/Y2009307367.ext
331 .../.../X2009307/y2009307367.ext
331 .../.../X2009307/y2009307367.ext
332 .../.../X2009307/Y2009307367.ext
332 .../.../X2009307/Y2009307367.ext
333 siendo para este caso, la ultima combinacion de letras, identica al file buscado
333 siendo para este caso, la ultima combinacion de letras, identica al file buscado
334
334
335 Return:
335 Return:
336 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
336 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
337 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
337 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
338 para el filename
338 para el filename
339 """
339 """
340 fullfilename = None
340 fullfilename = None
341 find_flag = False
341 find_flag = False
342 filename = None
342 filename = None
343
343
344 prefixDirList = [None,'d','D']
344 prefixDirList = [None,'d','D']
345 if ext.lower() == ".r": #voltage
345 if ext.lower() == ".r": #voltage
346 prefixFileList = ['d','D']
346 prefixFileList = ['d','D']
347 elif ext.lower() == ".pdata": #spectra
347 elif ext.lower() == ".pdata": #spectra
348 prefixFileList = ['p','P']
348 prefixFileList = ['p','P']
349 else:
349 else:
350 return None, filename
350 return None, filename
351
351
352 #barrido por las combinaciones posibles
352 #barrido por las combinaciones posibles
353 for prefixDir in prefixDirList:
353 for prefixDir in prefixDirList:
354 thispath = path
354 thispath = path
355 if prefixDir != None:
355 if prefixDir != None:
356 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
356 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
357 if foldercounter == 0:
357 if foldercounter == 0:
358 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
358 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
359 else:
359 else:
360 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
360 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
361 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
361 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
362 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
362 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
363 fullfilename = os.path.join( thispath, filename ) #formo el path completo
363 fullfilename = os.path.join( thispath, filename ) #formo el path completo
364
364
365 if os.path.exists( fullfilename ): #verifico que exista
365 if os.path.exists( fullfilename ): #verifico que exista
366 find_flag = True
366 find_flag = True
367 break
367 break
368 if find_flag:
368 if find_flag:
369 break
369 break
370
370
371 if not(find_flag):
371 if not(find_flag):
372 return None, filename
372 return None, filename
373
373
374 return fullfilename, filename
374 return fullfilename, filename
375
375
376 def isRadarFolder(folder):
376 def isRadarFolder(folder):
377 try:
377 try:
378 year = int(folder[1:5])
378 year = int(folder[1:5])
379 doy = int(folder[5:8])
379 doy = int(folder[5:8])
380 except:
380 except:
381 return 0
381 return 0
382
382
383 return 1
383 return 1
384
384
385 def isRadarFile(file):
385 def isRadarFile(file):
386 try:
386 try:
387 year = int(file[1:5])
387 year = int(file[1:5])
388 doy = int(file[5:8])
388 doy = int(file[5:8])
389 set = int(file[8:11])
389 set = int(file[8:11])
390 except:
390 except:
391 return 0
391 return 0
392
392
393 return 1
393 return 1
394
394
395 def getDateFromRadarFile(file):
395 def getDateFromRadarFile(file):
396 try:
396 try:
397 year = int(file[1:5])
397 year = int(file[1:5])
398 doy = int(file[5:8])
398 doy = int(file[5:8])
399 set = int(file[8:11])
399 set = int(file[8:11])
400 except:
400 except:
401 return None
401 return None
402
402
403 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
403 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
404 return thisDate
404 return thisDate
405
405
406 def getDateFromRadarFolder(folder):
406 def getDateFromRadarFolder(folder):
407 try:
407 try:
408 year = int(folder[1:5])
408 year = int(folder[1:5])
409 doy = int(folder[5:8])
409 doy = int(folder[5:8])
410 except:
410 except:
411 return None
411 return None
412
412
413 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
413 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
414 return thisDate
414 return thisDate
415
415
416 class JRODataIO:
416 class JRODataIO:
417
417
418 c = 3E8
418 c = 3E8
419
419
420 isConfig = False
420 isConfig = False
421
421
422 basicHeaderObj = None
422 basicHeaderObj = None
423
423
424 systemHeaderObj = None
424 systemHeaderObj = None
425
425
426 radarControllerHeaderObj = None
426 radarControllerHeaderObj = None
427
427
428 processingHeaderObj = None
428 processingHeaderObj = None
429
429
430 dtype = None
430 dtype = None
431
431
432 pathList = []
432 pathList = []
433
433
434 filenameList = []
434 filenameList = []
435
435
436 filename = None
436 filename = None
437
437
438 ext = None
438 ext = None
439
439
440 flagIsNewFile = 1
440 flagIsNewFile = 1
441
441
442 flagDiscontinuousBlock = 0
442 flagDiscontinuousBlock = 0
443
443
444 flagIsNewBlock = 0
444 flagIsNewBlock = 0
445
445
446 fp = None
446 fp = None
447
447
448 firstHeaderSize = 0
448 firstHeaderSize = 0
449
449
450 basicHeaderSize = 24
450 basicHeaderSize = 24
451
451
452 versionFile = 1103
452 versionFile = 1103
453
453
454 fileSize = None
454 fileSize = None
455
455
456 # ippSeconds = None
456 # ippSeconds = None
457
457
458 fileSizeByHeader = None
458 fileSizeByHeader = None
459
459
460 fileIndex = None
460 fileIndex = None
461
461
462 profileIndex = None
462 profileIndex = None
463
463
464 blockIndex = None
464 blockIndex = None
465
465
466 nTotalBlocks = None
466 nTotalBlocks = None
467
467
468 maxTimeStep = 30
468 maxTimeStep = 30
469
469
470 lastUTTime = None
470 lastUTTime = None
471
471
472 datablock = None
472 datablock = None
473
473
474 dataOut = None
474 dataOut = None
475
475
476 blocksize = None
476 blocksize = None
477
477
478 getByBlock = False
478 getByBlock = False
479
479
480 def __init__(self):
480 def __init__(self):
481
481
482 raise NotImplementedError
482 raise NotImplementedError
483
483
484 def run(self):
484 def run(self):
485
485
486 raise NotImplementedError
486 raise NotImplementedError
487
487
488 def getDtypeWidth(self):
488 def getDtypeWidth(self):
489
489
490 dtype_index = get_dtype_index(self.dtype)
490 dtype_index = get_dtype_index(self.dtype)
491 dtype_width = get_dtype_width(dtype_index)
491 dtype_width = get_dtype_width(dtype_index)
492
492
493 return dtype_width
493 return dtype_width
494
494
495 class JRODataReader(JRODataIO):
495 class JRODataReader(JRODataIO):
496
496
497
497
498 online = 0
498 online = 0
499
499
500 realtime = 0
500 realtime = 0
501
501
502 nReadBlocks = 0
502 nReadBlocks = 0
503
503
504 delay = 10 #number of seconds waiting a new file
504 delay = 10 #number of seconds waiting a new file
505
505
506 nTries = 3 #quantity tries
506 nTries = 3 #quantity tries
507
507
508 nFiles = 3 #number of files for searching
508 nFiles = 3 #number of files for searching
509
509
510 path = None
510 path = None
511
511
512 foldercounter = 0
512 foldercounter = 0
513
513
514 flagNoMoreFiles = 0
514 flagNoMoreFiles = 0
515
515
516 datetimeList = []
516 datetimeList = []
517
517
518 __isFirstTimeOnline = 1
518 __isFirstTimeOnline = 1
519
519
520 __printInfo = True
520 __printInfo = True
521
521
522 profileIndex = None
522 profileIndex = None
523
523
524 nTxs = 1
524 nTxs = 1
525
525
526 txIndex = None
526 txIndex = None
527
527
528 def __init__(self):
528 def __init__(self):
529
529
530 """
530 """
531 This class is used to find data files
531 This class is used to find data files
532
532
533 Example:
533 Example:
534 reader = JRODataReader()
534 reader = JRODataReader()
535 fileList = reader.findDataFiles()
535 fileList = reader.findDataFiles()
536
536
537 """
537 """
538 pass
538 pass
539
539
540
540
541 def createObjByDefault(self):
541 def createObjByDefault(self):
542 """
542 """
543
543
544 """
544 """
545 raise NotImplementedError
545 raise NotImplementedError
546
546
547 def getBlockDimension(self):
547 def getBlockDimension(self):
548
548
549 raise NotImplementedError
549 raise NotImplementedError
550
550
551 def __searchFilesOffLine(self,
551 def __searchFilesOffLine(self,
552 path,
552 path,
553 startDate=None,
553 startDate=None,
554 endDate=None,
554 endDate=None,
555 startTime=datetime.time(0,0,0),
555 startTime=datetime.time(0,0,0),
556 endTime=datetime.time(23,59,59),
556 endTime=datetime.time(23,59,59),
557 set=None,
557 set=None,
558 expLabel='',
558 expLabel='',
559 ext='.r',
559 ext='.r',
560 walk=True):
560 walk=True):
561
561
562 self.filenameList = []
562 self.filenameList = []
563 self.datetimeList = []
563 self.datetimeList = []
564
564
565 pathList = []
565 pathList = []
566
566
567 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
567 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
568
568
569 if dateList == []:
569 if dateList == []:
570 # print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
570 # print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
571 # datetime.datetime.combine(startDate,startTime).ctime(),
571 # datetime.datetime.combine(startDate,startTime).ctime(),
572 # datetime.datetime.combine(endDate,endTime).ctime())
572 # datetime.datetime.combine(endDate,endTime).ctime())
573
573
574 return None, None
574 return None, None
575
575
576 if len(dateList) > 1:
576 if len(dateList) > 1:
577 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
577 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
578 else:
578 else:
579 print "[Reading] data was found for the date %s" %(dateList[0])
579 print "[Reading] data was found for the date %s" %(dateList[0])
580
580
581 filenameList = []
581 filenameList = []
582 datetimeList = []
582 datetimeList = []
583
583
584 for thisPath in pathList:
584 for thisPath in pathList:
585 # thisPath = pathList[pathDict[file]]
585 # thisPath = pathList[pathDict[file]]
586
586
587 fileList = glob.glob1(thisPath, "*%s" %ext)
587 fileList = glob.glob1(thisPath, "*%s" %ext)
588 fileList.sort()
588 fileList.sort()
589
589
590 for file in fileList:
590 for file in fileList:
591
591
592 filename = os.path.join(thisPath,file)
592 filename = os.path.join(thisPath,file)
593
593
594 if not isFileInDateRange(filename, startDate, endDate):
594 if not isFileInDateRange(filename, startDate, endDate):
595 continue
595 continue
596
596
597 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
597 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
598
598
599 if not(thisDatetime):
599 if not(thisDatetime):
600 continue
600 continue
601
601
602 filenameList.append(filename)
602 filenameList.append(filename)
603 datetimeList.append(thisDatetime)
603 datetimeList.append(thisDatetime)
604
604
605 if not(filenameList):
605 if not(filenameList):
606 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
606 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
607 return None, None
607 return None, None
608
608
609 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
609 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
610 print
610 print
611
611
612 for i in range(len(filenameList)):
612 for i in range(len(filenameList)):
613 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
613 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
614
614
615 self.filenameList = filenameList
615 self.filenameList = filenameList
616 self.datetimeList = datetimeList
616 self.datetimeList = datetimeList
617
617
618 return pathList, filenameList
618 return pathList, filenameList
619
619
620 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
620 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
621
621
622 """
622 """
623 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
623 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
624 devuelve el archivo encontrado ademas de otros datos.
624 devuelve el archivo encontrado ademas de otros datos.
625
625
626 Input:
626 Input:
627 path : carpeta donde estan contenidos los files que contiene data
627 path : carpeta donde estan contenidos los files que contiene data
628
628
629 expLabel : Nombre del subexperimento (subfolder)
629 expLabel : Nombre del subexperimento (subfolder)
630
630
631 ext : extension de los files
631 ext : extension de los files
632
632
633 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
633 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
634
634
635 Return:
635 Return:
636 directory : eL directorio donde esta el file encontrado
636 directory : eL directorio donde esta el file encontrado
637 filename : el ultimo file de una determinada carpeta
637 filename : el ultimo file de una determinada carpeta
638 year : el anho
638 year : el anho
639 doy : el numero de dia del anho
639 doy : el numero de dia del anho
640 set : el set del archivo
640 set : el set del archivo
641
641
642
642
643 """
643 """
644 if not os.path.isdir(path):
644 if not os.path.isdir(path):
645 return None, None, None, None, None, None
645 return None, None, None, None, None, None
646
646
647 dirList = []
647 dirList = []
648
648
649 if not walk:
649 if not walk:
650 fullpath = path
650 fullpath = path
651 foldercounter = 0
651 foldercounter = 0
652 else:
652 else:
653 #Filtra solo los directorios
653 #Filtra solo los directorios
654 for thisPath in os.listdir(path):
654 for thisPath in os.listdir(path):
655 if not os.path.isdir(os.path.join(path,thisPath)):
655 if not os.path.isdir(os.path.join(path,thisPath)):
656 continue
656 continue
657 if not isRadarFolder(thisPath):
657 if not isRadarFolder(thisPath):
658 continue
658 continue
659
659
660 dirList.append(thisPath)
660 dirList.append(thisPath)
661
661
662 if not(dirList):
662 if not(dirList):
663 return None, None, None, None, None, None
663 return None, None, None, None, None, None
664
664
665 dirList = sorted( dirList, key=str.lower )
665 dirList = sorted( dirList, key=str.lower )
666
666
667 doypath = dirList[-1]
667 doypath = dirList[-1]
668 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
668 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
669 fullpath = os.path.join(path, doypath, expLabel)
669 fullpath = os.path.join(path, doypath, expLabel)
670
670
671
671
672 print "[Reading] %s folder was found: " %(fullpath )
672 print "[Reading] %s folder was found: " %(fullpath )
673
673
674 if set == None:
674 if set == None:
675 filename = getlastFileFromPath(fullpath, ext)
675 filename = getlastFileFromPath(fullpath, ext)
676 else:
676 else:
677 filename = getFileFromSet(fullpath, ext, set)
677 filename = getFileFromSet(fullpath, ext, set)
678
678
679 if not(filename):
679 if not(filename):
680 return None, None, None, None, None, None
680 return None, None, None, None, None, None
681
681
682 print "[Reading] %s file was found" %(filename)
682 print "[Reading] %s file was found" %(filename)
683
683
684 if not(self.__verifyFile(os.path.join(fullpath, filename))):
684 if not(self.__verifyFile(os.path.join(fullpath, filename))):
685 return None, None, None, None, None, None
685 return None, None, None, None, None, None
686
686
687 year = int( filename[1:5] )
687 year = int( filename[1:5] )
688 doy = int( filename[5:8] )
688 doy = int( filename[5:8] )
689 set = int( filename[8:11] )
689 set = int( filename[8:11] )
690
690
691 return fullpath, foldercounter, filename, year, doy, set
691 return fullpath, foldercounter, filename, year, doy, set
692
692
693 def __setNextFileOffline(self):
693 def __setNextFileOffline(self):
694
694
695 idFile = self.fileIndex
695 idFile = self.fileIndex
696
696
697 while (True):
697 while (True):
698 idFile += 1
698 idFile += 1
699 if not(idFile < len(self.filenameList)):
699 if not(idFile < len(self.filenameList)):
700 self.flagNoMoreFiles = 1
700 self.flagNoMoreFiles = 1
701 # print "[Reading] No more Files"
701 # print "[Reading] No more Files"
702 return 0
702 return 0
703
703
704 filename = self.filenameList[idFile]
704 filename = self.filenameList[idFile]
705
705
706 if not(self.__verifyFile(filename)):
706 if not(self.__verifyFile(filename)):
707 continue
707 continue
708
708
709 fileSize = os.path.getsize(filename)
709 fileSize = os.path.getsize(filename)
710 fp = open(filename,'rb')
710 fp = open(filename,'rb')
711 break
711 break
712
712
713 self.flagIsNewFile = 1
713 self.flagIsNewFile = 1
714 self.fileIndex = idFile
714 self.fileIndex = idFile
715 self.filename = filename
715 self.filename = filename
716 self.fileSize = fileSize
716 self.fileSize = fileSize
717 self.fp = fp
717 self.fp = fp
718
718
719 # print "[Reading] Setting the file: %s"%self.filename
719 # print "[Reading] Setting the file: %s"%self.filename
720
720
721 return 1
721 return 1
722
722
723 def __setNextFileOnline(self):
723 def __setNextFileOnline(self):
724 """
724 """
725 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
725 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
726 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
726 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
727 siguientes.
727 siguientes.
728
728
729 Affected:
729 Affected:
730 self.flagIsNewFile
730 self.flagIsNewFile
731 self.filename
731 self.filename
732 self.fileSize
732 self.fileSize
733 self.fp
733 self.fp
734 self.set
734 self.set
735 self.flagNoMoreFiles
735 self.flagNoMoreFiles
736
736
737 Return:
737 Return:
738 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
738 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
739 1 : si el file fue abierto con exito y esta listo a ser leido
739 1 : si el file fue abierto con exito y esta listo a ser leido
740
740
741 Excepciones:
741 Excepciones:
742 Si un determinado file no puede ser abierto
742 Si un determinado file no puede ser abierto
743 """
743 """
744 nFiles = 0
744 nFiles = 0
745 fileOk_flag = False
745 fileOk_flag = False
746 firstTime_flag = True
746 firstTime_flag = True
747
747
748 self.set += 1
748 self.set += 1
749
749
750 if self.set > 999:
750 if self.set > 999:
751 self.set = 0
751 self.set = 0
752 self.foldercounter += 1
752 self.foldercounter += 1
753
753
754 #busca el 1er file disponible
754 #busca el 1er file disponible
755 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
755 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
756 if fullfilename:
756 if fullfilename:
757 if self.__verifyFile(fullfilename, False):
757 if self.__verifyFile(fullfilename, False):
758 fileOk_flag = True
758 fileOk_flag = True
759
759
760 #si no encuentra un file entonces espera y vuelve a buscar
760 #si no encuentra un file entonces espera y vuelve a buscar
761 if not(fileOk_flag):
761 if not(fileOk_flag):
762 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
762 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
763
763
764 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
764 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
765 tries = self.nTries
765 tries = self.nTries
766 else:
766 else:
767 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
767 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
768
768
769 for nTries in range( tries ):
769 for nTries in range( tries ):
770 if firstTime_flag:
770 if firstTime_flag:
771 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
771 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
772 sleep( self.delay )
772 sleep( self.delay )
773 else:
773 else:
774 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
774 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
775
775
776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
776 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
777 if fullfilename:
777 if fullfilename:
778 if self.__verifyFile(fullfilename):
778 if self.__verifyFile(fullfilename):
779 fileOk_flag = True
779 fileOk_flag = True
780 break
780 break
781
781
782 if fileOk_flag:
782 if fileOk_flag:
783 break
783 break
784
784
785 firstTime_flag = False
785 firstTime_flag = False
786
786
787 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
787 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
788 self.set += 1
788 self.set += 1
789
789
790 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
790 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
791 self.set = 0
791 self.set = 0
792 self.doy += 1
792 self.doy += 1
793 self.foldercounter = 0
793 self.foldercounter = 0
794
794
795 if fileOk_flag:
795 if fileOk_flag:
796 self.fileSize = os.path.getsize( fullfilename )
796 self.fileSize = os.path.getsize( fullfilename )
797 self.filename = fullfilename
797 self.filename = fullfilename
798 self.flagIsNewFile = 1
798 self.flagIsNewFile = 1
799 if self.fp != None: self.fp.close()
799 if self.fp != None: self.fp.close()
800 self.fp = open(fullfilename, 'rb')
800 self.fp = open(fullfilename, 'rb')
801 self.flagNoMoreFiles = 0
801 self.flagNoMoreFiles = 0
802 # print '[Reading] Setting the file: %s' % fullfilename
802 # print '[Reading] Setting the file: %s' % fullfilename
803 else:
803 else:
804 self.fileSize = 0
804 self.fileSize = 0
805 self.filename = None
805 self.filename = None
806 self.flagIsNewFile = 0
806 self.flagIsNewFile = 0
807 self.fp = None
807 self.fp = None
808 self.flagNoMoreFiles = 1
808 self.flagNoMoreFiles = 1
809 # print '[Reading] No more files to read'
809 # print '[Reading] No more files to read'
810
810
811 return fileOk_flag
811 return fileOk_flag
812
812
813 def setNextFile(self):
813 def setNextFile(self):
814 if self.fp != None:
814 if self.fp != None:
815 self.fp.close()
815 self.fp.close()
816
816
817 if self.online:
817 if self.online:
818 newFile = self.__setNextFileOnline()
818 newFile = self.__setNextFileOnline()
819 else:
819 else:
820 newFile = self.__setNextFileOffline()
820 newFile = self.__setNextFileOffline()
821
821
822 if not(newFile):
822 if not(newFile):
823 print '[Reading] No more files to read'
823 print '[Reading] No more files to read'
824 return 0
824 return 0
825
825
826 print '[Reading] Setting the file: %s' % self.filename
826 print '[Reading] Setting the file: %s' % self.filename
827
827
828 self.__readFirstHeader()
828 self.__readFirstHeader()
829 self.nReadBlocks = 0
829 self.nReadBlocks = 0
830 return 1
830 return 1
831
831
832 def __waitNewBlock(self):
832 def __waitNewBlock(self):
833 """
833 """
834 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
834 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
835
835
836 Si el modo de lectura es OffLine siempre retorn 0
836 Si el modo de lectura es OffLine siempre retorn 0
837 """
837 """
838 if not self.online:
838 if not self.online:
839 return 0
839 return 0
840
840
841 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
841 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
842 return 0
842 return 0
843
843
844 currentPointer = self.fp.tell()
844 currentPointer = self.fp.tell()
845
845
846 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
846 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
847
847
848 for nTries in range( self.nTries ):
848 for nTries in range( self.nTries ):
849
849
850 self.fp.close()
850 self.fp.close()
851 self.fp = open( self.filename, 'rb' )
851 self.fp = open( self.filename, 'rb' )
852 self.fp.seek( currentPointer )
852 self.fp.seek( currentPointer )
853
853
854 self.fileSize = os.path.getsize( self.filename )
854 self.fileSize = os.path.getsize( self.filename )
855 currentSize = self.fileSize - currentPointer
855 currentSize = self.fileSize - currentPointer
856
856
857 if ( currentSize >= neededSize ):
857 if ( currentSize >= neededSize ):
858 self.basicHeaderObj.read(self.fp)
858 self.basicHeaderObj.read(self.fp)
859 return 1
859 return 1
860
860
861 if self.fileSize == self.fileSizeByHeader:
861 if self.fileSize == self.fileSizeByHeader:
862 # self.flagEoF = True
862 # self.flagEoF = True
863 return 0
863 return 0
864
864
865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
865 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
866 sleep( self.delay )
866 sleep( self.delay )
867
867
868
868
869 return 0
869 return 0
870
870
871 def waitDataBlock(self,pointer_location):
871 def waitDataBlock(self,pointer_location):
872
872
873 currentPointer = pointer_location
873 currentPointer = pointer_location
874
874
875 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
875 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
876
876
877 for nTries in range( self.nTries ):
877 for nTries in range( self.nTries ):
878 self.fp.close()
878 self.fp.close()
879 self.fp = open( self.filename, 'rb' )
879 self.fp = open( self.filename, 'rb' )
880 self.fp.seek( currentPointer )
880 self.fp.seek( currentPointer )
881
881
882 self.fileSize = os.path.getsize( self.filename )
882 self.fileSize = os.path.getsize( self.filename )
883 currentSize = self.fileSize - currentPointer
883 currentSize = self.fileSize - currentPointer
884
884
885 if ( currentSize >= neededSize ):
885 if ( currentSize >= neededSize ):
886 return 1
886 return 1
887
887
888 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
888 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
889 sleep( self.delay )
889 sleep( self.delay )
890
890
891 return 0
891 return 0
892
892
893 def __jumpToLastBlock(self):
893 def __jumpToLastBlock(self):
894
894
895 if not(self.__isFirstTimeOnline):
895 if not(self.__isFirstTimeOnline):
896 return
896 return
897
897
898 csize = self.fileSize - self.fp.tell()
898 csize = self.fileSize - self.fp.tell()
899 blocksize = self.processingHeaderObj.blockSize
899 blocksize = self.processingHeaderObj.blockSize
900
900
901 #salta el primer bloque de datos
901 #salta el primer bloque de datos
902 if csize > self.processingHeaderObj.blockSize:
902 if csize > self.processingHeaderObj.blockSize:
903 self.fp.seek(self.fp.tell() + blocksize)
903 self.fp.seek(self.fp.tell() + blocksize)
904 else:
904 else:
905 return
905 return
906
906
907 csize = self.fileSize - self.fp.tell()
907 csize = self.fileSize - self.fp.tell()
908 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
908 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
909 while True:
909 while True:
910
910
911 if self.fp.tell()<self.fileSize:
911 if self.fp.tell()<self.fileSize:
912 self.fp.seek(self.fp.tell() + neededsize)
912 self.fp.seek(self.fp.tell() + neededsize)
913 else:
913 else:
914 self.fp.seek(self.fp.tell() - neededsize)
914 self.fp.seek(self.fp.tell() - neededsize)
915 break
915 break
916
916
917 # csize = self.fileSize - self.fp.tell()
917 # csize = self.fileSize - self.fp.tell()
918 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
918 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
919 # factor = int(csize/neededsize)
919 # factor = int(csize/neededsize)
920 # if factor > 0:
920 # if factor > 0:
921 # self.fp.seek(self.fp.tell() + factor*neededsize)
921 # self.fp.seek(self.fp.tell() + factor*neededsize)
922
922
923 self.flagIsNewFile = 0
923 self.flagIsNewFile = 0
924 self.__isFirstTimeOnline = 0
924 self.__isFirstTimeOnline = 0
925
925
926 def __setNewBlock(self):
926 def __setNewBlock(self):
927
927
928 if self.fp == None:
928 if self.fp == None:
929 return 0
929 return 0
930
930
931 # if self.online:
931 # if self.online:
932 # self.__jumpToLastBlock()
932 # self.__jumpToLastBlock()
933
933
934 if self.flagIsNewFile:
934 if self.flagIsNewFile:
935 self.lastUTTime = self.basicHeaderObj.utc
935 self.lastUTTime = self.basicHeaderObj.utc
936 return 1
936 return 1
937
937
938 if self.realtime:
938 if self.realtime:
939 self.flagDiscontinuousBlock = 1
939 self.flagDiscontinuousBlock = 1
940 if not(self.setNextFile()):
940 if not(self.setNextFile()):
941 return 0
941 return 0
942 else:
942 else:
943 return 1
943 return 1
944
944
945 currentSize = self.fileSize - self.fp.tell()
945 currentSize = self.fileSize - self.fp.tell()
946 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
946 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
947
947
948 if (currentSize >= neededSize):
948 if (currentSize >= neededSize):
949 self.basicHeaderObj.read(self.fp)
949 self.basicHeaderObj.read(self.fp)
950 self.lastUTTime = self.basicHeaderObj.utc
950 self.lastUTTime = self.basicHeaderObj.utc
951 return 1
951 return 1
952
952
953 if self.__waitNewBlock():
953 if self.__waitNewBlock():
954 self.lastUTTime = self.basicHeaderObj.utc
954 self.lastUTTime = self.basicHeaderObj.utc
955 return 1
955 return 1
956
956
957 if not(self.setNextFile()):
957 if not(self.setNextFile()):
958 return 0
958 return 0
959
959
960 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
960 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
961 self.lastUTTime = self.basicHeaderObj.utc
961 self.lastUTTime = self.basicHeaderObj.utc
962
962
963 self.flagDiscontinuousBlock = 0
963 self.flagDiscontinuousBlock = 0
964
964
965 if deltaTime > self.maxTimeStep:
965 if deltaTime > self.maxTimeStep:
966 self.flagDiscontinuousBlock = 1
966 self.flagDiscontinuousBlock = 1
967
967
968 return 1
968 return 1
969
969
970 def readNextBlock(self):
970 def readNextBlock(self):
971
971
972 if not(self.__setNewBlock()):
972 if not(self.__setNewBlock()):
973 return 0
973 return 0
974
974
975 if not(self.readBlock()):
975 if not(self.readBlock()):
976 return 0
976 return 0
977
977
978 self.getBasicHeader()
978 self.getBasicHeader()
979
979
980 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
980 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
981 self.processingHeaderObj.dataBlocksPerFile,
981 self.processingHeaderObj.dataBlocksPerFile,
982 self.dataOut.datatime.ctime())
982 self.dataOut.datatime.ctime())
983 return 1
983 return 1
984
984
985 def __readFirstHeader(self):
985 def __readFirstHeader(self):
986
986
987 self.basicHeaderObj.read(self.fp)
987 self.basicHeaderObj.read(self.fp)
988 self.systemHeaderObj.read(self.fp)
988 self.systemHeaderObj.read(self.fp)
989 self.radarControllerHeaderObj.read(self.fp)
989 self.radarControllerHeaderObj.read(self.fp)
990 self.processingHeaderObj.read(self.fp)
990 self.processingHeaderObj.read(self.fp)
991
991
992 self.firstHeaderSize = self.basicHeaderObj.size
992 self.firstHeaderSize = self.basicHeaderObj.size
993
993
994 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
994 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
995 if datatype == 0:
995 if datatype == 0:
996 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
996 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
997 elif datatype == 1:
997 elif datatype == 1:
998 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
998 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
999 elif datatype == 2:
999 elif datatype == 2:
1000 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1000 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1001 elif datatype == 3:
1001 elif datatype == 3:
1002 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1002 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1003 elif datatype == 4:
1003 elif datatype == 4:
1004 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1004 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1005 elif datatype == 5:
1005 elif datatype == 5:
1006 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1006 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1007 else:
1007 else:
1008 raise ValueError, 'Data type was not defined'
1008 raise ValueError, 'Data type was not defined'
1009
1009
1010 self.dtype = datatype_str
1010 self.dtype = datatype_str
1011 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1011 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1012 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1012 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1013 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1013 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1014 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1014 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1015 self.getBlockDimension()
1015 self.getBlockDimension()
1016
1016
1017 def __verifyFile(self, filename, msgFlag=True):
1017 def __verifyFile(self, filename, msgFlag=True):
1018
1018
1019 msg = None
1019 msg = None
1020
1020
1021 try:
1021 try:
1022 fp = open(filename, 'rb')
1022 fp = open(filename, 'rb')
1023 except IOError:
1023 except IOError:
1024
1024
1025 if msgFlag:
1025 if msgFlag:
1026 print "[Reading] File %s can't be opened" % (filename)
1026 print "[Reading] File %s can't be opened" % (filename)
1027
1027
1028 return False
1028 return False
1029
1029
1030 currentPosition = fp.tell()
1030 currentPosition = fp.tell()
1031 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1031 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1032
1032
1033 if neededSize == 0:
1033 if neededSize == 0:
1034 basicHeaderObj = BasicHeader(LOCALTIME)
1034 basicHeaderObj = BasicHeader(LOCALTIME)
1035 systemHeaderObj = SystemHeader()
1035 systemHeaderObj = SystemHeader()
1036 radarControllerHeaderObj = RadarControllerHeader()
1036 radarControllerHeaderObj = RadarControllerHeader()
1037 processingHeaderObj = ProcessingHeader()
1037 processingHeaderObj = ProcessingHeader()
1038
1038
1039 if not( basicHeaderObj.read(fp) ):
1039 if not( basicHeaderObj.read(fp) ):
1040 fp.close()
1040 fp.close()
1041 return False
1041 return False
1042
1042
1043 if not( systemHeaderObj.read(fp) ):
1043 if not( systemHeaderObj.read(fp) ):
1044 fp.close()
1044 fp.close()
1045 return False
1045 return False
1046
1046
1047 if not( radarControllerHeaderObj.read(fp) ):
1047 if not( radarControllerHeaderObj.read(fp) ):
1048 fp.close()
1048 fp.close()
1049 return False
1049 return False
1050
1050
1051 if not( processingHeaderObj.read(fp) ):
1051 if not( processingHeaderObj.read(fp) ):
1052 fp.close()
1052 fp.close()
1053 return False
1053 return False
1054
1054
1055 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1055 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1056 else:
1056 else:
1057 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1057 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1058
1058
1059 fp.close()
1059 fp.close()
1060
1060
1061 fileSize = os.path.getsize(filename)
1061 fileSize = os.path.getsize(filename)
1062 currentSize = fileSize - currentPosition
1062 currentSize = fileSize - currentPosition
1063
1063
1064 if currentSize < neededSize:
1064 if currentSize < neededSize:
1065 if msgFlag and (msg != None):
1065 if msgFlag and (msg != None):
1066 print msg
1066 print msg
1067 return False
1067 return False
1068
1068
1069 return True
1069 return True
1070
1070
1071 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1071 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1072
1072
1073 dateList = []
1073 dateList = []
1074 pathList = []
1074 pathList = []
1075
1075
1076 multi_path = path.split(',')
1076 multi_path = path.split(',')
1077
1077
1078 if not walk:
1078 if not walk:
1079
1079
1080 for single_path in multi_path:
1080 for single_path in multi_path:
1081
1081
1082 if not os.path.isdir(single_path):
1082 if not os.path.isdir(single_path):
1083 continue
1083 continue
1084
1084
1085 fileList = glob.glob1(single_path, "*"+ext)
1085 fileList = glob.glob1(single_path, "*"+ext)
1086
1086
1087 if not fileList:
1088 continue
1089
1090 fileList.sort()
1091
1087 for thisFile in fileList:
1092 for thisFile in fileList:
1088
1093
1089 if not os.path.isfile(os.path.join(single_path, thisFile)):
1094 if not os.path.isfile(os.path.join(single_path, thisFile)):
1090 continue
1095 continue
1091
1096
1092 if not isRadarFile(thisFile):
1097 if not isRadarFile(thisFile):
1093 continue
1098 continue
1094
1099
1095 if not isFileInDateRange(thisFile, startDate, endDate):
1100 if not isFileInDateRange(thisFile, startDate, endDate):
1096 continue
1101 continue
1097
1102
1098 thisDate = getDateFromRadarFile(thisFile)
1103 thisDate = getDateFromRadarFile(thisFile)
1099
1104
1100 if thisDate in dateList:
1105 if thisDate in dateList:
1101 continue
1106 continue
1102
1107
1103 dateList.append(thisDate)
1108 dateList.append(thisDate)
1104 pathList.append(single_path)
1109 pathList.append(single_path)
1105
1110
1106 else:
1111 else:
1107 for single_path in multi_path:
1112 for single_path in multi_path:
1108
1113
1109 if not os.path.isdir(single_path):
1114 if not os.path.isdir(single_path):
1110 continue
1115 continue
1111
1116
1112 dirList = []
1117 dirList = []
1113
1118
1114 for thisPath in os.listdir(single_path):
1119 for thisPath in os.listdir(single_path):
1115
1120
1116 if not os.path.isdir(os.path.join(single_path,thisPath)):
1121 if not os.path.isdir(os.path.join(single_path,thisPath)):
1117 continue
1122 continue
1118
1123
1119 if not isRadarFolder(thisPath):
1124 if not isRadarFolder(thisPath):
1120 continue
1125 continue
1121
1126
1122 if not isFolderInDateRange(thisPath, startDate, endDate):
1127 if not isFolderInDateRange(thisPath, startDate, endDate):
1123 continue
1128 continue
1124
1129
1125 dirList.append(thisPath)
1130 dirList.append(thisPath)
1126
1131
1127 if not dirList:
1132 if not dirList:
1128 continue
1133 continue
1129
1134
1135 dirList.sort()
1136
1130 for thisDir in dirList:
1137 for thisDir in dirList:
1131
1138
1132 datapath = os.path.join(single_path, thisDir, expLabel)
1139 datapath = os.path.join(single_path, thisDir, expLabel)
1133 fileList = glob.glob1(datapath, "*"+ext)
1140 fileList = glob.glob1(datapath, "*"+ext)
1134
1141
1135 if len(fileList) < 1:
1142 if len(fileList) < 1:
1136 continue
1143 continue
1137
1144
1138 thisDate = getDateFromRadarFolder(thisDir)
1145 thisDate = getDateFromRadarFolder(thisDir)
1139
1146
1140 pathList.append(datapath)
1147 pathList.append(datapath)
1141 dateList.append(thisDate)
1148 dateList.append(thisDate)
1142
1149
1143 dateList.sort()
1150 dateList.sort()
1144
1151
1145 if walk:
1152 if walk:
1146 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1153 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1147 else:
1154 else:
1148 pattern_path = multi_path[0]
1155 pattern_path = multi_path[0]
1149
1156
1150 if not dateList:
1157 if not dateList:
1151 print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
1158 print "[Reading] No *%s files in %s from %s to %s" %(ext, pattern_path, startDate, endDate)
1152
1159
1153 if include_path:
1160 if include_path:
1154 return dateList, pathList
1161 return dateList, pathList
1155
1162
1156 return dateList
1163 return dateList
1157
1164
1158 def setup(self,
1165 def setup(self,
1159 path=None,
1166 path=None,
1160 startDate=None,
1167 startDate=None,
1161 endDate=None,
1168 endDate=None,
1162 startTime=datetime.time(0,0,0),
1169 startTime=datetime.time(0,0,0),
1163 endTime=datetime.time(23,59,59),
1170 endTime=datetime.time(23,59,59),
1164 set=None,
1171 set=None,
1165 expLabel = "",
1172 expLabel = "",
1166 ext = None,
1173 ext = None,
1167 online = False,
1174 online = False,
1168 delay = 60,
1175 delay = 60,
1169 walk = True,
1176 walk = True,
1170 getblock = False,
1177 getblock = False,
1171 nTxs = 1,
1178 nTxs = 1,
1172 realtime=False):
1179 realtime=False):
1173
1180
1174 if path == None:
1181 if path == None:
1175 raise ValueError, "[Reading] The path is not valid"
1182 raise ValueError, "[Reading] The path is not valid"
1176
1183
1177 if ext == None:
1184 if ext == None:
1178 ext = self.ext
1185 ext = self.ext
1179
1186
1180 if online:
1187 if online:
1181 print "[Reading] Searching files in online mode..."
1188 print "[Reading] Searching files in online mode..."
1182
1189
1183 for nTries in range( self.nTries ):
1190 for nTries in range( self.nTries ):
1184 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1191 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1185
1192
1186 if fullpath:
1193 if fullpath:
1187 break
1194 break
1188
1195
1189 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1196 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1190 sleep( self.delay )
1197 sleep( self.delay )
1191
1198
1192 if not(fullpath):
1199 if not(fullpath):
1193 print "[Reading] There 'isn't any valid file in %s" % path
1200 print "[Reading] There 'isn't any valid file in %s" % path
1194 return
1201 return
1195
1202
1196 self.year = year
1203 self.year = year
1197 self.doy = doy
1204 self.doy = doy
1198 self.set = set - 1
1205 self.set = set - 1
1199 self.path = path
1206 self.path = path
1200 self.foldercounter = foldercounter
1207 self.foldercounter = foldercounter
1201 last_set = None
1208 last_set = None
1202
1209
1203 else:
1210 else:
1204 print "[Reading] Searching files in offline mode ..."
1211 print "[Reading] Searching files in offline mode ..."
1205 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1212 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1206 startTime=startTime, endTime=endTime,
1213 startTime=startTime, endTime=endTime,
1207 set=set, expLabel=expLabel, ext=ext,
1214 set=set, expLabel=expLabel, ext=ext,
1208 walk=walk)
1215 walk=walk)
1209
1216
1210 if not(pathList):
1217 if not(pathList):
1211 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1218 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1212 # datetime.datetime.combine(startDate,startTime).ctime(),
1219 # datetime.datetime.combine(startDate,startTime).ctime(),
1213 # datetime.datetime.combine(endDate,endTime).ctime())
1220 # datetime.datetime.combine(endDate,endTime).ctime())
1214
1221
1215 # sys.exit(-1)
1222 # sys.exit(-1)
1216
1223
1217 self.fileIndex = -1
1224 self.fileIndex = -1
1218 self.pathList = []
1225 self.pathList = []
1219 self.filenameList = []
1226 self.filenameList = []
1220 return
1227 return
1221
1228
1222 self.fileIndex = -1
1229 self.fileIndex = -1
1223 self.pathList = pathList
1230 self.pathList = pathList
1224 self.filenameList = filenameList
1231 self.filenameList = filenameList
1225 file_name = os.path.basename(filenameList[-1])
1232 file_name = os.path.basename(filenameList[-1])
1226 basename, ext = os.path.splitext(file_name)
1233 basename, ext = os.path.splitext(file_name)
1227 last_set = int(basename[-3:])
1234 last_set = int(basename[-3:])
1228
1235
1229 self.online = online
1236 self.online = online
1230 self.realtime = realtime
1237 self.realtime = realtime
1231 self.delay = delay
1238 self.delay = delay
1232 ext = ext.lower()
1239 ext = ext.lower()
1233 self.ext = ext
1240 self.ext = ext
1234 self.getByBlock = getblock
1241 self.getByBlock = getblock
1235 self.nTxs = int(nTxs)
1242 self.nTxs = int(nTxs)
1236
1243
1237 if not(self.setNextFile()):
1244 if not(self.setNextFile()):
1238 if (startDate!=None) and (endDate!=None):
1245 if (startDate!=None) and (endDate!=None):
1239 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1246 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1240 elif startDate != None:
1247 elif startDate != None:
1241 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1248 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1242 else:
1249 else:
1243 print "[Reading] No files"
1250 print "[Reading] No files"
1244
1251
1245 self.fileIndex = -1
1252 self.fileIndex = -1
1246 self.pathList = []
1253 self.pathList = []
1247 self.filenameList = []
1254 self.filenameList = []
1248 return
1255 return
1249
1256
1250 # self.getBasicHeader()
1257 # self.getBasicHeader()
1251
1258
1252 if last_set != None:
1259 if last_set != None:
1253 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1260 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1254 return
1261 return
1255
1262
1256 def getBasicHeader(self):
1263 def getBasicHeader(self):
1257
1264
1258 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1265 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1259
1266
1260 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1267 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1261
1268
1262 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1269 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1263
1270
1264 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1271 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1265
1272
1266 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1273 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1267
1274
1268 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1275 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1269
1276
1270 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1277 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1271
1278
1272 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1279 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1273
1280
1274
1281
1275 def getFirstHeader(self):
1282 def getFirstHeader(self):
1276
1283
1277 raise NotImplementedError
1284 raise NotImplementedError
1278
1285
1279 def getData(self):
1286 def getData(self):
1280
1287
1281 raise NotImplementedError
1288 raise NotImplementedError
1282
1289
1283 def hasNotDataInBuffer(self):
1290 def hasNotDataInBuffer(self):
1284
1291
1285 raise NotImplementedError
1292 raise NotImplementedError
1286
1293
1287 def readBlock(self):
1294 def readBlock(self):
1288
1295
1289 raise NotImplementedError
1296 raise NotImplementedError
1290
1297
1291 def isEndProcess(self):
1298 def isEndProcess(self):
1292
1299
1293 return self.flagNoMoreFiles
1300 return self.flagNoMoreFiles
1294
1301
1295 def printReadBlocks(self):
1302 def printReadBlocks(self):
1296
1303
1297 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1304 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1298
1305
1299 def printTotalBlocks(self):
1306 def printTotalBlocks(self):
1300
1307
1301 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1308 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1302
1309
1303 def printNumberOfBlock(self):
1310 def printNumberOfBlock(self):
1304
1311
1305 if self.flagIsNewBlock:
1312 if self.flagIsNewBlock:
1306 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1313 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1307 self.processingHeaderObj.dataBlocksPerFile,
1314 self.processingHeaderObj.dataBlocksPerFile,
1308 self.dataOut.datatime.ctime())
1315 self.dataOut.datatime.ctime())
1309
1316
1310 def printInfo(self):
1317 def printInfo(self):
1311
1318
1312 if self.__printInfo == False:
1319 if self.__printInfo == False:
1313 return
1320 return
1314
1321
1315 self.basicHeaderObj.printInfo()
1322 self.basicHeaderObj.printInfo()
1316 self.systemHeaderObj.printInfo()
1323 self.systemHeaderObj.printInfo()
1317 self.radarControllerHeaderObj.printInfo()
1324 self.radarControllerHeaderObj.printInfo()
1318 self.processingHeaderObj.printInfo()
1325 self.processingHeaderObj.printInfo()
1319
1326
1320 self.__printInfo = False
1327 self.__printInfo = False
1321
1328
1322
1329
1323 def run(self, **kwargs):
1330 def run(self, **kwargs):
1324
1331
1325 if not(self.isConfig):
1332 if not(self.isConfig):
1326
1333
1327 # self.dataOut = dataOut
1334 # self.dataOut = dataOut
1328 self.setup(**kwargs)
1335 self.setup(**kwargs)
1329 self.isConfig = True
1336 self.isConfig = True
1330
1337
1331 self.getData()
1338 self.getData()
1332
1339
1333 class JRODataWriter(JRODataIO):
1340 class JRODataWriter(JRODataIO):
1334
1341
1335 """
1342 """
1336 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1343 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1337 de los datos siempre se realiza por bloques.
1344 de los datos siempre se realiza por bloques.
1338 """
1345 """
1339
1346
1340 blockIndex = 0
1347 blockIndex = 0
1341
1348
1342 path = None
1349 path = None
1343
1350
1344 setFile = None
1351 setFile = None
1345
1352
1346 profilesPerBlock = None
1353 profilesPerBlock = None
1347
1354
1348 blocksPerFile = None
1355 blocksPerFile = None
1349
1356
1350 nWriteBlocks = 0
1357 nWriteBlocks = 0
1351
1358
1352 fileDate = None
1359 fileDate = None
1353
1360
1354 def __init__(self, dataOut=None):
1361 def __init__(self, dataOut=None):
1355 raise NotImplementedError
1362 raise NotImplementedError
1356
1363
1357
1364
1358 def hasAllDataInBuffer(self):
1365 def hasAllDataInBuffer(self):
1359 raise NotImplementedError
1366 raise NotImplementedError
1360
1367
1361
1368
1362 def setBlockDimension(self):
1369 def setBlockDimension(self):
1363 raise NotImplementedError
1370 raise NotImplementedError
1364
1371
1365
1372
1366 def writeBlock(self):
1373 def writeBlock(self):
1367 raise NotImplementedError
1374 raise NotImplementedError
1368
1375
1369
1376
1370 def putData(self):
1377 def putData(self):
1371 raise NotImplementedError
1378 raise NotImplementedError
1372
1379
1373
1380
1374 def getProcessFlags(self):
1381 def getProcessFlags(self):
1375
1382
1376 processFlags = 0
1383 processFlags = 0
1377
1384
1378 dtype_index = get_dtype_index(self.dtype)
1385 dtype_index = get_dtype_index(self.dtype)
1379 procflag_dtype = get_procflag_dtype(dtype_index)
1386 procflag_dtype = get_procflag_dtype(dtype_index)
1380
1387
1381 processFlags += procflag_dtype
1388 processFlags += procflag_dtype
1382
1389
1383 if self.dataOut.flagDecodeData:
1390 if self.dataOut.flagDecodeData:
1384 processFlags += PROCFLAG.DECODE_DATA
1391 processFlags += PROCFLAG.DECODE_DATA
1385
1392
1386 if self.dataOut.flagDeflipData:
1393 if self.dataOut.flagDeflipData:
1387 processFlags += PROCFLAG.DEFLIP_DATA
1394 processFlags += PROCFLAG.DEFLIP_DATA
1388
1395
1389 if self.dataOut.code is not None:
1396 if self.dataOut.code is not None:
1390 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1397 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1391
1398
1392 if self.dataOut.nCohInt > 1:
1399 if self.dataOut.nCohInt > 1:
1393 processFlags += PROCFLAG.COHERENT_INTEGRATION
1400 processFlags += PROCFLAG.COHERENT_INTEGRATION
1394
1401
1395 if self.dataOut.type == "Spectra":
1402 if self.dataOut.type == "Spectra":
1396 if self.dataOut.nIncohInt > 1:
1403 if self.dataOut.nIncohInt > 1:
1397 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1404 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1398
1405
1399 if self.dataOut.data_dc is not None:
1406 if self.dataOut.data_dc is not None:
1400 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1407 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1401
1408
1402 if self.dataOut.flagShiftFFT:
1409 if self.dataOut.flagShiftFFT:
1403 processFlags += PROCFLAG.SHIFT_FFT_DATA
1410 processFlags += PROCFLAG.SHIFT_FFT_DATA
1404
1411
1405 return processFlags
1412 return processFlags
1406
1413
1407 def setBasicHeader(self):
1414 def setBasicHeader(self):
1408
1415
1409 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1416 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1410 self.basicHeaderObj.version = self.versionFile
1417 self.basicHeaderObj.version = self.versionFile
1411 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1418 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1412
1419
1413 utc = numpy.floor(self.dataOut.utctime)
1420 utc = numpy.floor(self.dataOut.utctime)
1414 milisecond = (self.dataOut.utctime - utc)* 1000.0
1421 milisecond = (self.dataOut.utctime - utc)* 1000.0
1415
1422
1416 self.basicHeaderObj.utc = utc
1423 self.basicHeaderObj.utc = utc
1417 self.basicHeaderObj.miliSecond = milisecond
1424 self.basicHeaderObj.miliSecond = milisecond
1418 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1425 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1419 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1426 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1420 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1427 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1421
1428
1422 def setFirstHeader(self):
1429 def setFirstHeader(self):
1423 """
1430 """
1424 Obtiene una copia del First Header
1431 Obtiene una copia del First Header
1425
1432
1426 Affected:
1433 Affected:
1427
1434
1428 self.basicHeaderObj
1435 self.basicHeaderObj
1429 self.systemHeaderObj
1436 self.systemHeaderObj
1430 self.radarControllerHeaderObj
1437 self.radarControllerHeaderObj
1431 self.processingHeaderObj self.
1438 self.processingHeaderObj self.
1432
1439
1433 Return:
1440 Return:
1434 None
1441 None
1435 """
1442 """
1436
1443
1437 raise NotImplementedError
1444 raise NotImplementedError
1438
1445
1439 def __writeFirstHeader(self):
1446 def __writeFirstHeader(self):
1440 """
1447 """
1441 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1448 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1442
1449
1443 Affected:
1450 Affected:
1444 __dataType
1451 __dataType
1445
1452
1446 Return:
1453 Return:
1447 None
1454 None
1448 """
1455 """
1449
1456
1450 # CALCULAR PARAMETROS
1457 # CALCULAR PARAMETROS
1451
1458
1452 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1459 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1453 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1460 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1454
1461
1455 self.basicHeaderObj.write(self.fp)
1462 self.basicHeaderObj.write(self.fp)
1456 self.systemHeaderObj.write(self.fp)
1463 self.systemHeaderObj.write(self.fp)
1457 self.radarControllerHeaderObj.write(self.fp)
1464 self.radarControllerHeaderObj.write(self.fp)
1458 self.processingHeaderObj.write(self.fp)
1465 self.processingHeaderObj.write(self.fp)
1459
1466
1460 def __setNewBlock(self):
1467 def __setNewBlock(self):
1461 """
1468 """
1462 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1469 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1463
1470
1464 Return:
1471 Return:
1465 0 : si no pudo escribir nada
1472 0 : si no pudo escribir nada
1466 1 : Si escribio el Basic el First Header
1473 1 : Si escribio el Basic el First Header
1467 """
1474 """
1468 if self.fp == None:
1475 if self.fp == None:
1469 self.setNextFile()
1476 self.setNextFile()
1470
1477
1471 if self.flagIsNewFile:
1478 if self.flagIsNewFile:
1472 return 1
1479 return 1
1473
1480
1474 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1481 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1475 self.basicHeaderObj.write(self.fp)
1482 self.basicHeaderObj.write(self.fp)
1476 return 1
1483 return 1
1477
1484
1478 if not( self.setNextFile() ):
1485 if not( self.setNextFile() ):
1479 return 0
1486 return 0
1480
1487
1481 return 1
1488 return 1
1482
1489
1483
1490
1484 def writeNextBlock(self):
1491 def writeNextBlock(self):
1485 """
1492 """
1486 Selecciona el bloque siguiente de datos y los escribe en un file
1493 Selecciona el bloque siguiente de datos y los escribe en un file
1487
1494
1488 Return:
1495 Return:
1489 0 : Si no hizo pudo escribir el bloque de datos
1496 0 : Si no hizo pudo escribir el bloque de datos
1490 1 : Si no pudo escribir el bloque de datos
1497 1 : Si no pudo escribir el bloque de datos
1491 """
1498 """
1492 if not( self.__setNewBlock() ):
1499 if not( self.__setNewBlock() ):
1493 return 0
1500 return 0
1494
1501
1495 self.writeBlock()
1502 self.writeBlock()
1496
1503
1497 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1504 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1498 self.processingHeaderObj.dataBlocksPerFile)
1505 self.processingHeaderObj.dataBlocksPerFile)
1499
1506
1500 return 1
1507 return 1
1501
1508
1502 def setNextFile(self):
1509 def setNextFile(self):
1503 """
1510 """
1504 Determina el siguiente file que sera escrito
1511 Determina el siguiente file que sera escrito
1505
1512
1506 Affected:
1513 Affected:
1507 self.filename
1514 self.filename
1508 self.subfolder
1515 self.subfolder
1509 self.fp
1516 self.fp
1510 self.setFile
1517 self.setFile
1511 self.flagIsNewFile
1518 self.flagIsNewFile
1512
1519
1513 Return:
1520 Return:
1514 0 : Si el archivo no puede ser escrito
1521 0 : Si el archivo no puede ser escrito
1515 1 : Si el archivo esta listo para ser escrito
1522 1 : Si el archivo esta listo para ser escrito
1516 """
1523 """
1517 ext = self.ext
1524 ext = self.ext
1518 path = self.path
1525 path = self.path
1519
1526
1520 if self.fp != None:
1527 if self.fp != None:
1521 self.fp.close()
1528 self.fp.close()
1522
1529
1523 timeTuple = time.localtime( self.dataOut.utctime)
1530 timeTuple = time.localtime( self.dataOut.utctime)
1524 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1531 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1525
1532
1526 fullpath = os.path.join( path, subfolder )
1533 fullpath = os.path.join( path, subfolder )
1527 setFile = self.setFile
1534 setFile = self.setFile
1528
1535
1529 if not( os.path.exists(fullpath) ):
1536 if not( os.path.exists(fullpath) ):
1530 os.mkdir(fullpath)
1537 os.mkdir(fullpath)
1531 setFile = -1 #inicializo mi contador de seteo
1538 setFile = -1 #inicializo mi contador de seteo
1532 else:
1539 else:
1533 filesList = os.listdir( fullpath )
1540 filesList = os.listdir( fullpath )
1534 if len( filesList ) > 0:
1541 if len( filesList ) > 0:
1535 filesList = sorted( filesList, key=str.lower )
1542 filesList = sorted( filesList, key=str.lower )
1536 filen = filesList[-1]
1543 filen = filesList[-1]
1537 # el filename debera tener el siguiente formato
1544 # el filename debera tener el siguiente formato
1538 # 0 1234 567 89A BCDE (hex)
1545 # 0 1234 567 89A BCDE (hex)
1539 # x YYYY DDD SSS .ext
1546 # x YYYY DDD SSS .ext
1540 if isNumber( filen[8:11] ):
1547 if isNumber( filen[8:11] ):
1541 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1548 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1542 else:
1549 else:
1543 setFile = -1
1550 setFile = -1
1544 else:
1551 else:
1545 setFile = -1 #inicializo mi contador de seteo
1552 setFile = -1 #inicializo mi contador de seteo
1546
1553
1547 setFile += 1
1554 setFile += 1
1548
1555
1549 #If this is a new day it resets some values
1556 #If this is a new day it resets some values
1550 if self.dataOut.datatime.date() > self.fileDate:
1557 if self.dataOut.datatime.date() > self.fileDate:
1551 setFile = 0
1558 setFile = 0
1552 self.nTotalBlocks = 0
1559 self.nTotalBlocks = 0
1553
1560
1554 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1561 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1555
1562
1556 filename = os.path.join( path, subfolder, filen )
1563 filename = os.path.join( path, subfolder, filen )
1557
1564
1558 fp = open( filename,'wb' )
1565 fp = open( filename,'wb' )
1559
1566
1560 self.blockIndex = 0
1567 self.blockIndex = 0
1561
1568
1562 #guardando atributos
1569 #guardando atributos
1563 self.filename = filename
1570 self.filename = filename
1564 self.subfolder = subfolder
1571 self.subfolder = subfolder
1565 self.fp = fp
1572 self.fp = fp
1566 self.setFile = setFile
1573 self.setFile = setFile
1567 self.flagIsNewFile = 1
1574 self.flagIsNewFile = 1
1568 self.fileDate = self.dataOut.datatime.date()
1575 self.fileDate = self.dataOut.datatime.date()
1569
1576
1570 self.setFirstHeader()
1577 self.setFirstHeader()
1571
1578
1572 print '[Writing] Opening file: %s'%self.filename
1579 print '[Writing] Opening file: %s'%self.filename
1573
1580
1574 self.__writeFirstHeader()
1581 self.__writeFirstHeader()
1575
1582
1576 return 1
1583 return 1
1577
1584
1578 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1585 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1579 """
1586 """
1580 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1587 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1581
1588
1582 Inputs:
1589 Inputs:
1583 path : directory where data will be saved
1590 path : directory where data will be saved
1584 profilesPerBlock : number of profiles per block
1591 profilesPerBlock : number of profiles per block
1585 set : initial file set
1592 set : initial file set
1586 datatype : An integer number that defines data type:
1593 datatype : An integer number that defines data type:
1587 0 : int8 (1 byte)
1594 0 : int8 (1 byte)
1588 1 : int16 (2 bytes)
1595 1 : int16 (2 bytes)
1589 2 : int32 (4 bytes)
1596 2 : int32 (4 bytes)
1590 3 : int64 (8 bytes)
1597 3 : int64 (8 bytes)
1591 4 : float32 (4 bytes)
1598 4 : float32 (4 bytes)
1592 5 : double64 (8 bytes)
1599 5 : double64 (8 bytes)
1593
1600
1594 Return:
1601 Return:
1595 0 : Si no realizo un buen seteo
1602 0 : Si no realizo un buen seteo
1596 1 : Si realizo un buen seteo
1603 1 : Si realizo un buen seteo
1597 """
1604 """
1598
1605
1599 if ext == None:
1606 if ext == None:
1600 ext = self.ext
1607 ext = self.ext
1601
1608
1602 self.ext = ext.lower()
1609 self.ext = ext.lower()
1603
1610
1604 self.path = path
1611 self.path = path
1605
1612
1606 if set is None:
1613 if set is None:
1607 self.setFile = -1
1614 self.setFile = -1
1608 else:
1615 else:
1609 self.setFile = set - 1
1616 self.setFile = set - 1
1610
1617
1611 self.blocksPerFile = blocksPerFile
1618 self.blocksPerFile = blocksPerFile
1612
1619
1613 self.profilesPerBlock = profilesPerBlock
1620 self.profilesPerBlock = profilesPerBlock
1614
1621
1615 self.dataOut = dataOut
1622 self.dataOut = dataOut
1616 self.fileDate = self.dataOut.datatime.date()
1623 self.fileDate = self.dataOut.datatime.date()
1617 #By default
1624 #By default
1618 self.dtype = self.dataOut.dtype
1625 self.dtype = self.dataOut.dtype
1619
1626
1620 if datatype is not None:
1627 if datatype is not None:
1621 self.dtype = get_numpy_dtype(datatype)
1628 self.dtype = get_numpy_dtype(datatype)
1622
1629
1623 if not(self.setNextFile()):
1630 if not(self.setNextFile()):
1624 print "[Writing] There isn't a next file"
1631 print "[Writing] There isn't a next file"
1625 return 0
1632 return 0
1626
1633
1627 self.setBlockDimension()
1634 self.setBlockDimension()
1628
1635
1629 return 1
1636 return 1
1630
1637
1631 def run(self, dataOut, **kwargs):
1638 def run(self, dataOut, **kwargs):
1632
1639
1633 if not(self.isConfig):
1640 if not(self.isConfig):
1634
1641
1635 self.setup(dataOut, **kwargs)
1642 self.setup(dataOut, **kwargs)
1636 self.isConfig = True
1643 self.isConfig = True
1637
1644
1638 self.putData()
1645 self.putData()
1639
1646
General Comments 0
You need to be logged in to leave comments. Login now