##// END OF EJS Templates
schain
RSS
base primer
José Chávez -
r1038:2da89ff4f915
parent child
Show More
Side by Side Unified
Context file:
r1038:2da89ff4f915 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1976 +1,1866
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time, datetime
13 import time, datetime
14 import traceback
14 import traceback
15 import zmq
15 import zmq
16
16
17 try:
17 try:
18 from gevent import sleep
18 from gevent import sleep
19 except:
19 except:
20 from time import sleep
20 from time import sleep
21
21
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24
24
25 LOCALTIME = True
25 LOCALTIME = True
26
26
27 def isNumber(cad):
27 def isNumber(cad):
28 """
28 """
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30
30
31 Excepciones:
31 Excepciones:
32 Si un determinado string no puede ser convertido a numero
32 Si un determinado string no puede ser convertido a numero
33 Input:
33 Input:
34 str, string al cual se le analiza para determinar si convertible a un numero o no
34 str, string al cual se le analiza para determinar si convertible a un numero o no
35
35
36 Return:
36 Return:
37 True : si el string es uno numerico
37 True : si el string es uno numerico
38 False : no es un string numerico
38 False : no es un string numerico
39 """
39 """
40 try:
40 try:
41 float( cad )
41 float( cad )
42 return True
42 return True
43 except:
43 except:
44 return False
44 return False
45
45
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 """
47 """
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49
49
50 Inputs:
50 Inputs:
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52
52
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 segundos contados desde 01/01/1970.
54 segundos contados desde 01/01/1970.
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 segundos contados desde 01/01/1970.
56 segundos contados desde 01/01/1970.
57
57
58 Return:
58 Return:
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 fecha especificado, de lo contrario retorna False.
60 fecha especificado, de lo contrario retorna False.
61
61
62 Excepciones:
62 Excepciones:
63 Si el archivo no existe o no puede ser abierto
63 Si el archivo no existe o no puede ser abierto
64 Si la cabecera no puede ser leida.
64 Si la cabecera no puede ser leida.
65
65
66 """
66 """
67 basicHeaderObj = BasicHeader(LOCALTIME)
67 basicHeaderObj = BasicHeader(LOCALTIME)
68
68
69 try:
69 try:
70 fp = open(filename,'rb')
70 fp = open(filename,'rb')
71 except IOError:
71 except IOError:
72 print "The file %s can't be opened" %(filename)
72 print "The file %s can't be opened" %(filename)
73 return 0
73 return 0
74
74
75 sts = basicHeaderObj.read(fp)
75 sts = basicHeaderObj.read(fp)
76 fp.close()
76 fp.close()
77
77
78 if not(sts):
78 if not(sts):
79 print "Skipping the file %s because it has not a valid header" %(filename)
79 print "Skipping the file %s because it has not a valid header" %(filename)
80 return 0
80 return 0
81
81
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 return 0
83 return 0
84
84
85 return 1
85 return 1
86
86
87 def isTimeInRange(thisTime, startTime, endTime):
87 def isTimeInRange(thisTime, startTime, endTime):
88
88
89 if endTime >= startTime:
89 if endTime >= startTime:
90 if (thisTime < startTime) or (thisTime > endTime):
90 if (thisTime < startTime) or (thisTime > endTime):
91 return 0
91 return 0
92
92
93 return 1
93 return 1
94 else:
94 else:
95 if (thisTime < startTime) and (thisTime > endTime):
95 if (thisTime < startTime) and (thisTime > endTime):
96 return 0
96 return 0
97
97
98 return 1
98 return 1
99
99
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 """
101 """
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103
103
104 Inputs:
104 Inputs:
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106
106
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108
108
109 endDate : fecha final del rango seleccionado en formato datetime.date
109 endDate : fecha final del rango seleccionado en formato datetime.date
110
110
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112
112
113 endTime : tiempo final del rango seleccionado en formato datetime.time
113 endTime : tiempo final del rango seleccionado en formato datetime.time
114
114
115 Return:
115 Return:
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 fecha especificado, de lo contrario retorna False.
117 fecha especificado, de lo contrario retorna False.
118
118
119 Excepciones:
119 Excepciones:
120 Si el archivo no existe o no puede ser abierto
120 Si el archivo no existe o no puede ser abierto
121 Si la cabecera no puede ser leida.
121 Si la cabecera no puede ser leida.
122
122
123 """
123 """
124
124
125
125
126 try:
126 try:
127 fp = open(filename,'rb')
127 fp = open(filename,'rb')
128 except IOError:
128 except IOError:
129 print "The file %s can't be opened" %(filename)
129 print "The file %s can't be opened" %(filename)
130 return None
130 return None
131
131
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 systemHeaderObj = SystemHeader()
133 systemHeaderObj = SystemHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
135 processingHeaderObj = ProcessingHeader()
135 processingHeaderObj = ProcessingHeader()
136
136
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138
138
139 sts = firstBasicHeaderObj.read(fp)
139 sts = firstBasicHeaderObj.read(fp)
140
140
141 if not(sts):
141 if not(sts):
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 return None
143 return None
144
144
145 if not systemHeaderObj.read(fp):
145 if not systemHeaderObj.read(fp):
146 return None
146 return None
147
147
148 if not radarControllerHeaderObj.read(fp):
148 if not radarControllerHeaderObj.read(fp):
149 return None
149 return None
150
150
151 if not processingHeaderObj.read(fp):
151 if not processingHeaderObj.read(fp):
152 return None
152 return None
153
153
154 filesize = os.path.getsize(filename)
154 filesize = os.path.getsize(filename)
155
155
156 offset = processingHeaderObj.blockSize + 24 #header size
156 offset = processingHeaderObj.blockSize + 24 #header size
157
157
158 if filesize <= offset:
158 if filesize <= offset:
159 print "[Reading] %s: This file has not enough data" %filename
159 print "[Reading] %s: This file has not enough data" %filename
160 return None
160 return None
161
161
162 fp.seek(-offset, 2)
162 fp.seek(-offset, 2)
163
163
164 sts = lastBasicHeaderObj.read(fp)
164 sts = lastBasicHeaderObj.read(fp)
165
165
166 fp.close()
166 fp.close()
167
167
168 thisDatetime = lastBasicHeaderObj.datatime
168 thisDatetime = lastBasicHeaderObj.datatime
169 thisTime_last_block = thisDatetime.time()
169 thisTime_last_block = thisDatetime.time()
170
170
171 thisDatetime = firstBasicHeaderObj.datatime
171 thisDatetime = firstBasicHeaderObj.datatime
172 thisDate = thisDatetime.date()
172 thisDate = thisDatetime.date()
173 thisTime_first_block = thisDatetime.time()
173 thisTime_first_block = thisDatetime.time()
174
174
175 #General case
175 #General case
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 #-----------o----------------------------o-----------
177 #-----------o----------------------------o-----------
178 # startTime endTime
178 # startTime endTime
179
179
180 if endTime >= startTime:
180 if endTime >= startTime:
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 return None
182 return None
183
183
184 return thisDatetime
184 return thisDatetime
185
185
186 #If endTime < startTime then endTime belongs to the next day
186 #If endTime < startTime then endTime belongs to the next day
187
187
188
188
189 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 #-----------o----------------------------o-----------
190 #-----------o----------------------------o-----------
191 # endTime startTime
191 # endTime startTime
192
192
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 return None
194 return None
195
195
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 return None
197 return None
198
198
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 return None
200 return None
201
201
202 return thisDatetime
202 return thisDatetime
203
203
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 """
205 """
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207
207
208 Inputs:
208 Inputs:
209 folder : nombre completo del directorio.
209 folder : nombre completo del directorio.
210 Su formato deberia ser "/path_root/?YYYYDDD"
210 Su formato deberia ser "/path_root/?YYYYDDD"
211
211
212 siendo:
212 siendo:
213 YYYY : Anio (ejemplo 2015)
213 YYYY : Anio (ejemplo 2015)
214 DDD : Dia del anio (ejemplo 305)
214 DDD : Dia del anio (ejemplo 305)
215
215
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217
217
218 endDate : fecha final del rango seleccionado en formato datetime.date
218 endDate : fecha final del rango seleccionado en formato datetime.date
219
219
220 Return:
220 Return:
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 fecha especificado, de lo contrario retorna False.
222 fecha especificado, de lo contrario retorna False.
223 Excepciones:
223 Excepciones:
224 Si el directorio no tiene el formato adecuado
224 Si el directorio no tiene el formato adecuado
225 """
225 """
226
226
227 basename = os.path.basename(folder)
227 basename = os.path.basename(folder)
228
228
229 if not isRadarFolder(basename):
229 if not isRadarFolder(basename):
230 print "The folder %s has not the rigth format" %folder
230 print "The folder %s has not the rigth format" %folder
231 return 0
231 return 0
232
232
233 if startDate and endDate:
233 if startDate and endDate:
234 thisDate = getDateFromRadarFolder(basename)
234 thisDate = getDateFromRadarFolder(basename)
235
235
236 if thisDate < startDate:
236 if thisDate < startDate:
237 return 0
237 return 0
238
238
239 if thisDate > endDate:
239 if thisDate > endDate:
240 return 0
240 return 0
241
241
242 return 1
242 return 1
243
243
244 def isFileInDateRange(filename, startDate=None, endDate=None):
244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 """
245 """
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247
247
248 Inputs:
248 Inputs:
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250
250
251 Su formato deberia ser "?YYYYDDDsss"
251 Su formato deberia ser "?YYYYDDDsss"
252
252
253 siendo:
253 siendo:
254 YYYY : Anio (ejemplo 2015)
254 YYYY : Anio (ejemplo 2015)
255 DDD : Dia del anio (ejemplo 305)
255 DDD : Dia del anio (ejemplo 305)
256 sss : set
256 sss : set
257
257
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259
259
260 endDate : fecha final del rango seleccionado en formato datetime.date
260 endDate : fecha final del rango seleccionado en formato datetime.date
261
261
262 Return:
262 Return:
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 fecha especificado, de lo contrario retorna False.
264 fecha especificado, de lo contrario retorna False.
265 Excepciones:
265 Excepciones:
266 Si el archivo no tiene el formato adecuado
266 Si el archivo no tiene el formato adecuado
267 """
267 """
268
268
269 basename = os.path.basename(filename)
269 basename = os.path.basename(filename)
270
270
271 if not isRadarFile(basename):
271 if not isRadarFile(basename):
272 print "The filename %s has not the rigth format" %filename
272 print "The filename %s has not the rigth format" %filename
273 return 0
273 return 0
274
274
275 if startDate and endDate:
275 if startDate and endDate:
276 thisDate = getDateFromRadarFile(basename)
276 thisDate = getDateFromRadarFile(basename)
277
277
278 if thisDate < startDate:
278 if thisDate < startDate:
279 return 0
279 return 0
280
280
281 if thisDate > endDate:
281 if thisDate > endDate:
282 return 0
282 return 0
283
283
284 return 1
284 return 1
285
285
286 def getFileFromSet(path, ext, set):
286 def getFileFromSet(path, ext, set):
287 validFilelist = []
287 validFilelist = []
288 fileList = os.listdir(path)
288 fileList = os.listdir(path)
289
289
290 # 0 1234 567 89A BCDE
290 # 0 1234 567 89A BCDE
291 # H YYYY DDD SSS .ext
291 # H YYYY DDD SSS .ext
292
292
293 for thisFile in fileList:
293 for thisFile in fileList:
294 try:
294 try:
295 year = int(thisFile[1:5])
295 year = int(thisFile[1:5])
296 doy = int(thisFile[5:8])
296 doy = int(thisFile[5:8])
297 except:
297 except:
298 continue
298 continue
299
299
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 continue
301 continue
302
302
303 validFilelist.append(thisFile)
303 validFilelist.append(thisFile)
304
304
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306
306
307 if len(myfile)!= 0:
307 if len(myfile)!= 0:
308 return myfile[0]
308 return myfile[0]
309 else:
309 else:
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 print 'the filename %s does not exist'%filename
311 print 'the filename %s does not exist'%filename
312 print '...going to the last file: '
312 print '...going to the last file: '
313
313
314 if validFilelist:
314 if validFilelist:
315 validFilelist = sorted( validFilelist, key=str.lower )
315 validFilelist = sorted( validFilelist, key=str.lower )
316 return validFilelist[-1]
316 return validFilelist[-1]
317
317
318 return None
318 return None
319
319
320 def getlastFileFromPath(path, ext):
320 def getlastFileFromPath(path, ext):
321 """
321 """
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324
324
325 Input:
325 Input:
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 ext : extension de los files contenidos en una carpeta
327 ext : extension de los files contenidos en una carpeta
328
328
329 Return:
329 Return:
330 El ultimo file de una determinada carpeta, no se considera el path.
330 El ultimo file de una determinada carpeta, no se considera el path.
331 """
331 """
332 validFilelist = []
332 validFilelist = []
333 fileList = os.listdir(path)
333 fileList = os.listdir(path)
334
334
335 # 0 1234 567 89A BCDE
335 # 0 1234 567 89A BCDE
336 # H YYYY DDD SSS .ext
336 # H YYYY DDD SSS .ext
337
337
338 for thisFile in fileList:
338 for thisFile in fileList:
339
339
340 year = thisFile[1:5]
340 year = thisFile[1:5]
341 if not isNumber(year):
341 if not isNumber(year):
342 continue
342 continue
343
343
344 doy = thisFile[5:8]
344 doy = thisFile[5:8]
345 if not isNumber(doy):
345 if not isNumber(doy):
346 continue
346 continue
347
347
348 year = int(year)
348 year = int(year)
349 doy = int(doy)
349 doy = int(doy)
350
350
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 continue
352 continue
353
353
354 validFilelist.append(thisFile)
354 validFilelist.append(thisFile)
355
355
356 if validFilelist:
356 if validFilelist:
357 validFilelist = sorted( validFilelist, key=str.lower )
357 validFilelist = sorted( validFilelist, key=str.lower )
358 return validFilelist[-1]
358 return validFilelist[-1]
359
359
360 return None
360 return None
361
361
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 """
363 """
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 el path exacto de un determinado file.
366 el path exacto de un determinado file.
367
367
368 Example :
368 Example :
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370
370
371 Entonces la funcion prueba con las siguientes combinaciones
371 Entonces la funcion prueba con las siguientes combinaciones
372 .../.../y2009307367.ext
372 .../.../y2009307367.ext
373 .../.../Y2009307367.ext
373 .../.../Y2009307367.ext
374 .../.../x2009307/y2009307367.ext
374 .../.../x2009307/y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
376 .../.../X2009307/y2009307367.ext
376 .../.../X2009307/y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379
379
380 Return:
380 Return:
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 para el filename
383 para el filename
384 """
384 """
385 fullfilename = None
385 fullfilename = None
386 find_flag = False
386 find_flag = False
387 filename = None
387 filename = None
388
388
389 prefixDirList = [None,'d','D']
389 prefixDirList = [None,'d','D']
390 if ext.lower() == ".r": #voltage
390 if ext.lower() == ".r": #voltage
391 prefixFileList = ['d','D']
391 prefixFileList = ['d','D']
392 elif ext.lower() == ".pdata": #spectra
392 elif ext.lower() == ".pdata": #spectra
393 prefixFileList = ['p','P']
393 prefixFileList = ['p','P']
394 else:
394 else:
395 return None, filename
395 return None, filename
396
396
397 #barrido por las combinaciones posibles
397 #barrido por las combinaciones posibles
398 for prefixDir in prefixDirList:
398 for prefixDir in prefixDirList:
399 thispath = path
399 thispath = path
400 if prefixDir != None:
400 if prefixDir != None:
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 if foldercounter == 0:
402 if foldercounter == 0:
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 else:
404 else:
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409
409
410 if os.path.exists( fullfilename ): #verifico que exista
410 if os.path.exists( fullfilename ): #verifico que exista
411 find_flag = True
411 find_flag = True
412 break
412 break
413 if find_flag:
413 if find_flag:
414 break
414 break
415
415
416 if not(find_flag):
416 if not(find_flag):
417 return None, filename
417 return None, filename
418
418
419 return fullfilename, filename
419 return fullfilename, filename
420
420
421 def isRadarFolder(folder):
421 def isRadarFolder(folder):
422 try:
422 try:
423 year = int(folder[1:5])
423 year = int(folder[1:5])
424 doy = int(folder[5:8])
424 doy = int(folder[5:8])
425 except:
425 except:
426 return 0
426 return 0
427
427
428 return 1
428 return 1
429
429
430 def isRadarFile(file):
430 def isRadarFile(file):
431 try:
431 try:
432 year = int(file[1:5])
432 year = int(file[1:5])
433 doy = int(file[5:8])
433 doy = int(file[5:8])
434 set = int(file[8:11])
434 set = int(file[8:11])
435 except:
435 except:
436 return 0
436 return 0
437
437
438 return 1
438 return 1
439
439
440 def getDateFromRadarFile(file):
440 def getDateFromRadarFile(file):
441 try:
441 try:
442 year = int(file[1:5])
442 year = int(file[1:5])
443 doy = int(file[5:8])
443 doy = int(file[5:8])
444 set = int(file[8:11])
444 set = int(file[8:11])
445 except:
445 except:
446 return None
446 return None
447
447
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 return thisDate
449 return thisDate
450
450
451 def getDateFromRadarFolder(folder):
451 def getDateFromRadarFolder(folder):
452 try:
452 try:
453 year = int(folder[1:5])
453 year = int(folder[1:5])
454 doy = int(folder[5:8])
454 doy = int(folder[5:8])
455 except:
455 except:
456 return None
456 return None
457
457
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 return thisDate
459 return thisDate
460
460
461 class JRODataIO:
461 class JRODataIO:
462
462
463 c = 3E8
463 c = 3E8
464
464
465 isConfig = False
465 isConfig = False
466
466
467 basicHeaderObj = None
467 basicHeaderObj = None
468
468
469 systemHeaderObj = None
469 systemHeaderObj = None
470
470
471 radarControllerHeaderObj = None
471 radarControllerHeaderObj = None
472
472
473 processingHeaderObj = None
473 processingHeaderObj = None
474
474
475 dtype = None
475 dtype = None
476
476
477 pathList = []
477 pathList = []
478
478
479 filenameList = []
479 filenameList = []
480
480
481 filename = None
481 filename = None
482
482
483 ext = None
483 ext = None
484
484
485 flagIsNewFile = 1
485 flagIsNewFile = 1
486
486
487 flagDiscontinuousBlock = 0
487 flagDiscontinuousBlock = 0
488
488
489 flagIsNewBlock = 0
489 flagIsNewBlock = 0
490
490
491 fp = None
491 fp = None
492
492
493 firstHeaderSize = 0
493 firstHeaderSize = 0
494
494
495 basicHeaderSize = 24
495 basicHeaderSize = 24
496
496
497 versionFile = 1103
497 versionFile = 1103
498
498
499 fileSize = None
499 fileSize = None
500
500
501 # ippSeconds = None
501 # ippSeconds = None
502
502
503 fileSizeByHeader = None
503 fileSizeByHeader = None
504
504
505 fileIndex = None
505 fileIndex = None
506
506
507 profileIndex = None
507 profileIndex = None
508
508
509 blockIndex = None
509 blockIndex = None
510
510
511 nTotalBlocks = None
511 nTotalBlocks = None
512
512
513 maxTimeStep = 30
513 maxTimeStep = 30
514
514
515 lastUTTime = None
515 lastUTTime = None
516
516
517 datablock = None
517 datablock = None
518
518
519 dataOut = None
519 dataOut = None
520
520
521 blocksize = None
521 blocksize = None
522
522
523 getByBlock = False
523 getByBlock = False
524
524
525 def __init__(self):
525 def __init__(self):
526
526
527 raise NotImplementedError
527 raise NotImplementedError
528
528
529 def run(self):
529 def run(self):
530
530
531 raise NotImplementedError
531 raise NotImplementedError
532
532
533 def getDtypeWidth(self):
533 def getDtypeWidth(self):
534
534
535 dtype_index = get_dtype_index(self.dtype)
535 dtype_index = get_dtype_index(self.dtype)
536 dtype_width = get_dtype_width(dtype_index)
536 dtype_width = get_dtype_width(dtype_index)
537
537
538 return dtype_width
538 return dtype_width
539
539
540 def getAllowedArgs(self):
540 def getAllowedArgs(self):
541 return inspect.getargspec(self.run).args
541 return inspect.getargspec(self.run).args
542
542
543 class JRODataReader(JRODataIO):
543 class JRODataReader(JRODataIO):
544
544
545 firstTime = True
545 firstTime = True
546 online = 0
546 online = 0
547
547
548 realtime = 0
548 realtime = 0
549
549
550 nReadBlocks = 0
550 nReadBlocks = 0
551
551
552 delay = 10 #number of seconds waiting a new file
552 delay = 10 #number of seconds waiting a new file
553
553
554 nTries = 3 #quantity tries
554 nTries = 3 #quantity tries
555
555
556 nFiles = 3 #number of files for searching
556 nFiles = 3 #number of files for searching
557
557
558 path = None
558 path = None
559
559
560 foldercounter = 0
560 foldercounter = 0
561
561
562 flagNoMoreFiles = 0
562 flagNoMoreFiles = 0
563
563
564 datetimeList = []
564 datetimeList = []
565
565
566 __isFirstTimeOnline = 1
566 __isFirstTimeOnline = 1
567
567
568 __printInfo = True
568 __printInfo = True
569
569
570 profileIndex = None
570 profileIndex = None
571
571
572 nTxs = 1
572 nTxs = 1
573
573
574 txIndex = None
574 txIndex = None
575
575
576 #Added--------------------
576 #Added--------------------
577
577
578 selBlocksize = None
578 selBlocksize = None
579
579
580 selBlocktime = None
580 selBlocktime = None
581
581
582 onlineWithDate = False
582 onlineWithDate = False
583 def __init__(self):
583 def __init__(self):
584
584
585 """
585 """
586 This class is used to find data files
586 This class is used to find data files
587
587
588 Example:
588 Example:
589 reader = JRODataReader()
589 reader = JRODataReader()
590 fileList = reader.findDataFiles()
590 fileList = reader.findDataFiles()
591
591
592 """
592 """
593 pass
593 pass
594
594
595
595
596 def createObjByDefault(self):
596 def createObjByDefault(self):
597 """
597 """
598
598
599 """
599 """
600 raise NotImplementedError
600 raise NotImplementedError
601
601
602 def getBlockDimension(self):
602 def getBlockDimension(self):
603
603
604 raise NotImplementedError
604 raise NotImplementedError
605
605
606 def __searchFilesOffLine(self,
606 def __searchFilesOffLine(self,
607 path,
607 path,
608 startDate=None,
608 startDate=None,
609 endDate=None,
609 endDate=None,
610 startTime=datetime.time(0,0,0),
610 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
611 endTime=datetime.time(23,59,59),
612 set=None,
612 set=None,
613 expLabel='',
613 expLabel='',
614 ext='.r',
614 ext='.r',
615 queue=None,
615 queue=None,
616 cursor=None,
616 cursor=None,
617 skip=None,
617 skip=None,
618 walk=True):
618 walk=True):
619 self.filenameList = []
619 self.filenameList = []
620 self.datetimeList = []
620 self.datetimeList = []
621
621
622 pathList = []
622 pathList = []
623
623
624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625
625
626 if dateList == []:
626 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 return None, None
628 return None, None
629
629
630 if len(dateList) > 1:
630 if len(dateList) > 1:
631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 else:
632 else:
633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634
634
635 filenameList = []
635 filenameList = []
636 datetimeList = []
636 datetimeList = []
637
637
638 for thisPath in pathList:
638 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
639 # thisPath = pathList[pathDict[file]]
640
640
641 fileList = glob.glob1(thisPath, "*%s" %ext)
641 fileList = glob.glob1(thisPath, "*%s" %ext)
642 fileList.sort()
642 fileList.sort()
643
643
644 skippedFileList = []
644 skippedFileList = []
645
645
646 if cursor is not None and skip is not None:
646 if cursor is not None and skip is not None:
647 # if cursor*skip > len(fileList):
647 # if cursor*skip > len(fileList):
648 if skip == 0:
648 if skip == 0:
649 if queue is not None:
649 if queue is not None:
650 queue.put(len(fileList))
650 queue.put(len(fileList))
651 skippedFileList = []
651 skippedFileList = []
652 else:
652 else:
653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654
654
655 else:
655 else:
656 skippedFileList = fileList
656 skippedFileList = fileList
657
657
658 for file in skippedFileList:
658 for file in skippedFileList:
659
659
660 filename = os.path.join(thisPath,file)
660 filename = os.path.join(thisPath,file)
661
661
662 if not isFileInDateRange(filename, startDate, endDate):
662 if not isFileInDateRange(filename, startDate, endDate):
663 continue
663 continue
664
664
665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666
666
667 if not(thisDatetime):
667 if not(thisDatetime):
668 continue
668 continue
669
669
670 filenameList.append(filename)
670 filenameList.append(filename)
671 datetimeList.append(thisDatetime)
671 datetimeList.append(thisDatetime)
672
672
673 if not(filenameList):
673 if not(filenameList):
674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
675 return None, None
676
676
677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 print
678 print
679
679
680 for i in range(len(filenameList)):
680 for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682
682
683 self.filenameList = filenameList
683 self.filenameList = filenameList
684 self.datetimeList = datetimeList
684 self.datetimeList = datetimeList
685 return pathList, filenameList
685 return pathList, filenameList
686
686
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688
688
689 """
689 """
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
691 devuelve el archivo encontrado ademas de otros datos.
692
692
693 Input:
693 Input:
694 path : carpeta donde estan contenidos los files que contiene data
694 path : carpeta donde estan contenidos los files que contiene data
695
695
696 expLabel : Nombre del subexperimento (subfolder)
696 expLabel : Nombre del subexperimento (subfolder)
697
697
698 ext : extension de los files
698 ext : extension de los files
699
699
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701
701
702 Return:
702 Return:
703 directory : eL directorio donde esta el file encontrado
703 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
704 filename : el ultimo file de una determinada carpeta
705 year : el anho
705 year : el anho
706 doy : el numero de dia del anho
706 doy : el numero de dia del anho
707 set : el set del archivo
707 set : el set del archivo
708
708
709
709
710 """
710 """
711 pathList = None
711 pathList = None
712 filenameList = None
712 filenameList = None
713 if not os.path.isdir(path):
713 if not os.path.isdir(path):
714 return None, None, None, None, None, None
714 return None, None, None, None, None, None
715
715
716 dirList = []
716 dirList = []
717
717
718 if not walk:
718 if not walk:
719 fullpath = path
719 fullpath = path
720 foldercounter = 0
720 foldercounter = 0
721 else:
721 else:
722 # Filtra solo los directorios
722 # Filtra solo los directorios
723 for thisPath in os.listdir(path):
723 for thisPath in os.listdir(path):
724 if not os.path.isdir(os.path.join(path,thisPath)):
724 if not os.path.isdir(os.path.join(path,thisPath)):
725 continue
725 continue
726 if not isRadarFolder(thisPath):
726 if not isRadarFolder(thisPath):
727 continue
727 continue
728
728
729 dirList.append(thisPath)
729 dirList.append(thisPath)
730
730
731 if not(dirList):
731 if not(dirList):
732 return None, None, None, None, None, None
732 return None, None, None, None, None, None
733
733
734 dirList = sorted( dirList, key=str.lower )
734 dirList = sorted( dirList, key=str.lower )
735
735
736 doypath = dirList[-1]
736 doypath = dirList[-1]
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 fullpath = os.path.join(path, doypath, expLabel)
738 fullpath = os.path.join(path, doypath, expLabel)
739
739
740
740
741 print "[Reading] %s folder was found: " %(fullpath )
741 print "[Reading] %s folder was found: " %(fullpath )
742
742
743 if set == None:
743 if set == None:
744 filename = getlastFileFromPath(fullpath, ext)
744 filename = getlastFileFromPath(fullpath, ext)
745 else:
745 else:
746 filename = getFileFromSet(fullpath, ext, set)
746 filename = getFileFromSet(fullpath, ext, set)
747
747
748 if not(filename):
748 if not(filename):
749 return None, None, None, None, None, None
749 return None, None, None, None, None, None
750
750
751 print "[Reading] %s file was found" %(filename)
751 print "[Reading] %s file was found" %(filename)
752
752
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 return None, None, None, None, None, None
754 return None, None, None, None, None, None
755
755
756 year = int( filename[1:5] )
756 year = int( filename[1:5] )
757 doy = int( filename[5:8] )
757 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
758 set = int( filename[8:11] )
759
759
760 return fullpath, foldercounter, filename, year, doy, set
760 return fullpath, foldercounter, filename, year, doy, set
761
761
762 def __setNextFileOffline(self):
762 def __setNextFileOffline(self):
763
763
764 idFile = self.fileIndex
764 idFile = self.fileIndex
765
765
766 while (True):
766 while (True):
767 idFile += 1
767 idFile += 1
768 if not(idFile < len(self.filenameList)):
768 if not(idFile < len(self.filenameList)):
769 self.flagNoMoreFiles = 1
769 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
770 # print "[Reading] No more Files"
771 return 0
771 return 0
772
772
773 filename = self.filenameList[idFile]
773 filename = self.filenameList[idFile]
774
774
775 if not(self.__verifyFile(filename)):
775 if not(self.__verifyFile(filename)):
776 continue
776 continue
777
777
778 fileSize = os.path.getsize(filename)
778 fileSize = os.path.getsize(filename)
779 fp = open(filename,'rb')
779 fp = open(filename,'rb')
780 break
780 break
781
781
782 self.flagIsNewFile = 1
782 self.flagIsNewFile = 1
783 self.fileIndex = idFile
783 self.fileIndex = idFile
784 self.filename = filename
784 self.filename = filename
785 self.fileSize = fileSize
785 self.fileSize = fileSize
786 self.fp = fp
786 self.fp = fp
787
787
788 #print "[Reading] Setting the file: %s"%self.filename
788 #print "[Reading] Setting the file: %s"%self.filename
789
789
790 return 1
790 return 1
791
791
792 def __setNextFileOnline(self):
792 def __setNextFileOnline(self):
793 """
793 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
796 siguientes.
797
797
798 Affected:
798 Affected:
799 self.flagIsNewFile
799 self.flagIsNewFile
800 self.filename
800 self.filename
801 self.fileSize
801 self.fileSize
802 self.fp
802 self.fp
803 self.set
803 self.set
804 self.flagNoMoreFiles
804 self.flagNoMoreFiles
805
805
806 Return:
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
809
810 Excepciones:
810 Excepciones:
811 Si un determinado file no puede ser abierto
811 Si un determinado file no puede ser abierto
812 """
812 """
813
813
814 nFiles = 0
814 nFiles = 0
815 fileOk_flag = False
815 fileOk_flag = False
816 firstTime_flag = True
816 firstTime_flag = True
817
817
818 self.set += 1
818 self.set += 1
819
819
820 if self.set > 999:
820 if self.set > 999:
821 self.set = 0
821 self.set = 0
822 self.foldercounter += 1
822 self.foldercounter += 1
823
823
824 #busca el 1er file disponible
824 #busca el 1er file disponible
825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
826 if fullfilename:
826 if fullfilename:
827 if self.__verifyFile(fullfilename, False):
827 if self.__verifyFile(fullfilename, False):
828 fileOk_flag = True
828 fileOk_flag = True
829
829
830 #si no encuentra un file entonces espera y vuelve a buscar
830 #si no encuentra un file entonces espera y vuelve a buscar
831 if not(fileOk_flag):
831 if not(fileOk_flag):
832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
833
833
834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
835 tries = self.nTries
835 tries = self.nTries
836 else:
836 else:
837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
838
838
839 for nTries in range( tries ):
839 for nTries in range( tries ):
840 if firstTime_flag:
840 if firstTime_flag:
841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
842 sleep( self.delay )
842 sleep( self.delay )
843 else:
843 else:
844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
845
845
846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
847 if fullfilename:
847 if fullfilename:
848 if self.__verifyFile(fullfilename):
848 if self.__verifyFile(fullfilename):
849 fileOk_flag = True
849 fileOk_flag = True
850 break
850 break
851
851
852 if fileOk_flag:
852 if fileOk_flag:
853 break
853 break
854
854
855 firstTime_flag = False
855 firstTime_flag = False
856
856
857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
858 self.set += 1
858 self.set += 1
859
859
860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
861 self.set = 0
861 self.set = 0
862 self.doy += 1
862 self.doy += 1
863 self.foldercounter = 0
863 self.foldercounter = 0
864
864
865 if fileOk_flag:
865 if fileOk_flag:
866 self.fileSize = os.path.getsize( fullfilename )
866 self.fileSize = os.path.getsize( fullfilename )
867 self.filename = fullfilename
867 self.filename = fullfilename
868 self.flagIsNewFile = 1
868 self.flagIsNewFile = 1
869 if self.fp != None: self.fp.close()
869 if self.fp != None: self.fp.close()
870 self.fp = open(fullfilename, 'rb')
870 self.fp = open(fullfilename, 'rb')
871 self.flagNoMoreFiles = 0
871 self.flagNoMoreFiles = 0
872 # print '[Reading] Setting the file: %s' % fullfilename
872 # print '[Reading] Setting the file: %s' % fullfilename
873 else:
873 else:
874 self.fileSize = 0
874 self.fileSize = 0
875 self.filename = None
875 self.filename = None
876 self.flagIsNewFile = 0
876 self.flagIsNewFile = 0
877 self.fp = None
877 self.fp = None
878 self.flagNoMoreFiles = 1
878 self.flagNoMoreFiles = 1
879 # print '[Reading] No more files to read'
879 # print '[Reading] No more files to read'
880
880
881 return fileOk_flag
881 return fileOk_flag
882
882
883 def setNextFile(self):
883 def setNextFile(self):
884 if self.fp != None:
884 if self.fp != None:
885 self.fp.close()
885 self.fp.close()
886 if self.online:
886 if self.online:
887 newFile = self.__setNextFileOnline()
887 newFile = self.__setNextFileOnline()
888 else:
888 else:
889 newFile = self.__setNextFileOffline()
889 newFile = self.__setNextFileOffline()
890 if not(newFile):
890 if not(newFile):
891 if self.onlineWithDate is True:
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
892 self.onlineWithDate=False
893 self.online = True
893 self.online = True
894 self.firstTime = False
894 self.firstTime = False
895 self.setup(
895 self.setup(
896 path=self.path,
896 path=self.path,
897 startDate=self.startDate,
897 startDate=self.startDate,
898 endDate=self.endDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
899 startTime=self.startTime ,
900 endTime=self.endTime,
900 endTime=self.endTime,
901 set=self.set,
901 set=self.set,
902 expLabel=self.expLabel,
902 expLabel=self.expLabel,
903 ext=self.ext,
903 ext=self.ext,
904 online=self.online,
904 online=self.online,
905 delay=self.delay,
905 delay=self.delay,
906 walk=self.walk,
906 walk=self.walk,
907 getblock=self.getblock,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
911 blocktime=self.blocktime
912 )
912 )
913 return 1
913 return 1
914 print '[Reading] No more files to read'
914 print '[Reading] No more files to read'
915 return 0
915 return 0
916
916
917 if self.verbose:
917 if self.verbose:
918 print '[Reading] Setting the file: %s' % self.filename
918 print '[Reading] Setting the file: %s' % self.filename
919
919
920 self.__readFirstHeader()
920 self.__readFirstHeader()
921 self.nReadBlocks = 0
921 self.nReadBlocks = 0
922 return 1
922 return 1
923
923
924 def __waitNewBlock(self):
924 def __waitNewBlock(self):
925 """
925 """
926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
927
927
928 Si el modo de lectura es OffLine siempre retorn 0
928 Si el modo de lectura es OffLine siempre retorn 0
929 """
929 """
930 if not self.online:
930 if not self.online:
931 return 0
931 return 0
932
932
933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
934 return 0
934 return 0
935
935
936 currentPointer = self.fp.tell()
936 currentPointer = self.fp.tell()
937
937
938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939
939
940 for nTries in range( self.nTries ):
940 for nTries in range( self.nTries ):
941
941
942 self.fp.close()
942 self.fp.close()
943 self.fp = open( self.filename, 'rb' )
943 self.fp = open( self.filename, 'rb' )
944 self.fp.seek( currentPointer )
944 self.fp.seek( currentPointer )
945
945
946 self.fileSize = os.path.getsize( self.filename )
946 self.fileSize = os.path.getsize( self.filename )
947 currentSize = self.fileSize - currentPointer
947 currentSize = self.fileSize - currentPointer
948
948
949 if ( currentSize >= neededSize ):
949 if ( currentSize >= neededSize ):
950 self.basicHeaderObj.read(self.fp)
950 self.basicHeaderObj.read(self.fp)
951 return 1
951 return 1
952
952
953 if self.fileSize == self.fileSizeByHeader:
953 if self.fileSize == self.fileSizeByHeader:
954 # self.flagEoF = True
954 # self.flagEoF = True
955 return 0
955 return 0
956
956
957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 sleep( self.delay )
958 sleep( self.delay )
959
959
960
960
961 return 0
961 return 0
962
962
963 def waitDataBlock(self,pointer_location):
963 def waitDataBlock(self,pointer_location):
964
964
965 currentPointer = pointer_location
965 currentPointer = pointer_location
966
966
967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
968
968
969 for nTries in range( self.nTries ):
969 for nTries in range( self.nTries ):
970 self.fp.close()
970 self.fp.close()
971 self.fp = open( self.filename, 'rb' )
971 self.fp = open( self.filename, 'rb' )
972 self.fp.seek( currentPointer )
972 self.fp.seek( currentPointer )
973
973
974 self.fileSize = os.path.getsize( self.filename )
974 self.fileSize = os.path.getsize( self.filename )
975 currentSize = self.fileSize - currentPointer
975 currentSize = self.fileSize - currentPointer
976
976
977 if ( currentSize >= neededSize ):
977 if ( currentSize >= neededSize ):
978 return 1
978 return 1
979
979
980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
981 sleep( self.delay )
981 sleep( self.delay )
982
982
983 return 0
983 return 0
984
984
985 def __jumpToLastBlock(self):
985 def __jumpToLastBlock(self):
986
986
987 if not(self.__isFirstTimeOnline):
987 if not(self.__isFirstTimeOnline):
988 return
988 return
989
989
990 csize = self.fileSize - self.fp.tell()
990 csize = self.fileSize - self.fp.tell()
991 blocksize = self.processingHeaderObj.blockSize
991 blocksize = self.processingHeaderObj.blockSize
992
992
993 #salta el primer bloque de datos
993 #salta el primer bloque de datos
994 if csize > self.processingHeaderObj.blockSize:
994 if csize > self.processingHeaderObj.blockSize:
995 self.fp.seek(self.fp.tell() + blocksize)
995 self.fp.seek(self.fp.tell() + blocksize)
996 else:
996 else:
997 return
997 return
998
998
999 csize = self.fileSize - self.fp.tell()
999 csize = self.fileSize - self.fp.tell()
1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 while True:
1001 while True:
1002
1002
1003 if self.fp.tell()<self.fileSize:
1003 if self.fp.tell()<self.fileSize:
1004 self.fp.seek(self.fp.tell() + neededsize)
1004 self.fp.seek(self.fp.tell() + neededsize)
1005 else:
1005 else:
1006 self.fp.seek(self.fp.tell() - neededsize)
1006 self.fp.seek(self.fp.tell() - neededsize)
1007 break
1007 break
1008
1008
1009 # csize = self.fileSize - self.fp.tell()
1009 # csize = self.fileSize - self.fp.tell()
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 # factor = int(csize/neededsize)
1011 # factor = int(csize/neededsize)
1012 # if factor > 0:
1012 # if factor > 0:
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014
1014
1015 self.flagIsNewFile = 0
1015 self.flagIsNewFile = 0
1016 self.__isFirstTimeOnline = 0
1016 self.__isFirstTimeOnline = 0
1017
1017
1018 def __setNewBlock(self):
1018 def __setNewBlock(self):
1019 #if self.server is None:
1019 #if self.server is None:
1020 if self.fp == None:
1020 if self.fp == None:
1021 return 0
1021 return 0
1022
1022
1023 # if self.online:
1023 # if self.online:
1024 # self.__jumpToLastBlock()
1024 # self.__jumpToLastBlock()
1025
1025
1026 if self.flagIsNewFile:
1026 if self.flagIsNewFile:
1027 self.lastUTTime = self.basicHeaderObj.utc
1027 self.lastUTTime = self.basicHeaderObj.utc
1028 return 1
1028 return 1
1029
1029
1030 if self.realtime:
1030 if self.realtime:
1031 self.flagDiscontinuousBlock = 1
1031 self.flagDiscontinuousBlock = 1
1032 if not(self.setNextFile()):
1032 if not(self.setNextFile()):
1033 return 0
1033 return 0
1034 else:
1034 else:
1035 return 1
1035 return 1
1036 #if self.server is None:
1036 #if self.server is None:
1037 currentSize = self.fileSize - self.fp.tell()
1037 currentSize = self.fileSize - self.fp.tell()
1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1039 if (currentSize >= neededSize):
1039 if (currentSize >= neededSize):
1040 self.basicHeaderObj.read(self.fp)
1040 self.basicHeaderObj.read(self.fp)
1041 self.lastUTTime = self.basicHeaderObj.utc
1041 self.lastUTTime = self.basicHeaderObj.utc
1042 return 1
1042 return 1
1043 # else:
1043 # else:
1044 # self.basicHeaderObj.read(self.zHeader)
1044 # self.basicHeaderObj.read(self.zHeader)
1045 # self.lastUTTime = self.basicHeaderObj.utc
1045 # self.lastUTTime = self.basicHeaderObj.utc
1046 # return 1
1046 # return 1
1047 if self.__waitNewBlock():
1047 if self.__waitNewBlock():
1048 self.lastUTTime = self.basicHeaderObj.utc
1048 self.lastUTTime = self.basicHeaderObj.utc
1049 return 1
1049 return 1
1050 #if self.server is None:
1050 #if self.server is None:
1051 if not(self.setNextFile()):
1051 if not(self.setNextFile()):
1052 return 0
1052 return 0
1053
1053
1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1055 self.lastUTTime = self.basicHeaderObj.utc
1055 self.lastUTTime = self.basicHeaderObj.utc
1056
1056
1057 self.flagDiscontinuousBlock = 0
1057 self.flagDiscontinuousBlock = 0
1058
1058
1059 if deltaTime > self.maxTimeStep:
1059 if deltaTime > self.maxTimeStep:
1060 self.flagDiscontinuousBlock = 1
1060 self.flagDiscontinuousBlock = 1
1061
1061
1062 return 1
1062 return 1
1063
1063
1064 def readNextBlock(self):
1064 def readNextBlock(self):
1065
1065
1066 #Skip block out of startTime and endTime
1066 #Skip block out of startTime and endTime
1067 while True:
1067 while True:
1068 if not(self.__setNewBlock()):
1068 if not(self.__setNewBlock()):
1069 print 'returning'
1069 print 'returning'
1070 return 0
1070 return 0
1071 if not(self.readBlock()):
1071 if not(self.readBlock()):
1072 return 0
1072 return 0
1073 self.getBasicHeader()
1073 self.getBasicHeader()
1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075
1075
1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1077 self.processingHeaderObj.dataBlocksPerFile,
1077 self.processingHeaderObj.dataBlocksPerFile,
1078 self.dataOut.datatime.ctime())
1078 self.dataOut.datatime.ctime())
1079 continue
1079 continue
1080
1080
1081 break
1081 break
1082
1082
1083 if self.verbose:
1083 if self.verbose:
1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 self.processingHeaderObj.dataBlocksPerFile,
1085 self.processingHeaderObj.dataBlocksPerFile,
1086 self.dataOut.datatime.ctime())
1086 self.dataOut.datatime.ctime())
1087 return 1
1087 return 1
1088
1088
1089 def __readFirstHeader(self):
1089 def __readFirstHeader(self):
1090
1090
1091 self.basicHeaderObj.read(self.fp)
1091 self.basicHeaderObj.read(self.fp)
1092 self.systemHeaderObj.read(self.fp)
1092 self.systemHeaderObj.read(self.fp)
1093 self.radarControllerHeaderObj.read(self.fp)
1093 self.radarControllerHeaderObj.read(self.fp)
1094 self.processingHeaderObj.read(self.fp)
1094 self.processingHeaderObj.read(self.fp)
1095
1095
1096 self.firstHeaderSize = self.basicHeaderObj.size
1096 self.firstHeaderSize = self.basicHeaderObj.size
1097
1097
1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1099 if datatype == 0:
1099 if datatype == 0:
1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1101 elif datatype == 1:
1101 elif datatype == 1:
1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1103 elif datatype == 2:
1103 elif datatype == 2:
1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1105 elif datatype == 3:
1105 elif datatype == 3:
1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1107 elif datatype == 4:
1107 elif datatype == 4:
1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1109 elif datatype == 5:
1109 elif datatype == 5:
1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1111 else:
1111 else:
1112 raise ValueError, 'Data type was not defined'
1112 raise ValueError, 'Data type was not defined'
1113
1113
1114 self.dtype = datatype_str
1114 self.dtype = datatype_str
1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 self.getBlockDimension()
1119 self.getBlockDimension()
1120
1120
1121 def __verifyFile(self, filename, msgFlag=True):
1121 def __verifyFile(self, filename, msgFlag=True):
1122
1122
1123 msg = None
1123 msg = None
1124
1124
1125 try:
1125 try:
1126 fp = open(filename, 'rb')
1126 fp = open(filename, 'rb')
1127 except IOError:
1127 except IOError:
1128
1128
1129 if msgFlag:
1129 if msgFlag:
1130 print "[Reading] File %s can't be opened" % (filename)
1130 print "[Reading] File %s can't be opened" % (filename)
1131
1131
1132 return False
1132 return False
1133
1133
1134 currentPosition = fp.tell()
1134 currentPosition = fp.tell()
1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1136
1136
1137 if neededSize == 0:
1137 if neededSize == 0:
1138 basicHeaderObj = BasicHeader(LOCALTIME)
1138 basicHeaderObj = BasicHeader(LOCALTIME)
1139 systemHeaderObj = SystemHeader()
1139 systemHeaderObj = SystemHeader()
1140 radarControllerHeaderObj = RadarControllerHeader()
1140 radarControllerHeaderObj = RadarControllerHeader()
1141 processingHeaderObj = ProcessingHeader()
1141 processingHeaderObj = ProcessingHeader()
1142
1142
1143 if not( basicHeaderObj.read(fp) ):
1143 if not( basicHeaderObj.read(fp) ):
1144 fp.close()
1144 fp.close()
1145 return False
1145 return False
1146
1146
1147 if not( systemHeaderObj.read(fp) ):
1147 if not( systemHeaderObj.read(fp) ):
1148 fp.close()
1148 fp.close()
1149 return False
1149 return False
1150
1150
1151 if not( radarControllerHeaderObj.read(fp) ):
1151 if not( radarControllerHeaderObj.read(fp) ):
1152 fp.close()
1152 fp.close()
1153 return False
1153 return False
1154
1154
1155 if not( processingHeaderObj.read(fp) ):
1155 if not( processingHeaderObj.read(fp) ):
1156 fp.close()
1156 fp.close()
1157 return False
1157 return False
1158
1158
1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1160 else:
1160 else:
1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1162
1162
1163 fp.close()
1163 fp.close()
1164
1164
1165 fileSize = os.path.getsize(filename)
1165 fileSize = os.path.getsize(filename)
1166 currentSize = fileSize - currentPosition
1166 currentSize = fileSize - currentPosition
1167
1167
1168 if currentSize < neededSize:
1168 if currentSize < neededSize:
1169 if msgFlag and (msg != None):
1169 if msgFlag and (msg != None):
1170 print msg
1170 print msg
1171 return False
1171 return False
1172
1172
1173 return True
1173 return True
1174
1174
1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1176
1176
1177 path_empty = True
1177 path_empty = True
1178
1178
1179 dateList = []
1179 dateList = []
1180 pathList = []
1180 pathList = []
1181
1181
1182 multi_path = path.split(',')
1182 multi_path = path.split(',')
1183
1183
1184 if not walk:
1184 if not walk:
1185
1185
1186 for single_path in multi_path:
1186 for single_path in multi_path:
1187
1187
1188 if not os.path.isdir(single_path):
1188 if not os.path.isdir(single_path):
1189 continue
1189 continue
1190
1190
1191 fileList = glob.glob1(single_path, "*"+ext)
1191 fileList = glob.glob1(single_path, "*"+ext)
1192
1192
1193 if not fileList:
1193 if not fileList:
1194 continue
1194 continue
1195
1195
1196 path_empty = False
1196 path_empty = False
1197
1197
1198 fileList.sort()
1198 fileList.sort()
1199
1199
1200 for thisFile in fileList:
1200 for thisFile in fileList:
1201
1201
1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1203 continue
1203 continue
1204
1204
1205 if not isRadarFile(thisFile):
1205 if not isRadarFile(thisFile):
1206 continue
1206 continue
1207
1207
1208 if not isFileInDateRange(thisFile, startDate, endDate):
1208 if not isFileInDateRange(thisFile, startDate, endDate):
1209 continue
1209 continue
1210
1210
1211 thisDate = getDateFromRadarFile(thisFile)
1211 thisDate = getDateFromRadarFile(thisFile)
1212
1212
1213 if thisDate in dateList:
1213 if thisDate in dateList:
1214 continue
1214 continue
1215
1215
1216 dateList.append(thisDate)
1216 dateList.append(thisDate)
1217 pathList.append(single_path)
1217 pathList.append(single_path)
1218
1218
1219 else:
1219 else:
1220 for single_path in multi_path:
1220 for single_path in multi_path:
1221
1221
1222 if not os.path.isdir(single_path):
1222 if not os.path.isdir(single_path):
1223 continue
1223 continue
1224
1224
1225 dirList = []
1225 dirList = []
1226
1226
1227 for thisPath in os.listdir(single_path):
1227 for thisPath in os.listdir(single_path):
1228
1228
1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1230 continue
1230 continue
1231
1231
1232 if not isRadarFolder(thisPath):
1232 if not isRadarFolder(thisPath):
1233 continue
1233 continue
1234
1234
1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1236 continue
1236 continue
1237
1237
1238 dirList.append(thisPath)
1238 dirList.append(thisPath)
1239
1239
1240 if not dirList:
1240 if not dirList:
1241 continue
1241 continue
1242
1242
1243 dirList.sort()
1243 dirList.sort()
1244
1244
1245 for thisDir in dirList:
1245 for thisDir in dirList:
1246
1246
1247 datapath = os.path.join(single_path, thisDir, expLabel)
1247 datapath = os.path.join(single_path, thisDir, expLabel)
1248 fileList = glob.glob1(datapath, "*"+ext)
1248 fileList = glob.glob1(datapath, "*"+ext)
1249
1249
1250 if not fileList:
1250 if not fileList:
1251 continue
1251 continue
1252
1252
1253 path_empty = False
1253 path_empty = False
1254
1254
1255 thisDate = getDateFromRadarFolder(thisDir)
1255 thisDate = getDateFromRadarFolder(thisDir)
1256
1256
1257 pathList.append(datapath)
1257 pathList.append(datapath)
1258 dateList.append(thisDate)
1258 dateList.append(thisDate)
1259
1259
1260 dateList.sort()
1260 dateList.sort()
1261
1261
1262 if walk:
1262 if walk:
1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1264 else:
1264 else:
1265 pattern_path = multi_path[0]
1265 pattern_path = multi_path[0]
1266
1266
1267 if path_empty:
1267 if path_empty:
1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1269 else:
1269 else:
1270 if not dateList:
1270 if not dateList:
1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1272
1272
1273 if include_path:
1273 if include_path:
1274 return dateList, pathList
1274 return dateList, pathList
1275
1275
1276 return dateList
1276 return dateList
1277
1277
1278 def setup(self,
1278 def setup(self,
1279 path=None,
1279 path=None,
1280 startDate=None,
1280 startDate=None,
1281 endDate=None,
1281 endDate=None,
1282 startTime=datetime.time(0,0,0),
1282 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1283 endTime=datetime.time(23,59,59),
1284 set=None,
1284 set=None,
1285 expLabel = "",
1285 expLabel = "",
1286 ext = None,
1286 ext = None,
1287 online = False,
1287 online = False,
1288 delay = 60,
1288 delay = 60,
1289 walk = True,
1289 walk = True,
1290 getblock = False,
1290 getblock = False,
1291 nTxs = 1,
1291 nTxs = 1,
1292 realtime=False,
1292 realtime=False,
1293 blocksize=None,
1293 blocksize=None,
1294 blocktime=None,
1294 blocktime=None):
1295 queue=None,
1296 skip=None,
1297 cursor=None,
1298 warnings=True,
1299 verbose=True,
1300 server=None):
1301 if server is not None:
1302 if 'tcp://' in server:
1303 address = server
1304 else:
1305 address = 'ipc:///tmp/%s' % server
1306 self.server = address
1307 self.context = zmq.Context()
1308 self.receiver = self.context.socket(zmq.PULL)
1309 self.receiver.connect(self.server)
1310 time.sleep(0.5)
1311 print '[Starting] ReceiverData from {}'.format(self.server)
1312 else:
1313 self.server = None
1314 if path == None:
1315 raise ValueError, "[Reading] The path is not valid"
1316
1317 if ext == None:
1318 ext = self.ext
1319
1320 if online:
1321 print "[Reading] Searching files in online mode..."
1322
1323 for nTries in range( self.nTries ):
1324 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1325
1326 if fullpath:
1327 break
1328
1329 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1330 sleep( self.delay )
1331
1332 if not(fullpath):
1333 print "[Reading] There 'isn't any valid file in %s" % path
1334 return
1335
1336 self.year = year
1337 self.doy = doy
1338 self.set = set - 1
1339 self.path = path
1340 self.foldercounter = foldercounter
1341 last_set = None
1342 else:
1343 print "[Reading] Searching files in offline mode ..."
1344 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1345 startTime=startTime, endTime=endTime,
1346 set=set, expLabel=expLabel, ext=ext,
1347 walk=walk, cursor=cursor,
1348 skip=skip, queue=queue)
1349
1350 <<<<<<< HEAD
1351 if not(pathList):
1352 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1353 # datetime.datetime.combine(startDate,startTime).ctime(),
1354 # datetime.datetime.combine(endDate,endTime).ctime())
1355
1356 # sys.exit(-1)
1357
1295
1358 self.fileIndex = -1
1359 self.pathList = []
1360 self.filenameList = []
1361 return
1362
1363 =======
1364 if path == None:
1296 if path == None:
1365 raise ValueError, "[Reading] The path is not valid"
1297 raise ValueError, "[Reading] The path is not valid"
1366
1298
1367
1299
1368 if ext == None:
1300 if ext == None:
1369 ext = self.ext
1301 ext = self.ext
1370
1302
1371 self.path = path
1303 self.path = path
1372 self.startDate = startDate
1304 self.startDate = startDate
1373 self.endDate = endDate
1305 self.endDate = endDate
1374 self.startTime = startTime
1306 self.startTime = startTime
1375 self.endTime = endTime
1307 self.endTime = endTime
1376 self.set = set
1308 self.set = set
1377 self.expLabel = expLabel
1309 self.expLabel = expLabel
1378 self.ext = ext
1310 self.ext = ext
1379 self.online = online
1311 self.online = online
1380 self.delay = delay
1312 self.delay = delay
1381 self.walk = walk
1313 self.walk = walk
1382 self.getblock = getblock
1314 self.getblock = getblock
1383 self.nTxs = nTxs
1315 self.nTxs = nTxs
1384 self.realtime = realtime
1316 self.realtime = realtime
1385 self.blocksize = blocksize
1317 self.blocksize = blocksize
1386 self.blocktime = blocktime
1318 self.blocktime = blocktime
1387
1319
1388
1320
1389 if self.firstTime is True:
1321 if self.firstTime is True:
1390 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1322 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1391 startTime=startTime, endTime=endTime,
1323 startTime=startTime, endTime=endTime,
1392 set=set, expLabel=expLabel, ext=ext,
1324 set=set, expLabel=expLabel, ext=ext,
1393 walk=walk)
1325 walk=walk)
1394 filenameList = filenameList[:-1]
1326 filenameList = filenameList[:-1]
1395
1327
1396 if pathList is not None and filenameList is not None and online:
1328 if pathList is not None and filenameList is not None and online:
1397 self.onlineWithDate = True
1329 self.onlineWithDate = True
1398 online = False
1330 online = False
1399 self.fileIndex = -1
1331 self.fileIndex = -1
1400 self.pathList = pathList
1332 self.pathList = pathList
1401 self.filenameList = filenameList
1333 self.filenameList = filenameList
1402 file_name = os.path.basename(filenameList[-1])
1334 file_name = os.path.basename(filenameList[-1])
1403 basename, ext = os.path.splitext(file_name)
1335 basename, ext = os.path.splitext(file_name)
1404 last_set = int(basename[-3:])
1336 last_set = int(basename[-3:])
1405
1337
1406 if online:
1338 if online:
1407 print "[Reading] Searching files in online mode..."
1339 print "[Reading] Searching files in online mode..."
1408
1340
1409 for nTries in range(self.nTries):
1341 for nTries in range(self.nTries):
1410 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1342 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1411 expLabel=expLabel,
1343 expLabel=expLabel,
1412 ext=ext,
1344 ext=ext,
1413 walk=walk,
1345 walk=walk,
1414 startDate=startDate,
1346 startDate=startDate,
1415 startTime=startTime,
1347 startTime=startTime,
1416 set=set)
1348 set=set)
1417
1349
1418 if fullpath:
1350 if fullpath:
1419 break
1351 break
1420 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1352 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1421 sleep( self.delay )
1353 sleep( self.delay )
1422
1354
1423 if not(fullpath):
1355 if not(fullpath):
1424 print "[Reading] There 'isn't any valid file in %s" % path
1356 print "[Reading] There 'isn't any valid file in %s" % path
1425 return
1357 return
1426
1358
1427 self.year = year
1359 self.year = year
1428 self.doy = doy
1360 self.doy = doy
1429 self.set = set - 1
1361 self.set = set - 1
1430 self.path = path
1362 self.path = path
1431 self.foldercounter = foldercounter
1363 self.foldercounter = foldercounter
1432 last_set = None
1364 last_set = None
1433
1365
1434 else:
1366 else:
1435 print "[Reading] Searching files in offline mode ..."
1367 print "[Reading] Searching files in offline mode ..."
1436 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1368 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1437 startTime=startTime, endTime=endTime,
1369 startTime=startTime, endTime=endTime,
1438 set=set, expLabel=expLabel, ext=ext,
1370 set=set, expLabel=expLabel, ext=ext,
1439 walk=walk)
1371 walk=walk)
1440
1372
1441 if not(pathList):
1373 if not(pathList):
1442 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1374 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1443 # datetime.datetime.combine(startDate,startTime).ctime(),
1375 # datetime.datetime.combine(startDate,startTime).ctime(),
1444 # datetime.datetime.combine(endDate,endTime).ctime())
1376 # datetime.datetime.combine(endDate,endTime).ctime())
1445
1377
1446 # sys.exit(-1)
1378 # sys.exit(-1)
1447
1379
1448 self.fileIndex = -1
1380 self.fileIndex = -1
1449 self.pathList = []
1381 self.pathList = []
1450 self.filenameList = []
1382 self.filenameList = []
1451 return
1383 return
1452
1384
1453 self.fileIndex = -1
1385 self.fileIndex = -1
1454 self.pathList = pathList
1386 self.pathList = pathList
1455 self.filenameList = filenameList
1387 self.filenameList = filenameList
1456 file_name = os.path.basename(filenameList[-1])
1388 file_name = os.path.basename(filenameList[-1])
1457 basename, ext = os.path.splitext(file_name)
1389 basename, ext = os.path.splitext(file_name)
1458 last_set = int(basename[-3:])
1390 last_set = int(basename[-3:])
1459
1391
1460
1392
1461 self.online = online
1393 self.online = online
1462 self.realtime = realtime
1394 self.realtime = realtime
1463 self.delay = delay
1395 self.delay = delay
1464 ext = ext.lower()
1396 ext = ext.lower()
1465 self.ext = ext
1397 self.ext = ext
1466 self.getByBlock = getblock
1398 self.getByBlock = getblock
1467 self.nTxs = nTxs
1399 self.nTxs = nTxs
1468 self.startTime = startTime
1400 self.startTime = startTime
1469 self.endTime = endTime
1401 self.endTime = endTime
1470
1402
1471
1403
1472 #Added-----------------
1404 #Added-----------------
1473 self.selBlocksize = blocksize
1405 self.selBlocksize = blocksize
1474 self.selBlocktime = blocktime
1406 self.selBlocktime = blocktime
1475
1407
1476
1408
1477 if not(self.setNextFile()):
1409 if not(self.setNextFile()):
1478 if (startDate!=None) and (endDate!=None):
1410 if (startDate!=None) and (endDate!=None):
1479 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1411 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1480 elif startDate != None:
1412 elif startDate != None:
1481 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1413 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1482 else:
1414 else:
1483 print "[Reading] No files"
1415 print "[Reading] No files"
1484
1416
1485 >>>>>>> online_data_hour
1486 self.fileIndex = -1
1417 self.fileIndex = -1
1487 self.pathList = pathList
1418 self.pathList = []
1488 self.filenameList = filenameList
1419 self.filenameList = []
1489 file_name = os.path.basename(filenameList[-1])
1420 return
1490 basename, ext = os.path.splitext(file_name)
1491 last_set = int(basename[-3:])
1492
1493 self.online = online
1494 self.realtime = realtime
1495 self.delay = delay
1496 ext = ext.lower()
1497 self.ext = ext
1498 self.getByBlock = getblock
1499 self.nTxs = nTxs
1500 self.startTime = startTime
1501 self.endTime = endTime
1502
1503 #Added-----------------
1504 self.selBlocksize = blocksize
1505 self.selBlocktime = blocktime
1506
1507 # Verbose-----------
1508 self.verbose = verbose
1509 self.warnings = warnings
1510
1511 <<<<<<< HEAD
1512 if not(self.setNextFile()):
1513 if (startDate!=None) and (endDate!=None):
1514 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1515 elif startDate != None:
1516 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1517 else:
1518 print "[Reading] No files"
1519
1520 self.fileIndex = -1
1521 self.pathList = []
1522 self.filenameList = []
1523 return
1524
1525 # self.getBasicHeader()
1526
1421
1527 if last_set != None:
1528 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1529 return
1530 =======
1531 # self.getBasicHeader()
1422 # self.getBasicHeader()
1532
1423
1533 if last_set != None:
1424 if last_set != None:
1534 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1425 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1535 return
1426 return
1536 >>>>>>> online_data_hour
1537
1427
1538 def getBasicHeader(self):
1428 def getBasicHeader(self):
1539
1429
1540 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1430 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1541
1431
1542 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1432 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1543
1433
1544 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1434 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1545
1435
1546 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1436 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1547
1437
1548 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1438 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1549
1439
1550 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1440 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1551
1441
1552 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1442 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1553
1443
1554 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1444 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1555
1445
1556
1446
1557 def getFirstHeader(self):
1447 def getFirstHeader(self):
1558
1448
1559 raise NotImplementedError
1449 raise NotImplementedError
1560
1450
1561 def getData(self):
1451 def getData(self):
1562
1452
1563 raise NotImplementedError
1453 raise NotImplementedError
1564
1454
1565 def hasNotDataInBuffer(self):
1455 def hasNotDataInBuffer(self):
1566
1456
1567 raise NotImplementedError
1457 raise NotImplementedError
1568
1458
1569 def readBlock(self):
1459 def readBlock(self):
1570
1460
1571 raise NotImplementedError
1461 raise NotImplementedError
1572
1462
1573 def isEndProcess(self):
1463 def isEndProcess(self):
1574
1464
1575 return self.flagNoMoreFiles
1465 return self.flagNoMoreFiles
1576
1466
1577 def printReadBlocks(self):
1467 def printReadBlocks(self):
1578
1468
1579 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1469 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1580
1470
1581 def printTotalBlocks(self):
1471 def printTotalBlocks(self):
1582
1472
1583 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1473 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1584
1474
1585 def printNumberOfBlock(self):
1475 def printNumberOfBlock(self):
1586
1476
1587 if self.flagIsNewBlock:
1477 if self.flagIsNewBlock:
1588 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1478 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1589 self.processingHeaderObj.dataBlocksPerFile,
1479 self.processingHeaderObj.dataBlocksPerFile,
1590 self.dataOut.datatime.ctime())
1480 self.dataOut.datatime.ctime())
1591
1481
1592 def printInfo(self):
1482 def printInfo(self):
1593
1483
1594 if self.__printInfo == False:
1484 if self.__printInfo == False:
1595 return
1485 return
1596
1486
1597 self.basicHeaderObj.printInfo()
1487 self.basicHeaderObj.printInfo()
1598 self.systemHeaderObj.printInfo()
1488 self.systemHeaderObj.printInfo()
1599 self.radarControllerHeaderObj.printInfo()
1489 self.radarControllerHeaderObj.printInfo()
1600 self.processingHeaderObj.printInfo()
1490 self.processingHeaderObj.printInfo()
1601
1491
1602 self.__printInfo = False
1492 self.__printInfo = False
1603
1493
1604
1494
1605 def run(self,
1495 def run(self,
1606 path=None,
1496 path=None,
1607 startDate=None,
1497 startDate=None,
1608 endDate=None,
1498 endDate=None,
1609 startTime=datetime.time(0,0,0),
1499 startTime=datetime.time(0,0,0),
1610 endTime=datetime.time(23,59,59),
1500 endTime=datetime.time(23,59,59),
1611 set=None,
1501 set=None,
1612 expLabel = "",
1502 expLabel = "",
1613 ext = None,
1503 ext = None,
1614 online = False,
1504 online = False,
1615 delay = 60,
1505 delay = 60,
1616 walk = True,
1506 walk = True,
1617 getblock = False,
1507 getblock = False,
1618 nTxs = 1,
1508 nTxs = 1,
1619 realtime=False,
1509 realtime=False,
1620 blocksize=None,
1510 blocksize=None,
1621 blocktime=None,
1511 blocktime=None,
1622 queue=None,
1512 queue=None,
1623 skip=None,
1513 skip=None,
1624 cursor=None,
1514 cursor=None,
1625 warnings=True,
1515 warnings=True,
1626 server=None,
1516 server=None,
1627 verbose=True, **kwargs):
1517 verbose=True, **kwargs):
1628
1518
1629 if not(self.isConfig):
1519 if not(self.isConfig):
1630 <<<<<<< HEAD
1520 <<<<<<< HEAD
1631 # self.dataOut = dataOut
1521 # self.dataOut = dataOut
1632 self.setup( path=path,
1522 self.setup( path=path,
1633 startDate=startDate,
1523 startDate=startDate,
1634 endDate=endDate,
1524 endDate=endDate,
1635 startTime=startTime,
1525 startTime=startTime,
1636 endTime=endTime,
1526 endTime=endTime,
1637 set=set,
1527 set=set,
1638 expLabel=expLabel,
1528 expLabel=expLabel,
1639 ext=ext,
1529 ext=ext,
1640 online=online,
1530 online=online,
1641 delay=delay,
1531 delay=delay,
1642 walk=walk,
1532 walk=walk,
1643 getblock=getblock,
1533 getblock=getblock,
1644 nTxs=nTxs,
1534 nTxs=nTxs,
1645 realtime=realtime,
1535 realtime=realtime,
1646 blocksize=blocksize,
1536 blocksize=blocksize,
1647 blocktime=blocktime,
1537 blocktime=blocktime,
1648 queue=queue,
1538 queue=queue,
1649 skip=skip,
1539 skip=skip,
1650 cursor=cursor,
1540 cursor=cursor,
1651 warnings=warnings,
1541 warnings=warnings,
1652 server=server,
1542 server=server,
1653 verbose=verbose)
1543 verbose=verbose)
1654 =======
1544 =======
1655
1545
1656 # self.dataOut = dataOut
1546 # self.dataOut = dataOut
1657 self.setup(**kwargs)
1547 self.setup(**kwargs)
1658 >>>>>>> online_data_hour
1548 >>>>>>> online_data_hour
1659 self.isConfig = True
1549 self.isConfig = True
1660 if server is None:
1550 if server is None:
1661 self.getData()
1551 self.getData()
1662 else:
1552 else:
1663 self.getFromServer()
1553 self.getFromServer()
1664
1554
1665 class JRODataWriter(JRODataIO):
1555 class JRODataWriter(JRODataIO):
1666
1556
1667 """
1557 """
1668 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1558 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1669 de los datos siempre se realiza por bloques.
1559 de los datos siempre se realiza por bloques.
1670 """
1560 """
1671
1561
1672 blockIndex = 0
1562 blockIndex = 0
1673
1563
1674 path = None
1564 path = None
1675
1565
1676 setFile = None
1566 setFile = None
1677
1567
1678 profilesPerBlock = None
1568 profilesPerBlock = None
1679
1569
1680 blocksPerFile = None
1570 blocksPerFile = None
1681
1571
1682 nWriteBlocks = 0
1572 nWriteBlocks = 0
1683
1573
1684 fileDate = None
1574 fileDate = None
1685
1575
1686 def __init__(self, dataOut=None):
1576 def __init__(self, dataOut=None):
1687 raise NotImplementedError
1577 raise NotImplementedError
1688
1578
1689
1579
1690 def hasAllDataInBuffer(self):
1580 def hasAllDataInBuffer(self):
1691 raise NotImplementedError
1581 raise NotImplementedError
1692
1582
1693
1583
1694 def setBlockDimension(self):
1584 def setBlockDimension(self):
1695 raise NotImplementedError
1585 raise NotImplementedError
1696
1586
1697
1587
1698 def writeBlock(self):
1588 def writeBlock(self):
1699 raise NotImplementedError
1589 raise NotImplementedError
1700
1590
1701
1591
1702 def putData(self):
1592 def putData(self):
1703 raise NotImplementedError
1593 raise NotImplementedError
1704
1594
1705
1595
1706 def getProcessFlags(self):
1596 def getProcessFlags(self):
1707
1597
1708 processFlags = 0
1598 processFlags = 0
1709
1599
1710 dtype_index = get_dtype_index(self.dtype)
1600 dtype_index = get_dtype_index(self.dtype)
1711 procflag_dtype = get_procflag_dtype(dtype_index)
1601 procflag_dtype = get_procflag_dtype(dtype_index)
1712
1602
1713 processFlags += procflag_dtype
1603 processFlags += procflag_dtype
1714
1604
1715 if self.dataOut.flagDecodeData:
1605 if self.dataOut.flagDecodeData:
1716 processFlags += PROCFLAG.DECODE_DATA
1606 processFlags += PROCFLAG.DECODE_DATA
1717
1607
1718 if self.dataOut.flagDeflipData:
1608 if self.dataOut.flagDeflipData:
1719 processFlags += PROCFLAG.DEFLIP_DATA
1609 processFlags += PROCFLAG.DEFLIP_DATA
1720
1610
1721 if self.dataOut.code is not None:
1611 if self.dataOut.code is not None:
1722 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1612 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1723
1613
1724 if self.dataOut.nCohInt > 1:
1614 if self.dataOut.nCohInt > 1:
1725 processFlags += PROCFLAG.COHERENT_INTEGRATION
1615 processFlags += PROCFLAG.COHERENT_INTEGRATION
1726
1616
1727 if self.dataOut.type == "Spectra":
1617 if self.dataOut.type == "Spectra":
1728 if self.dataOut.nIncohInt > 1:
1618 if self.dataOut.nIncohInt > 1:
1729 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1619 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1730
1620
1731 if self.dataOut.data_dc is not None:
1621 if self.dataOut.data_dc is not None:
1732 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1622 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1733
1623
1734 if self.dataOut.flagShiftFFT:
1624 if self.dataOut.flagShiftFFT:
1735 processFlags += PROCFLAG.SHIFT_FFT_DATA
1625 processFlags += PROCFLAG.SHIFT_FFT_DATA
1736
1626
1737 return processFlags
1627 return processFlags
1738
1628
1739 def setBasicHeader(self):
1629 def setBasicHeader(self):
1740
1630
1741 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1631 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1742 self.basicHeaderObj.version = self.versionFile
1632 self.basicHeaderObj.version = self.versionFile
1743 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1633 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1744
1634
1745 utc = numpy.floor(self.dataOut.utctime)
1635 utc = numpy.floor(self.dataOut.utctime)
1746 milisecond = (self.dataOut.utctime - utc)* 1000.0
1636 milisecond = (self.dataOut.utctime - utc)* 1000.0
1747
1637
1748 self.basicHeaderObj.utc = utc
1638 self.basicHeaderObj.utc = utc
1749 self.basicHeaderObj.miliSecond = milisecond
1639 self.basicHeaderObj.miliSecond = milisecond
1750 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1640 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1751 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1641 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1752 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1642 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1753
1643
1754 def setFirstHeader(self):
1644 def setFirstHeader(self):
1755 """
1645 """
1756 Obtiene una copia del First Header
1646 Obtiene una copia del First Header
1757
1647
1758 Affected:
1648 Affected:
1759
1649
1760 self.basicHeaderObj
1650 self.basicHeaderObj
1761 self.systemHeaderObj
1651 self.systemHeaderObj
1762 self.radarControllerHeaderObj
1652 self.radarControllerHeaderObj
1763 self.processingHeaderObj self.
1653 self.processingHeaderObj self.
1764
1654
1765 Return:
1655 Return:
1766 None
1656 None
1767 """
1657 """
1768
1658
1769 raise NotImplementedError
1659 raise NotImplementedError
1770
1660
1771 def __writeFirstHeader(self):
1661 def __writeFirstHeader(self):
1772 """
1662 """
1773 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1663 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1774
1664
1775 Affected:
1665 Affected:
1776 __dataType
1666 __dataType
1777
1667
1778 Return:
1668 Return:
1779 None
1669 None
1780 """
1670 """
1781 <<<<<<< HEAD
1671 <<<<<<< HEAD
1782
1672
1783 # CALCULAR PARAMETROS
1673 # CALCULAR PARAMETROS
1784
1674
1785 =======
1675 =======
1786
1676
1787 # CALCULAR PARAMETROS
1677 # CALCULAR PARAMETROS
1788
1678
1789 >>>>>>> online_data_hour
1679 >>>>>>> online_data_hour
1790 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1680 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1791 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1681 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1792
1682
1793 self.basicHeaderObj.write(self.fp)
1683 self.basicHeaderObj.write(self.fp)
1794 self.systemHeaderObj.write(self.fp)
1684 self.systemHeaderObj.write(self.fp)
1795 self.radarControllerHeaderObj.write(self.fp)
1685 self.radarControllerHeaderObj.write(self.fp)
1796 self.processingHeaderObj.write(self.fp)
1686 self.processingHeaderObj.write(self.fp)
1797
1687
1798 def __setNewBlock(self):
1688 def __setNewBlock(self):
1799 """
1689 """
1800 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1690 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1801
1691
1802 Return:
1692 Return:
1803 0 : si no pudo escribir nada
1693 0 : si no pudo escribir nada
1804 1 : Si escribio el Basic el First Header
1694 1 : Si escribio el Basic el First Header
1805 """
1695 """
1806 if self.fp == None:
1696 if self.fp == None:
1807 self.setNextFile()
1697 self.setNextFile()
1808
1698
1809 if self.flagIsNewFile:
1699 if self.flagIsNewFile:
1810 return 1
1700 return 1
1811
1701
1812 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1702 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1813 self.basicHeaderObj.write(self.fp)
1703 self.basicHeaderObj.write(self.fp)
1814 return 1
1704 return 1
1815
1705
1816 if not( self.setNextFile() ):
1706 if not( self.setNextFile() ):
1817 return 0
1707 return 0
1818
1708
1819 return 1
1709 return 1
1820
1710
1821
1711
1822 def writeNextBlock(self):
1712 def writeNextBlock(self):
1823 """
1713 """
1824 Selecciona el bloque siguiente de datos y los escribe en un file
1714 Selecciona el bloque siguiente de datos y los escribe en un file
1825
1715
1826 Return:
1716 Return:
1827 0 : Si no hizo pudo escribir el bloque de datos
1717 0 : Si no hizo pudo escribir el bloque de datos
1828 1 : Si no pudo escribir el bloque de datos
1718 1 : Si no pudo escribir el bloque de datos
1829 """
1719 """
1830 if not( self.__setNewBlock() ):
1720 if not( self.__setNewBlock() ):
1831 return 0
1721 return 0
1832
1722
1833 self.writeBlock()
1723 self.writeBlock()
1834
1724
1835 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1725 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1836 self.processingHeaderObj.dataBlocksPerFile)
1726 self.processingHeaderObj.dataBlocksPerFile)
1837
1727
1838 return 1
1728 return 1
1839
1729
1840 def setNextFile(self):
1730 def setNextFile(self):
1841 """
1731 """
1842 Determina el siguiente file que sera escrito
1732 Determina el siguiente file que sera escrito
1843
1733
1844 Affected:
1734 Affected:
1845 self.filename
1735 self.filename
1846 self.subfolder
1736 self.subfolder
1847 self.fp
1737 self.fp
1848 self.setFile
1738 self.setFile
1849 self.flagIsNewFile
1739 self.flagIsNewFile
1850
1740
1851 Return:
1741 Return:
1852 0 : Si el archivo no puede ser escrito
1742 0 : Si el archivo no puede ser escrito
1853 1 : Si el archivo esta listo para ser escrito
1743 1 : Si el archivo esta listo para ser escrito
1854 """
1744 """
1855 ext = self.ext
1745 ext = self.ext
1856 path = self.path
1746 path = self.path
1857
1747
1858 if self.fp != None:
1748 if self.fp != None:
1859 self.fp.close()
1749 self.fp.close()
1860
1750
1861 timeTuple = time.localtime( self.dataOut.utctime)
1751 timeTuple = time.localtime( self.dataOut.utctime)
1862 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1752 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1863
1753
1864 fullpath = os.path.join( path, subfolder )
1754 fullpath = os.path.join( path, subfolder )
1865 setFile = self.setFile
1755 setFile = self.setFile
1866
1756
1867 if not( os.path.exists(fullpath) ):
1757 if not( os.path.exists(fullpath) ):
1868 os.mkdir(fullpath)
1758 os.mkdir(fullpath)
1869 setFile = -1 #inicializo mi contador de seteo
1759 setFile = -1 #inicializo mi contador de seteo
1870 else:
1760 else:
1871 filesList = os.listdir( fullpath )
1761 filesList = os.listdir( fullpath )
1872 if len( filesList ) > 0:
1762 if len( filesList ) > 0:
1873 filesList = sorted( filesList, key=str.lower )
1763 filesList = sorted( filesList, key=str.lower )
1874 filen = filesList[-1]
1764 filen = filesList[-1]
1875 # el filename debera tener el siguiente formato
1765 # el filename debera tener el siguiente formato
1876 # 0 1234 567 89A BCDE (hex)
1766 # 0 1234 567 89A BCDE (hex)
1877 # x YYYY DDD SSS .ext
1767 # x YYYY DDD SSS .ext
1878 if isNumber( filen[8:11] ):
1768 if isNumber( filen[8:11] ):
1879 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1769 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1880 else:
1770 else:
1881 setFile = -1
1771 setFile = -1
1882 else:
1772 else:
1883 setFile = -1 #inicializo mi contador de seteo
1773 setFile = -1 #inicializo mi contador de seteo
1884
1774
1885 setFile += 1
1775 setFile += 1
1886
1776
1887 #If this is a new day it resets some values
1777 #If this is a new day it resets some values
1888 if self.dataOut.datatime.date() > self.fileDate:
1778 if self.dataOut.datatime.date() > self.fileDate:
1889 setFile = 0
1779 setFile = 0
1890 self.nTotalBlocks = 0
1780 self.nTotalBlocks = 0
1891
1781
1892 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1782 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1893
1783
1894 filename = os.path.join( path, subfolder, filen )
1784 filename = os.path.join( path, subfolder, filen )
1895
1785
1896 fp = open( filename,'wb' )
1786 fp = open( filename,'wb' )
1897
1787
1898 self.blockIndex = 0
1788 self.blockIndex = 0
1899
1789
1900 #guardando atributos
1790 #guardando atributos
1901 self.filename = filename
1791 self.filename = filename
1902 self.subfolder = subfolder
1792 self.subfolder = subfolder
1903 self.fp = fp
1793 self.fp = fp
1904 self.setFile = setFile
1794 self.setFile = setFile
1905 self.flagIsNewFile = 1
1795 self.flagIsNewFile = 1
1906 self.fileDate = self.dataOut.datatime.date()
1796 self.fileDate = self.dataOut.datatime.date()
1907
1797
1908 self.setFirstHeader()
1798 self.setFirstHeader()
1909
1799
1910 print '[Writing] Opening file: %s'%self.filename
1800 print '[Writing] Opening file: %s'%self.filename
1911
1801
1912 self.__writeFirstHeader()
1802 self.__writeFirstHeader()
1913
1803
1914 return 1
1804 return 1
1915
1805
1916 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1806 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1917 """
1807 """
1918 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1808 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1919
1809
1920 Inputs:
1810 Inputs:
1921 path : directory where data will be saved
1811 path : directory where data will be saved
1922 profilesPerBlock : number of profiles per block
1812 profilesPerBlock : number of profiles per block
1923 set : initial file set
1813 set : initial file set
1924 datatype : An integer number that defines data type:
1814 datatype : An integer number that defines data type:
1925 0 : int8 (1 byte)
1815 0 : int8 (1 byte)
1926 1 : int16 (2 bytes)
1816 1 : int16 (2 bytes)
1927 2 : int32 (4 bytes)
1817 2 : int32 (4 bytes)
1928 3 : int64 (8 bytes)
1818 3 : int64 (8 bytes)
1929 4 : float32 (4 bytes)
1819 4 : float32 (4 bytes)
1930 5 : double64 (8 bytes)
1820 5 : double64 (8 bytes)
1931
1821
1932 Return:
1822 Return:
1933 0 : Si no realizo un buen seteo
1823 0 : Si no realizo un buen seteo
1934 1 : Si realizo un buen seteo
1824 1 : Si realizo un buen seteo
1935 """
1825 """
1936
1826
1937 if ext == None:
1827 if ext == None:
1938 ext = self.ext
1828 ext = self.ext
1939
1829
1940 self.ext = ext.lower()
1830 self.ext = ext.lower()
1941
1831
1942 self.path = path
1832 self.path = path
1943
1833
1944 if set is None:
1834 if set is None:
1945 self.setFile = -1
1835 self.setFile = -1
1946 else:
1836 else:
1947 self.setFile = set - 1
1837 self.setFile = set - 1
1948
1838
1949 self.blocksPerFile = blocksPerFile
1839 self.blocksPerFile = blocksPerFile
1950
1840
1951 self.profilesPerBlock = profilesPerBlock
1841 self.profilesPerBlock = profilesPerBlock
1952
1842
1953 self.dataOut = dataOut
1843 self.dataOut = dataOut
1954 self.fileDate = self.dataOut.datatime.date()
1844 self.fileDate = self.dataOut.datatime.date()
1955 #By default
1845 #By default
1956 self.dtype = self.dataOut.dtype
1846 self.dtype = self.dataOut.dtype
1957
1847
1958 if datatype is not None:
1848 if datatype is not None:
1959 self.dtype = get_numpy_dtype(datatype)
1849 self.dtype = get_numpy_dtype(datatype)
1960
1850
1961 if not(self.setNextFile()):
1851 if not(self.setNextFile()):
1962 print "[Writing] There isn't a next file"
1852 print "[Writing] There isn't a next file"
1963 return 0
1853 return 0
1964
1854
1965 self.setBlockDimension()
1855 self.setBlockDimension()
1966
1856
1967 return 1
1857 return 1
1968
1858
1969 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1859 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1970
1860
1971 if not(self.isConfig):
1861 if not(self.isConfig):
1972
1862
1973 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1863 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1974 self.isConfig = True
1864 self.isConfig = True
1975
1865
1976 self.putData()
1866 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now