##// END OF EJS Templates
schain
RSS
Bug fixed in findDataFiles
Miguel Valdez -
r601:af37133b1bca
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1451 +1,1451
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22
22
23 LOCALTIME = True
23 LOCALTIME = True
24
24
25 def isNumber(cad):
25 def isNumber(cad):
26 """
26 """
27 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
27 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28
28
29 Excepciones:
29 Excepciones:
30 Si un determinado string no puede ser convertido a numero
30 Si un determinado string no puede ser convertido a numero
31 Input:
31 Input:
32 str, string al cual se le analiza para determinar si convertible a un numero o no
32 str, string al cual se le analiza para determinar si convertible a un numero o no
33
33
34 Return:
34 Return:
35 True : si el string es uno numerico
35 True : si el string es uno numerico
36 False : no es un string numerico
36 False : no es un string numerico
37 """
37 """
38 try:
38 try:
39 float( cad )
39 float( cad )
40 return True
40 return True
41 except:
41 except:
42 return False
42 return False
43
43
44 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
44 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 """
45 """
46 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
46 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47
47
48 Inputs:
48 Inputs:
49 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
49 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50
50
51 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
51 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 segundos contados desde 01/01/1970.
52 segundos contados desde 01/01/1970.
53 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
53 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 segundos contados desde 01/01/1970.
54 segundos contados desde 01/01/1970.
55
55
56 Return:
56 Return:
57 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
57 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 fecha especificado, de lo contrario retorna False.
58 fecha especificado, de lo contrario retorna False.
59
59
60 Excepciones:
60 Excepciones:
61 Si el archivo no existe o no puede ser abierto
61 Si el archivo no existe o no puede ser abierto
62 Si la cabecera no puede ser leida.
62 Si la cabecera no puede ser leida.
63
63
64 """
64 """
65 basicHeaderObj = BasicHeader(LOCALTIME)
65 basicHeaderObj = BasicHeader(LOCALTIME)
66
66
67 try:
67 try:
68 fp = open(filename,'rb')
68 fp = open(filename,'rb')
69 except IOError:
69 except IOError:
70 traceback.print_exc()
70 traceback.print_exc()
71 raise IOError, "The file %s can't be opened" %(filename)
71 raise IOError, "The file %s can't be opened" %(filename)
72
72
73 sts = basicHeaderObj.read(fp)
73 sts = basicHeaderObj.read(fp)
74 fp.close()
74 fp.close()
75
75
76 if not(sts):
76 if not(sts):
77 print "Skipping the file %s because it has not a valid header" %(filename)
77 print "Skipping the file %s because it has not a valid header" %(filename)
78 return 0
78 return 0
79
79
80 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
80 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 return 0
81 return 0
82
82
83 return 1
83 return 1
84
84
85 def isFileinThisTime(filename, startTime, endTime):
85 def isFileinThisTime(filename, startTime, endTime):
86 """
86 """
87 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
87 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88
88
89 Inputs:
89 Inputs:
90 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
90 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91
91
92 startTime : tiempo inicial del rango seleccionado en formato datetime.time
92 startTime : tiempo inicial del rango seleccionado en formato datetime.time
93
93
94 endTime : tiempo final del rango seleccionado en formato datetime.time
94 endTime : tiempo final del rango seleccionado en formato datetime.time
95
95
96 Return:
96 Return:
97 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
97 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
98 fecha especificado, de lo contrario retorna False.
98 fecha especificado, de lo contrario retorna False.
99
99
100 Excepciones:
100 Excepciones:
101 Si el archivo no existe o no puede ser abierto
101 Si el archivo no existe o no puede ser abierto
102 Si la cabecera no puede ser leida.
102 Si la cabecera no puede ser leida.
103
103
104 """
104 """
105
105
106
106
107 try:
107 try:
108 fp = open(filename,'rb')
108 fp = open(filename,'rb')
109 except IOError:
109 except IOError:
110 traceback.print_exc()
110 traceback.print_exc()
111 raise IOError, "The file %s can't be opened" %(filename)
111 raise IOError, "The file %s can't be opened" %(filename)
112
112
113 basicHeaderObj = BasicHeader(LOCALTIME)
113 basicHeaderObj = BasicHeader(LOCALTIME)
114 sts = basicHeaderObj.read(fp)
114 sts = basicHeaderObj.read(fp)
115 fp.close()
115 fp.close()
116
116
117 thisDatetime = basicHeaderObj.datatime
117 thisDatetime = basicHeaderObj.datatime
118 thisTime = thisDatetime.time()
118 thisTime = thisDatetime.time()
119
119
120 if not(sts):
120 if not(sts):
121 print "Skipping the file %s because it has not a valid header" %(filename)
121 print "Skipping the file %s because it has not a valid header" %(filename)
122 return None
122 return None
123
123
124 if not ((startTime <= thisTime) and (endTime > thisTime)):
124 if not ((startTime <= thisTime) and (endTime > thisTime)):
125 return None
125 return None
126
126
127 return thisDatetime
127 return thisDatetime
128
128
129 def getFileFromSet(path, ext, set):
129 def getFileFromSet(path, ext, set):
130 validFilelist = []
130 validFilelist = []
131 fileList = os.listdir(path)
131 fileList = os.listdir(path)
132
132
133 # 0 1234 567 89A BCDE
133 # 0 1234 567 89A BCDE
134 # H YYYY DDD SSS .ext
134 # H YYYY DDD SSS .ext
135
135
136 for thisFile in fileList:
136 for thisFile in fileList:
137 try:
137 try:
138 year = int(thisFile[1:5])
138 year = int(thisFile[1:5])
139 doy = int(thisFile[5:8])
139 doy = int(thisFile[5:8])
140 except:
140 except:
141 continue
141 continue
142
142
143 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
143 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
144 continue
144 continue
145
145
146 validFilelist.append(thisFile)
146 validFilelist.append(thisFile)
147
147
148 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
148 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
149
149
150 if len(myfile)!= 0:
150 if len(myfile)!= 0:
151 return myfile[0]
151 return myfile[0]
152 else:
152 else:
153 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
153 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
154 print 'the filename %s does not exist'%filename
154 print 'the filename %s does not exist'%filename
155 print '...going to the last file: '
155 print '...going to the last file: '
156
156
157 if validFilelist:
157 if validFilelist:
158 validFilelist = sorted( validFilelist, key=str.lower )
158 validFilelist = sorted( validFilelist, key=str.lower )
159 return validFilelist[-1]
159 return validFilelist[-1]
160
160
161 return None
161 return None
162
162
163 def getlastFileFromPath(path, ext):
163 def getlastFileFromPath(path, ext):
164 """
164 """
165 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
165 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
166 al final de la depuracion devuelve el ultimo file de la lista que quedo.
166 al final de la depuracion devuelve el ultimo file de la lista que quedo.
167
167
168 Input:
168 Input:
169 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
169 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
170 ext : extension de los files contenidos en una carpeta
170 ext : extension de los files contenidos en una carpeta
171
171
172 Return:
172 Return:
173 El ultimo file de una determinada carpeta, no se considera el path.
173 El ultimo file de una determinada carpeta, no se considera el path.
174 """
174 """
175 validFilelist = []
175 validFilelist = []
176 fileList = os.listdir(path)
176 fileList = os.listdir(path)
177
177
178 # 0 1234 567 89A BCDE
178 # 0 1234 567 89A BCDE
179 # H YYYY DDD SSS .ext
179 # H YYYY DDD SSS .ext
180
180
181 for thisFile in fileList:
181 for thisFile in fileList:
182
182
183 year = thisFile[1:5]
183 year = thisFile[1:5]
184 if not isNumber(year):
184 if not isNumber(year):
185 continue
185 continue
186
186
187 doy = thisFile[5:8]
187 doy = thisFile[5:8]
188 if not isNumber(doy):
188 if not isNumber(doy):
189 continue
189 continue
190
190
191 year = int(year)
191 year = int(year)
192 doy = int(doy)
192 doy = int(doy)
193
193
194 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
194 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
195 continue
195 continue
196
196
197 validFilelist.append(thisFile)
197 validFilelist.append(thisFile)
198
198
199 if validFilelist:
199 if validFilelist:
200 validFilelist = sorted( validFilelist, key=str.lower )
200 validFilelist = sorted( validFilelist, key=str.lower )
201 return validFilelist[-1]
201 return validFilelist[-1]
202
202
203 return None
203 return None
204
204
205 def checkForRealPath(path, foldercounter, year, doy, set, ext):
205 def checkForRealPath(path, foldercounter, year, doy, set, ext):
206 """
206 """
207 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
207 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
208 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
208 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
209 el path exacto de un determinado file.
209 el path exacto de un determinado file.
210
210
211 Example :
211 Example :
212 nombre correcto del file es .../.../D2009307/P2009307367.ext
212 nombre correcto del file es .../.../D2009307/P2009307367.ext
213
213
214 Entonces la funcion prueba con las siguientes combinaciones
214 Entonces la funcion prueba con las siguientes combinaciones
215 .../.../y2009307367.ext
215 .../.../y2009307367.ext
216 .../.../Y2009307367.ext
216 .../.../Y2009307367.ext
217 .../.../x2009307/y2009307367.ext
217 .../.../x2009307/y2009307367.ext
218 .../.../x2009307/Y2009307367.ext
218 .../.../x2009307/Y2009307367.ext
219 .../.../X2009307/y2009307367.ext
219 .../.../X2009307/y2009307367.ext
220 .../.../X2009307/Y2009307367.ext
220 .../.../X2009307/Y2009307367.ext
221 siendo para este caso, la ultima combinacion de letras, identica al file buscado
221 siendo para este caso, la ultima combinacion de letras, identica al file buscado
222
222
223 Return:
223 Return:
224 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
224 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
225 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
225 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
226 para el filename
226 para el filename
227 """
227 """
228 fullfilename = None
228 fullfilename = None
229 find_flag = False
229 find_flag = False
230 filename = None
230 filename = None
231
231
232 prefixDirList = [None,'d','D']
232 prefixDirList = [None,'d','D']
233 if ext.lower() == ".r": #voltage
233 if ext.lower() == ".r": #voltage
234 prefixFileList = ['d','D']
234 prefixFileList = ['d','D']
235 elif ext.lower() == ".pdata": #spectra
235 elif ext.lower() == ".pdata": #spectra
236 prefixFileList = ['p','P']
236 prefixFileList = ['p','P']
237 else:
237 else:
238 return None, filename
238 return None, filename
239
239
240 #barrido por las combinaciones posibles
240 #barrido por las combinaciones posibles
241 for prefixDir in prefixDirList:
241 for prefixDir in prefixDirList:
242 thispath = path
242 thispath = path
243 if prefixDir != None:
243 if prefixDir != None:
244 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
244 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
245 if foldercounter == 0:
245 if foldercounter == 0:
246 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
246 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
247 else:
247 else:
248 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
248 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
249 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
249 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
250 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
250 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
251 fullfilename = os.path.join( thispath, filename ) #formo el path completo
251 fullfilename = os.path.join( thispath, filename ) #formo el path completo
252
252
253 if os.path.exists( fullfilename ): #verifico que exista
253 if os.path.exists( fullfilename ): #verifico que exista
254 find_flag = True
254 find_flag = True
255 break
255 break
256 if find_flag:
256 if find_flag:
257 break
257 break
258
258
259 if not(find_flag):
259 if not(find_flag):
260 return None, filename
260 return None, filename
261
261
262 return fullfilename, filename
262 return fullfilename, filename
263
263
264 def isRadarFolder(folder):
264 def isRadarFolder(folder):
265 try:
265 try:
266 year = int(folder[1:5])
266 year = int(folder[1:5])
267 doy = int(folder[5:8])
267 doy = int(folder[5:8])
268 except:
268 except:
269 return 0
269 return 0
270
270
271 return 1
271 return 1
272
272
273 def isRadarFile(file):
273 def isRadarFile(file):
274 try:
274 try:
275 year = int(file[1:5])
275 year = int(file[1:5])
276 doy = int(file[5:8])
276 doy = int(file[5:8])
277 set = int(file[8:11])
277 set = int(file[8:11])
278 except:
278 except:
279 return 0
279 return 0
280
280
281 return 1
281 return 1
282
282
283 def getDateFromRadarFile(file):
283 def getDateFromRadarFile(file):
284 try:
284 try:
285 year = int(file[1:5])
285 year = int(file[1:5])
286 doy = int(file[5:8])
286 doy = int(file[5:8])
287 set = int(file[8:11])
287 set = int(file[8:11])
288 except:
288 except:
289 return None
289 return None
290
290
291 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
291 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
292 return thisDate
292 return thisDate
293
293
294 class JRODataIO:
294 class JRODataIO:
295
295
296 c = 3E8
296 c = 3E8
297
297
298 isConfig = False
298 isConfig = False
299
299
300 basicHeaderObj = None
300 basicHeaderObj = None
301
301
302 systemHeaderObj = None
302 systemHeaderObj = None
303
303
304 radarControllerHeaderObj = None
304 radarControllerHeaderObj = None
305
305
306 processingHeaderObj = None
306 processingHeaderObj = None
307
307
308 online = 0
308 online = 0
309
309
310 dtype = None
310 dtype = None
311
311
312 pathList = []
312 pathList = []
313
313
314 filenameList = []
314 filenameList = []
315
315
316 filename = None
316 filename = None
317
317
318 ext = None
318 ext = None
319
319
320 flagIsNewFile = 1
320 flagIsNewFile = 1
321
321
322 flagDiscontinuousBlock = 0
322 flagDiscontinuousBlock = 0
323
323
324 flagIsNewBlock = 0
324 flagIsNewBlock = 0
325
325
326 fp = None
326 fp = None
327
327
328 firstHeaderSize = 0
328 firstHeaderSize = 0
329
329
330 basicHeaderSize = 24
330 basicHeaderSize = 24
331
331
332 versionFile = 1103
332 versionFile = 1103
333
333
334 fileSize = None
334 fileSize = None
335
335
336 # ippSeconds = None
336 # ippSeconds = None
337
337
338 fileSizeByHeader = None
338 fileSizeByHeader = None
339
339
340 fileIndex = None
340 fileIndex = None
341
341
342 profileIndex = None
342 profileIndex = None
343
343
344 blockIndex = None
344 blockIndex = None
345
345
346 nTotalBlocks = None
346 nTotalBlocks = None
347
347
348 maxTimeStep = 30
348 maxTimeStep = 30
349
349
350 lastUTTime = None
350 lastUTTime = None
351
351
352 datablock = None
352 datablock = None
353
353
354 dataOut = None
354 dataOut = None
355
355
356 blocksize = None
356 blocksize = None
357
357
358 getByBlock = False
358 getByBlock = False
359
359
360 def __init__(self):
360 def __init__(self):
361
361
362 raise ValueError, "Not implemented"
362 raise ValueError, "Not implemented"
363
363
364 def run(self):
364 def run(self):
365
365
366 raise ValueError, "Not implemented"
366 raise ValueError, "Not implemented"
367
367
368 class JRODataReader(JRODataIO):
368 class JRODataReader(JRODataIO):
369
369
370 nReadBlocks = 0
370 nReadBlocks = 0
371
371
372 delay = 10 #number of seconds waiting a new file
372 delay = 10 #number of seconds waiting a new file
373
373
374 nTries = 3 #quantity tries
374 nTries = 3 #quantity tries
375
375
376 nFiles = 3 #number of files for searching
376 nFiles = 3 #number of files for searching
377
377
378 path = None
378 path = None
379
379
380 foldercounter = 0
380 foldercounter = 0
381
381
382 flagNoMoreFiles = 0
382 flagNoMoreFiles = 0
383
383
384 datetimeList = []
384 datetimeList = []
385
385
386 __isFirstTimeOnline = 1
386 __isFirstTimeOnline = 1
387
387
388 __printInfo = True
388 __printInfo = True
389
389
390 profileIndex = None
390 profileIndex = None
391
391
392 nTxs = 1
392 nTxs = 1
393
393
394 txIndex = None
394 txIndex = None
395
395
396 def __init__(self):
396 def __init__(self):
397
397
398 """
398 """
399
399
400 """
400 """
401
401
402 # raise NotImplementedError, "This method has not been implemented"
402 # raise NotImplementedError, "This method has not been implemented"
403
403
404
404
405 def createObjByDefault(self):
405 def createObjByDefault(self):
406 """
406 """
407
407
408 """
408 """
409 raise NotImplementedError, "This method has not been implemented"
409 raise NotImplementedError, "This method has not been implemented"
410
410
411 def getBlockDimension(self):
411 def getBlockDimension(self):
412
412
413 raise NotImplementedError, "No implemented"
413 raise NotImplementedError, "No implemented"
414
414
415 def __searchFilesOffLine(self,
415 def __searchFilesOffLine(self,
416 path,
416 path,
417 startDate=None,
417 startDate=None,
418 endDate=None,
418 endDate=None,
419 startTime=datetime.time(0,0,0),
419 startTime=datetime.time(0,0,0),
420 endTime=datetime.time(23,59,59),
420 endTime=datetime.time(23,59,59),
421 set=None,
421 set=None,
422 expLabel='',
422 expLabel='',
423 ext='.r',
423 ext='.r',
424 walk=True):
424 walk=True):
425
425
426 self.filenameList = []
426 self.filenameList = []
427 self.datetimeList = []
427 self.datetimeList = []
428
428
429 pathList = []
429 pathList = []
430
430
431 if not walk:
431 if not walk:
432 #pathList.append(path)
432 #pathList.append(path)
433 multi_path = path.split(',')
433 multi_path = path.split(',')
434 for single_path in multi_path:
434 for single_path in multi_path:
435 pathList.append(single_path)
435 pathList.append(single_path)
436
436
437 else:
437 else:
438 #dirList = []
438 #dirList = []
439 multi_path = path.split(',')
439 multi_path = path.split(',')
440 for single_path in multi_path:
440 for single_path in multi_path:
441 dirList = []
441 dirList = []
442 for thisPath in os.listdir(single_path):
442 for thisPath in os.listdir(single_path):
443 if not os.path.isdir(os.path.join(single_path,thisPath)):
443 if not os.path.isdir(os.path.join(single_path,thisPath)):
444 continue
444 continue
445 if not isRadarFolder(thisPath):
445 if not isRadarFolder(thisPath):
446 continue
446 continue
447
447
448 dirList.append(thisPath)
448 dirList.append(thisPath)
449
449
450 if not(dirList):
450 if not(dirList):
451 return None, None
451 return None, None
452
452
453 if startDate and endDate:
453 if startDate and endDate:
454 thisDate = startDate
454 thisDate = startDate
455
455
456 while(thisDate <= endDate):
456 while(thisDate <= endDate):
457 year = thisDate.timetuple().tm_year
457 year = thisDate.timetuple().tm_year
458 doy = thisDate.timetuple().tm_yday
458 doy = thisDate.timetuple().tm_yday
459
459
460 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
460 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
461 if len(matchlist) == 0:
461 if len(matchlist) == 0:
462 thisDate += datetime.timedelta(1)
462 thisDate += datetime.timedelta(1)
463 continue
463 continue
464 for match in matchlist:
464 for match in matchlist:
465 pathList.append(os.path.join(single_path,match,expLabel))
465 pathList.append(os.path.join(single_path,match,expLabel))
466
466
467 thisDate += datetime.timedelta(1)
467 thisDate += datetime.timedelta(1)
468 else:
468 else:
469 for thiDir in dirList:
469 for thiDir in dirList:
470 pathList.append(os.path.join(single_path,thiDir,expLabel))
470 pathList.append(os.path.join(single_path,thiDir,expLabel))
471
471
472 if pathList == []:
472 if pathList == []:
473 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
473 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
474 return None, None
474 return None, None
475
475
476 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
476 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
477
477
478 filenameList = []
478 filenameList = []
479 datetimeList = []
479 datetimeList = []
480 pathDict = {}
480 pathDict = {}
481 filenameList_to_sort = []
481 filenameList_to_sort = []
482
482
483 for i in range(len(pathList)):
483 for i in range(len(pathList)):
484
484
485 thisPath = pathList[i]
485 thisPath = pathList[i]
486
486
487 fileList = glob.glob1(thisPath, "*%s" %ext)
487 fileList = glob.glob1(thisPath, "*%s" %ext)
488 if len(fileList) < 1:
488 if len(fileList) < 1:
489 continue
489 continue
490 fileList.sort()
490 fileList.sort()
491 pathDict.setdefault(fileList[0])
491 pathDict.setdefault(fileList[0])
492 pathDict[fileList[0]] = i
492 pathDict[fileList[0]] = i
493 filenameList_to_sort.append(fileList[0])
493 filenameList_to_sort.append(fileList[0])
494
494
495 filenameList_to_sort.sort()
495 filenameList_to_sort.sort()
496
496
497 for file in filenameList_to_sort:
497 for file in filenameList_to_sort:
498 thisPath = pathList[pathDict[file]]
498 thisPath = pathList[pathDict[file]]
499
499
500 fileList = glob.glob1(thisPath, "*%s" %ext)
500 fileList = glob.glob1(thisPath, "*%s" %ext)
501 fileList.sort()
501 fileList.sort()
502
502
503 for file in fileList:
503 for file in fileList:
504
504
505 filename = os.path.join(thisPath,file)
505 filename = os.path.join(thisPath,file)
506 thisDatetime = isFileinThisTime(filename, startTime, endTime)
506 thisDatetime = isFileinThisTime(filename, startTime, endTime)
507
507
508 if not(thisDatetime):
508 if not(thisDatetime):
509 continue
509 continue
510
510
511 filenameList.append(filename)
511 filenameList.append(filename)
512 datetimeList.append(thisDatetime)
512 datetimeList.append(thisDatetime)
513
513
514 if not(filenameList):
514 if not(filenameList):
515 print "Any file was found for the time range %s - %s" %(startTime, endTime)
515 print "Any file was found for the time range %s - %s" %(startTime, endTime)
516 return None, None
516 return None, None
517
517
518 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
518 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
519 print
519 print
520
520
521 for i in range(len(filenameList)):
521 for i in range(len(filenameList)):
522 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
522 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
523
523
524 self.filenameList = filenameList
524 self.filenameList = filenameList
525 self.datetimeList = datetimeList
525 self.datetimeList = datetimeList
526
526
527 return pathList, filenameList
527 return pathList, filenameList
528
528
529 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
529 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
530
530
531 """
531 """
532 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
532 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
533 devuelve el archivo encontrado ademas de otros datos.
533 devuelve el archivo encontrado ademas de otros datos.
534
534
535 Input:
535 Input:
536 path : carpeta donde estan contenidos los files que contiene data
536 path : carpeta donde estan contenidos los files que contiene data
537
537
538 expLabel : Nombre del subexperimento (subfolder)
538 expLabel : Nombre del subexperimento (subfolder)
539
539
540 ext : extension de los files
540 ext : extension de los files
541
541
542 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
542 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
543
543
544 Return:
544 Return:
545 directory : eL directorio donde esta el file encontrado
545 directory : eL directorio donde esta el file encontrado
546 filename : el ultimo file de una determinada carpeta
546 filename : el ultimo file de una determinada carpeta
547 year : el anho
547 year : el anho
548 doy : el numero de dia del anho
548 doy : el numero de dia del anho
549 set : el set del archivo
549 set : el set del archivo
550
550
551
551
552 """
552 """
553 dirList = []
553 dirList = []
554
554
555 if not walk:
555 if not walk:
556 fullpath = path
556 fullpath = path
557 foldercounter = 0
557 foldercounter = 0
558 else:
558 else:
559 #Filtra solo los directorios
559 #Filtra solo los directorios
560 for thisPath in os.listdir(path):
560 for thisPath in os.listdir(path):
561 if not os.path.isdir(os.path.join(path,thisPath)):
561 if not os.path.isdir(os.path.join(path,thisPath)):
562 continue
562 continue
563 if not isRadarFolder(thisPath):
563 if not isRadarFolder(thisPath):
564 continue
564 continue
565
565
566 dirList.append(thisPath)
566 dirList.append(thisPath)
567
567
568 if not(dirList):
568 if not(dirList):
569 return None, None, None, None, None, None
569 return None, None, None, None, None, None
570
570
571 dirList = sorted( dirList, key=str.lower )
571 dirList = sorted( dirList, key=str.lower )
572
572
573 doypath = dirList[-1]
573 doypath = dirList[-1]
574 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
574 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
575 fullpath = os.path.join(path, doypath, expLabel)
575 fullpath = os.path.join(path, doypath, expLabel)
576
576
577
577
578 print "%s folder was found: " %(fullpath )
578 print "%s folder was found: " %(fullpath )
579
579
580 if set == None:
580 if set == None:
581 filename = getlastFileFromPath(fullpath, ext)
581 filename = getlastFileFromPath(fullpath, ext)
582 else:
582 else:
583 filename = getFileFromSet(fullpath, ext, set)
583 filename = getFileFromSet(fullpath, ext, set)
584
584
585 if not(filename):
585 if not(filename):
586 return None, None, None, None, None, None
586 return None, None, None, None, None, None
587
587
588 print "%s file was found" %(filename)
588 print "%s file was found" %(filename)
589
589
590 if not(self.__verifyFile(os.path.join(fullpath, filename))):
590 if not(self.__verifyFile(os.path.join(fullpath, filename))):
591 return None, None, None, None, None, None
591 return None, None, None, None, None, None
592
592
593 year = int( filename[1:5] )
593 year = int( filename[1:5] )
594 doy = int( filename[5:8] )
594 doy = int( filename[5:8] )
595 set = int( filename[8:11] )
595 set = int( filename[8:11] )
596
596
597 return fullpath, foldercounter, filename, year, doy, set
597 return fullpath, foldercounter, filename, year, doy, set
598
598
599 def __setNextFileOffline(self):
599 def __setNextFileOffline(self):
600
600
601 idFile = self.fileIndex
601 idFile = self.fileIndex
602
602
603 while (True):
603 while (True):
604 idFile += 1
604 idFile += 1
605 if not(idFile < len(self.filenameList)):
605 if not(idFile < len(self.filenameList)):
606 self.flagNoMoreFiles = 1
606 self.flagNoMoreFiles = 1
607 # print "[Reading] No more Files"
607 # print "[Reading] No more Files"
608 return 0
608 return 0
609
609
610 filename = self.filenameList[idFile]
610 filename = self.filenameList[idFile]
611
611
612 if not(self.__verifyFile(filename)):
612 if not(self.__verifyFile(filename)):
613 continue
613 continue
614
614
615 fileSize = os.path.getsize(filename)
615 fileSize = os.path.getsize(filename)
616 fp = open(filename,'rb')
616 fp = open(filename,'rb')
617 break
617 break
618
618
619 self.flagIsNewFile = 1
619 self.flagIsNewFile = 1
620 self.fileIndex = idFile
620 self.fileIndex = idFile
621 self.filename = filename
621 self.filename = filename
622 self.fileSize = fileSize
622 self.fileSize = fileSize
623 self.fp = fp
623 self.fp = fp
624
624
625 # print "[Reading] Setting the file: %s"%self.filename
625 # print "[Reading] Setting the file: %s"%self.filename
626
626
627 return 1
627 return 1
628
628
629 def __setNextFileOnline(self):
629 def __setNextFileOnline(self):
630 """
630 """
631 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
631 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
632 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
632 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
633 siguientes.
633 siguientes.
634
634
635 Affected:
635 Affected:
636 self.flagIsNewFile
636 self.flagIsNewFile
637 self.filename
637 self.filename
638 self.fileSize
638 self.fileSize
639 self.fp
639 self.fp
640 self.set
640 self.set
641 self.flagNoMoreFiles
641 self.flagNoMoreFiles
642
642
643 Return:
643 Return:
644 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
644 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
645 1 : si el file fue abierto con exito y esta listo a ser leido
645 1 : si el file fue abierto con exito y esta listo a ser leido
646
646
647 Excepciones:
647 Excepciones:
648 Si un determinado file no puede ser abierto
648 Si un determinado file no puede ser abierto
649 """
649 """
650 nFiles = 0
650 nFiles = 0
651 fileOk_flag = False
651 fileOk_flag = False
652 firstTime_flag = True
652 firstTime_flag = True
653
653
654 self.set += 1
654 self.set += 1
655
655
656 if self.set > 999:
656 if self.set > 999:
657 self.set = 0
657 self.set = 0
658 self.foldercounter += 1
658 self.foldercounter += 1
659
659
660 #busca el 1er file disponible
660 #busca el 1er file disponible
661 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
661 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
662 if fullfilename:
662 if fullfilename:
663 if self.__verifyFile(fullfilename, False):
663 if self.__verifyFile(fullfilename, False):
664 fileOk_flag = True
664 fileOk_flag = True
665
665
666 #si no encuentra un file entonces espera y vuelve a buscar
666 #si no encuentra un file entonces espera y vuelve a buscar
667 if not(fileOk_flag):
667 if not(fileOk_flag):
668 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
668 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
669
669
670 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
670 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
671 tries = self.nTries
671 tries = self.nTries
672 else:
672 else:
673 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
673 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
674
674
675 for nTries in range( tries ):
675 for nTries in range( tries ):
676 if firstTime_flag:
676 if firstTime_flag:
677 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
677 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
678 sleep( self.delay )
678 sleep( self.delay )
679 else:
679 else:
680 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
680 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
681
681
682 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
682 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
683 if fullfilename:
683 if fullfilename:
684 if self.__verifyFile(fullfilename):
684 if self.__verifyFile(fullfilename):
685 fileOk_flag = True
685 fileOk_flag = True
686 break
686 break
687
687
688 if fileOk_flag:
688 if fileOk_flag:
689 break
689 break
690
690
691 firstTime_flag = False
691 firstTime_flag = False
692
692
693 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
693 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
694 self.set += 1
694 self.set += 1
695
695
696 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
696 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
697 self.set = 0
697 self.set = 0
698 self.doy += 1
698 self.doy += 1
699 self.foldercounter = 0
699 self.foldercounter = 0
700
700
701 if fileOk_flag:
701 if fileOk_flag:
702 self.fileSize = os.path.getsize( fullfilename )
702 self.fileSize = os.path.getsize( fullfilename )
703 self.filename = fullfilename
703 self.filename = fullfilename
704 self.flagIsNewFile = 1
704 self.flagIsNewFile = 1
705 if self.fp != None: self.fp.close()
705 if self.fp != None: self.fp.close()
706 self.fp = open(fullfilename, 'rb')
706 self.fp = open(fullfilename, 'rb')
707 self.flagNoMoreFiles = 0
707 self.flagNoMoreFiles = 0
708 # print '[Reading] Setting the file: %s' % fullfilename
708 # print '[Reading] Setting the file: %s' % fullfilename
709 else:
709 else:
710 self.fileSize = 0
710 self.fileSize = 0
711 self.filename = None
711 self.filename = None
712 self.flagIsNewFile = 0
712 self.flagIsNewFile = 0
713 self.fp = None
713 self.fp = None
714 self.flagNoMoreFiles = 1
714 self.flagNoMoreFiles = 1
715 # print '[Reading] No more files to read'
715 # print '[Reading] No more files to read'
716
716
717 return fileOk_flag
717 return fileOk_flag
718
718
719 def setNextFile(self):
719 def setNextFile(self):
720 if self.fp != None:
720 if self.fp != None:
721 self.fp.close()
721 self.fp.close()
722
722
723 if self.online:
723 if self.online:
724 newFile = self.__setNextFileOnline()
724 newFile = self.__setNextFileOnline()
725 else:
725 else:
726 newFile = self.__setNextFileOffline()
726 newFile = self.__setNextFileOffline()
727
727
728 if not(newFile):
728 if not(newFile):
729 print '[Reading] No more files to read'
729 print '[Reading] No more files to read'
730 return 0
730 return 0
731
731
732 print '[Reading] Setting the file: %s' % self.filename
732 print '[Reading] Setting the file: %s' % self.filename
733
733
734 self.__readFirstHeader()
734 self.__readFirstHeader()
735 self.nReadBlocks = 0
735 self.nReadBlocks = 0
736 return 1
736 return 1
737
737
738 def __waitNewBlock(self):
738 def __waitNewBlock(self):
739 """
739 """
740 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
740 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
741
741
742 Si el modo de lectura es OffLine siempre retorn 0
742 Si el modo de lectura es OffLine siempre retorn 0
743 """
743 """
744 if not self.online:
744 if not self.online:
745 return 0
745 return 0
746
746
747 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
747 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
748 return 0
748 return 0
749
749
750 currentPointer = self.fp.tell()
750 currentPointer = self.fp.tell()
751
751
752 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
752 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
753
753
754 for nTries in range( self.nTries ):
754 for nTries in range( self.nTries ):
755
755
756 self.fp.close()
756 self.fp.close()
757 self.fp = open( self.filename, 'rb' )
757 self.fp = open( self.filename, 'rb' )
758 self.fp.seek( currentPointer )
758 self.fp.seek( currentPointer )
759
759
760 self.fileSize = os.path.getsize( self.filename )
760 self.fileSize = os.path.getsize( self.filename )
761 currentSize = self.fileSize - currentPointer
761 currentSize = self.fileSize - currentPointer
762
762
763 if ( currentSize >= neededSize ):
763 if ( currentSize >= neededSize ):
764 self.basicHeaderObj.read(self.fp)
764 self.basicHeaderObj.read(self.fp)
765 return 1
765 return 1
766
766
767 if self.fileSize == self.fileSizeByHeader:
767 if self.fileSize == self.fileSizeByHeader:
768 # self.flagEoF = True
768 # self.flagEoF = True
769 return 0
769 return 0
770
770
771 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
771 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
772 sleep( self.delay )
772 sleep( self.delay )
773
773
774
774
775 return 0
775 return 0
776
776
777 def waitDataBlock(self,pointer_location):
777 def waitDataBlock(self,pointer_location):
778
778
779 currentPointer = pointer_location
779 currentPointer = pointer_location
780
780
781 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
781 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
782
782
783 for nTries in range( self.nTries ):
783 for nTries in range( self.nTries ):
784 self.fp.close()
784 self.fp.close()
785 self.fp = open( self.filename, 'rb' )
785 self.fp = open( self.filename, 'rb' )
786 self.fp.seek( currentPointer )
786 self.fp.seek( currentPointer )
787
787
788 self.fileSize = os.path.getsize( self.filename )
788 self.fileSize = os.path.getsize( self.filename )
789 currentSize = self.fileSize - currentPointer
789 currentSize = self.fileSize - currentPointer
790
790
791 if ( currentSize >= neededSize ):
791 if ( currentSize >= neededSize ):
792 return 1
792 return 1
793
793
794 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
794 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
795 sleep( self.delay )
795 sleep( self.delay )
796
796
797 return 0
797 return 0
798
798
799 def __jumpToLastBlock(self):
799 def __jumpToLastBlock(self):
800
800
801 if not(self.__isFirstTimeOnline):
801 if not(self.__isFirstTimeOnline):
802 return
802 return
803
803
804 csize = self.fileSize - self.fp.tell()
804 csize = self.fileSize - self.fp.tell()
805 blocksize = self.processingHeaderObj.blockSize
805 blocksize = self.processingHeaderObj.blockSize
806
806
807 #salta el primer bloque de datos
807 #salta el primer bloque de datos
808 if csize > self.processingHeaderObj.blockSize:
808 if csize > self.processingHeaderObj.blockSize:
809 self.fp.seek(self.fp.tell() + blocksize)
809 self.fp.seek(self.fp.tell() + blocksize)
810 else:
810 else:
811 return
811 return
812
812
813 csize = self.fileSize - self.fp.tell()
813 csize = self.fileSize - self.fp.tell()
814 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
814 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
815 while True:
815 while True:
816
816
817 if self.fp.tell()<self.fileSize:
817 if self.fp.tell()<self.fileSize:
818 self.fp.seek(self.fp.tell() + neededsize)
818 self.fp.seek(self.fp.tell() + neededsize)
819 else:
819 else:
820 self.fp.seek(self.fp.tell() - neededsize)
820 self.fp.seek(self.fp.tell() - neededsize)
821 break
821 break
822
822
823 # csize = self.fileSize - self.fp.tell()
823 # csize = self.fileSize - self.fp.tell()
824 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
824 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
825 # factor = int(csize/neededsize)
825 # factor = int(csize/neededsize)
826 # if factor > 0:
826 # if factor > 0:
827 # self.fp.seek(self.fp.tell() + factor*neededsize)
827 # self.fp.seek(self.fp.tell() + factor*neededsize)
828
828
829 self.flagIsNewFile = 0
829 self.flagIsNewFile = 0
830 self.__isFirstTimeOnline = 0
830 self.__isFirstTimeOnline = 0
831
831
832 def __setNewBlock(self):
832 def __setNewBlock(self):
833
833
834 if self.fp == None:
834 if self.fp == None:
835 return 0
835 return 0
836
836
837 if self.online:
837 if self.online:
838 self.__jumpToLastBlock()
838 self.__jumpToLastBlock()
839
839
840 if self.flagIsNewFile:
840 if self.flagIsNewFile:
841 return 1
841 return 1
842
842
843 self.lastUTTime = self.basicHeaderObj.utc
843 self.lastUTTime = self.basicHeaderObj.utc
844 currentSize = self.fileSize - self.fp.tell()
844 currentSize = self.fileSize - self.fp.tell()
845 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
845 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
846
846
847 if (currentSize >= neededSize):
847 if (currentSize >= neededSize):
848 self.basicHeaderObj.read(self.fp)
848 self.basicHeaderObj.read(self.fp)
849 return 1
849 return 1
850
850
851 if self.__waitNewBlock():
851 if self.__waitNewBlock():
852 return 1
852 return 1
853
853
854 if not(self.setNextFile()):
854 if not(self.setNextFile()):
855 return 0
855 return 0
856
856
857 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
857 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
858
858
859 self.flagDiscontinuousBlock = 0
859 self.flagDiscontinuousBlock = 0
860
860
861 if deltaTime > self.maxTimeStep:
861 if deltaTime > self.maxTimeStep:
862 self.flagDiscontinuousBlock = 1
862 self.flagDiscontinuousBlock = 1
863
863
864 return 1
864 return 1
865
865
866 def readNextBlock(self):
866 def readNextBlock(self):
867 if not(self.__setNewBlock()):
867 if not(self.__setNewBlock()):
868 return 0
868 return 0
869
869
870 if not(self.readBlock()):
870 if not(self.readBlock()):
871 return 0
871 return 0
872
872
873 return 1
873 return 1
874
874
875 def __readFirstHeader(self):
875 def __readFirstHeader(self):
876
876
877 self.basicHeaderObj.read(self.fp)
877 self.basicHeaderObj.read(self.fp)
878 self.systemHeaderObj.read(self.fp)
878 self.systemHeaderObj.read(self.fp)
879 self.radarControllerHeaderObj.read(self.fp)
879 self.radarControllerHeaderObj.read(self.fp)
880 self.processingHeaderObj.read(self.fp)
880 self.processingHeaderObj.read(self.fp)
881
881
882 self.firstHeaderSize = self.basicHeaderObj.size
882 self.firstHeaderSize = self.basicHeaderObj.size
883
883
884 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
884 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
885 if datatype == 0:
885 if datatype == 0:
886 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
886 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
887 elif datatype == 1:
887 elif datatype == 1:
888 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
888 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
889 elif datatype == 2:
889 elif datatype == 2:
890 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
890 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
891 elif datatype == 3:
891 elif datatype == 3:
892 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
892 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
893 elif datatype == 4:
893 elif datatype == 4:
894 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
894 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
895 elif datatype == 5:
895 elif datatype == 5:
896 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
896 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
897 else:
897 else:
898 raise ValueError, 'Data type was not defined'
898 raise ValueError, 'Data type was not defined'
899
899
900 self.dtype = datatype_str
900 self.dtype = datatype_str
901 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
901 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
902 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
902 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
903 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
903 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
904 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
904 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
905 self.getBlockDimension()
905 self.getBlockDimension()
906
906
907 def __verifyFile(self, filename, msgFlag=True):
907 def __verifyFile(self, filename, msgFlag=True):
908 msg = None
908 msg = None
909 try:
909 try:
910 fp = open(filename, 'rb')
910 fp = open(filename, 'rb')
911 currentPosition = fp.tell()
911 currentPosition = fp.tell()
912 except IOError:
912 except IOError:
913 traceback.print_exc()
913 traceback.print_exc()
914 if msgFlag:
914 if msgFlag:
915 print "[Reading] The file %s can't be opened" % (filename)
915 print "[Reading] The file %s can't be opened" % (filename)
916 return False
916 return False
917
917
918 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
918 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
919
919
920 if neededSize == 0:
920 if neededSize == 0:
921 basicHeaderObj = BasicHeader(LOCALTIME)
921 basicHeaderObj = BasicHeader(LOCALTIME)
922 systemHeaderObj = SystemHeader()
922 systemHeaderObj = SystemHeader()
923 radarControllerHeaderObj = RadarControllerHeader()
923 radarControllerHeaderObj = RadarControllerHeader()
924 processingHeaderObj = ProcessingHeader()
924 processingHeaderObj = ProcessingHeader()
925
925
926 try:
926 try:
927 if not( basicHeaderObj.read(fp) ): raise IOError
927 if not( basicHeaderObj.read(fp) ): raise IOError
928 if not( systemHeaderObj.read(fp) ): raise IOError
928 if not( systemHeaderObj.read(fp) ): raise IOError
929 if not( radarControllerHeaderObj.read(fp) ): raise IOError
929 if not( radarControllerHeaderObj.read(fp) ): raise IOError
930 if not( processingHeaderObj.read(fp) ): raise IOError
930 if not( processingHeaderObj.read(fp) ): raise IOError
931 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
931 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
932
932
933 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
933 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
934
934
935 except IOError:
935 except IOError:
936 traceback.print_exc()
936 traceback.print_exc()
937 if msgFlag:
937 if msgFlag:
938 print "[Reading] The file %s is empty or it hasn't enough data" % filename
938 print "[Reading] The file %s is empty or it hasn't enough data" % filename
939
939
940 fp.close()
940 fp.close()
941 return False
941 return False
942 else:
942 else:
943 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
943 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
944
944
945 fp.close()
945 fp.close()
946 fileSize = os.path.getsize(filename)
946 fileSize = os.path.getsize(filename)
947 currentSize = fileSize - currentPosition
947 currentSize = fileSize - currentPosition
948 if currentSize < neededSize:
948 if currentSize < neededSize:
949 if msgFlag and (msg != None):
949 if msgFlag and (msg != None):
950 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
950 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
951 return False
951 return False
952
952
953 return True
953 return True
954
954
955 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
955 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
956
956
957 dateList = []
957 dateList = []
958 pathList = []
958 pathList = []
959
959
960 if not walk:
960 if not walk:
961 #pathList.append(path)
961 #pathList.append(path)
962 multi_path = path.split(',')
962 multi_path = path.split(',')
963 for single_path in multi_path:
963 for single_path in multi_path:
964
964
965 ok = False
965 ok = False
966 fileList = glob.glob1(single_path, "*"+ext)
966 fileList = glob.glob1(single_path, "*"+ext)
967
967
968 for thisFile in fileList:
968 for thisFile in fileList:
969
969
970 if not os.path.isfile(os.path.join(single_path, thisFile)):
970 if not os.path.isfile(os.path.join(single_path, thisFile)):
971 continue
971 continue
972
972
973 if not isRadarFile(thisFile):
973 if not isRadarFile(thisFile):
974 continue
974 continue
975
975
976 ok = True
976 ok = True
977 thisDate = getDateFromRadarFile(thisFile)
977 thisDate = getDateFromRadarFile(thisFile)
978
978
979 if thisDate not in dateList:
979 if thisDate not in dateList:
980 dateList.append(thisDate)
980 dateList.append(thisDate)
981
981
982 if ok:
982 if ok:
983 pathList.append(single_path)
983 pathList.append(single_path)
984
984
985 return dateList
985 return dateList
986
986
987 multi_path = path.split(',')
987 multi_path = path.split(',')
988 for single_path in multi_path:
988 for single_path in multi_path:
989
989
990 dirList = []
990 dirList = []
991
991
992 for thisPath in os.listdir(single_path):
992 for thisPath in os.listdir(single_path):
993
993
994 if not os.path.isdir(os.path.join(single_path,thisPath)):
994 if not os.path.isdir(os.path.join(single_path,thisPath)):
995 continue
995 continue
996
996
997 if not isRadarFolder(thisPath):
997 if not isRadarFolder(thisPath):
998 continue
998 continue
999
999
1000 dirList.append(thisPath)
1000 dirList.append(thisPath)
1001
1001
1002 if not dirList:
1002 if not dirList:
1003 return dateList
1003 return dateList
1004
1004
1005 if startDate and endDate:
1005 if startDate and endDate:
1006 thisDate = startDate
1006 thisDate = startDate
1007
1007
1008 while(thisDate <= endDate):
1008 while(thisDate <= endDate):
1009 year = thisDate.timetuple().tm_year
1009 year = thisDate.timetuple().tm_year
1010 doy = thisDate.timetuple().tm_yday
1010 doy = thisDate.timetuple().tm_yday
1011
1011
1012 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1012 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1013 if len(matchlist) == 0:
1013 if len(matchlist) == 0:
1014 thisDate += datetime.timedelta(1)
1014 thisDate += datetime.timedelta(1)
1015 continue
1015 continue
1016
1016
1017 for match in matchlist:
1017 for match in matchlist:
1018 pathList.append(os.path.join(single_path,match,expLabel))
1018 pathList.append(os.path.join(single_path,match,expLabel))
1019 dateList.append(thisDate)
1019 dateList.append(thisDate)
1020
1020
1021 thisDate += datetime.timedelta(1)
1021 thisDate += datetime.timedelta(1)
1022 else:
1022 else:
1023 for thiDir in dirList:
1023 for thisDir in dirList:
1024 year = int(folder[1:5])
1024 year = int(thisDir[1:5])
1025 doy = int(folder[5:8])
1025 doy = int(thisDir[5:8])
1026 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1026 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1027
1027
1028 pathList.append(os.path.join(single_path,thiDir,expLabel))
1028 pathList.append(os.path.join(single_path,thisDir,expLabel))
1029 dateList.append(thisDate)
1029 dateList.append(thisDate)
1030
1030
1031 return dateList
1031 return dateList
1032
1032
1033
1033
1034 def setup(self,
1034 def setup(self,
1035 path=None,
1035 path=None,
1036 startDate=None,
1036 startDate=None,
1037 endDate=None,
1037 endDate=None,
1038 startTime=datetime.time(0,0,0),
1038 startTime=datetime.time(0,0,0),
1039 endTime=datetime.time(23,59,59),
1039 endTime=datetime.time(23,59,59),
1040 set=None,
1040 set=None,
1041 expLabel = "",
1041 expLabel = "",
1042 ext = None,
1042 ext = None,
1043 online = False,
1043 online = False,
1044 delay = 60,
1044 delay = 60,
1045 walk = True,
1045 walk = True,
1046 getblock = False,
1046 getblock = False,
1047 nTxs = 1):
1047 nTxs = 1):
1048
1048
1049 if path == None:
1049 if path == None:
1050 raise ValueError, "[Reading] The path is not valid"
1050 raise ValueError, "[Reading] The path is not valid"
1051
1051
1052 if ext == None:
1052 if ext == None:
1053 ext = self.ext
1053 ext = self.ext
1054
1054
1055 if online:
1055 if online:
1056 print "[Reading] Searching files in online mode..."
1056 print "[Reading] Searching files in online mode..."
1057
1057
1058 for nTries in range( self.nTries ):
1058 for nTries in range( self.nTries ):
1059 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1059 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1060
1060
1061 if fullpath:
1061 if fullpath:
1062 break
1062 break
1063
1063
1064 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1064 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1065 sleep( self.delay )
1065 sleep( self.delay )
1066
1066
1067 if not(fullpath):
1067 if not(fullpath):
1068 print "[Reading] There 'isn't any valid file in %s" % path
1068 print "[Reading] There 'isn't any valid file in %s" % path
1069 return None
1069 return None
1070
1070
1071 self.year = year
1071 self.year = year
1072 self.doy = doy
1072 self.doy = doy
1073 self.set = set - 1
1073 self.set = set - 1
1074 self.path = path
1074 self.path = path
1075 self.foldercounter = foldercounter
1075 self.foldercounter = foldercounter
1076 last_set = None
1076 last_set = None
1077
1077
1078 else:
1078 else:
1079 print "[Reading] Searching files in offline mode ..."
1079 print "[Reading] Searching files in offline mode ..."
1080 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1080 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1081 startTime=startTime, endTime=endTime,
1081 startTime=startTime, endTime=endTime,
1082 set=set, expLabel=expLabel, ext=ext,
1082 set=set, expLabel=expLabel, ext=ext,
1083 walk=walk)
1083 walk=walk)
1084
1084
1085 if not(pathList):
1085 if not(pathList):
1086 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1086 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1087 datetime.datetime.combine(startDate,startTime).ctime(),
1087 datetime.datetime.combine(startDate,startTime).ctime(),
1088 datetime.datetime.combine(endDate,endTime).ctime())
1088 datetime.datetime.combine(endDate,endTime).ctime())
1089
1089
1090 sys.exit(-1)
1090 sys.exit(-1)
1091
1091
1092
1092
1093 self.fileIndex = -1
1093 self.fileIndex = -1
1094 self.pathList = pathList
1094 self.pathList = pathList
1095 self.filenameList = filenameList
1095 self.filenameList = filenameList
1096 file_name = os.path.basename(filenameList[-1])
1096 file_name = os.path.basename(filenameList[-1])
1097 basename, ext = os.path.splitext(file_name)
1097 basename, ext = os.path.splitext(file_name)
1098 last_set = int(basename[-3:])
1098 last_set = int(basename[-3:])
1099
1099
1100 self.online = online
1100 self.online = online
1101 self.delay = delay
1101 self.delay = delay
1102 ext = ext.lower()
1102 ext = ext.lower()
1103 self.ext = ext
1103 self.ext = ext
1104 self.getByBlock = getblock
1104 self.getByBlock = getblock
1105 self.nTxs = int(nTxs)
1105 self.nTxs = int(nTxs)
1106
1106
1107 if not(self.setNextFile()):
1107 if not(self.setNextFile()):
1108 if (startDate!=None) and (endDate!=None):
1108 if (startDate!=None) and (endDate!=None):
1109 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1109 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1110 elif startDate != None:
1110 elif startDate != None:
1111 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1111 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1112 else:
1112 else:
1113 print "[Reading] No files"
1113 print "[Reading] No files"
1114
1114
1115 sys.exit(-1)
1115 sys.exit(-1)
1116
1116
1117 # self.updateDataHeader()
1117 # self.updateDataHeader()
1118 if last_set != None:
1118 if last_set != None:
1119 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1119 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1120 return
1120 return
1121
1121
1122 def getBasicHeader(self):
1122 def getBasicHeader(self):
1123
1123
1124 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1124 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1125
1125
1126 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1126 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1127
1127
1128 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1128 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1129
1129
1130 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1130 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1131
1131
1132 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1132 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1133
1133
1134 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1134 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1135
1135
1136 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1136 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1137
1137
1138 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1138 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1139
1139
1140
1140
1141 def getFirstHeader(self):
1141 def getFirstHeader(self):
1142
1142
1143 raise ValueError, "This method has not been implemented"
1143 raise ValueError, "This method has not been implemented"
1144
1144
1145 def getData(self):
1145 def getData(self):
1146
1146
1147 raise ValueError, "This method has not been implemented"
1147 raise ValueError, "This method has not been implemented"
1148
1148
1149 def hasNotDataInBuffer(self):
1149 def hasNotDataInBuffer(self):
1150
1150
1151 raise ValueError, "This method has not been implemented"
1151 raise ValueError, "This method has not been implemented"
1152
1152
1153 def readBlock(self):
1153 def readBlock(self):
1154
1154
1155 raise ValueError, "This method has not been implemented"
1155 raise ValueError, "This method has not been implemented"
1156
1156
1157 def isEndProcess(self):
1157 def isEndProcess(self):
1158
1158
1159 return self.flagNoMoreFiles
1159 return self.flagNoMoreFiles
1160
1160
1161 def printReadBlocks(self):
1161 def printReadBlocks(self):
1162
1162
1163 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1163 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1164
1164
1165 def printTotalBlocks(self):
1165 def printTotalBlocks(self):
1166
1166
1167 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1167 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1168
1168
1169 def printNumberOfBlock(self):
1169 def printNumberOfBlock(self):
1170
1170
1171 if self.flagIsNewBlock:
1171 if self.flagIsNewBlock:
1172 print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1172 print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1173 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1173 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1174
1174
1175 def printInfo(self):
1175 def printInfo(self):
1176
1176
1177 if self.__printInfo == False:
1177 if self.__printInfo == False:
1178 return
1178 return
1179
1179
1180 self.basicHeaderObj.printInfo()
1180 self.basicHeaderObj.printInfo()
1181 self.systemHeaderObj.printInfo()
1181 self.systemHeaderObj.printInfo()
1182 self.radarControllerHeaderObj.printInfo()
1182 self.radarControllerHeaderObj.printInfo()
1183 self.processingHeaderObj.printInfo()
1183 self.processingHeaderObj.printInfo()
1184
1184
1185 self.__printInfo = False
1185 self.__printInfo = False
1186
1186
1187
1187
1188 def run(self, **kwargs):
1188 def run(self, **kwargs):
1189
1189
1190 if not(self.isConfig):
1190 if not(self.isConfig):
1191
1191
1192 # self.dataOut = dataOut
1192 # self.dataOut = dataOut
1193 self.setup(**kwargs)
1193 self.setup(**kwargs)
1194 self.isConfig = True
1194 self.isConfig = True
1195
1195
1196 self.getData()
1196 self.getData()
1197
1197
1198 class JRODataWriter(JRODataIO):
1198 class JRODataWriter(JRODataIO):
1199
1199
1200 """
1200 """
1201 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1201 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1202 de los datos siempre se realiza por bloques.
1202 de los datos siempre se realiza por bloques.
1203 """
1203 """
1204
1204
1205 blockIndex = 0
1205 blockIndex = 0
1206
1206
1207 path = None
1207 path = None
1208
1208
1209 setFile = None
1209 setFile = None
1210
1210
1211 profilesPerBlock = None
1211 profilesPerBlock = None
1212
1212
1213 blocksPerFile = None
1213 blocksPerFile = None
1214
1214
1215 nWriteBlocks = 0
1215 nWriteBlocks = 0
1216
1216
1217 def __init__(self, dataOut=None):
1217 def __init__(self, dataOut=None):
1218 raise ValueError, "Not implemented"
1218 raise ValueError, "Not implemented"
1219
1219
1220
1220
1221 def hasAllDataInBuffer(self):
1221 def hasAllDataInBuffer(self):
1222 raise ValueError, "Not implemented"
1222 raise ValueError, "Not implemented"
1223
1223
1224
1224
1225 def setBlockDimension(self):
1225 def setBlockDimension(self):
1226 raise ValueError, "Not implemented"
1226 raise ValueError, "Not implemented"
1227
1227
1228
1228
1229 def writeBlock(self):
1229 def writeBlock(self):
1230 raise ValueError, "No implemented"
1230 raise ValueError, "No implemented"
1231
1231
1232
1232
1233 def putData(self):
1233 def putData(self):
1234 raise ValueError, "No implemented"
1234 raise ValueError, "No implemented"
1235
1235
1236
1236
1237 def setBasicHeader(self):
1237 def setBasicHeader(self):
1238
1238
1239 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1239 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1240 self.basicHeaderObj.version = self.versionFile
1240 self.basicHeaderObj.version = self.versionFile
1241 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1241 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1242
1242
1243 utc = numpy.floor(self.dataOut.utctime)
1243 utc = numpy.floor(self.dataOut.utctime)
1244 milisecond = (self.dataOut.utctime - utc)* 1000.0
1244 milisecond = (self.dataOut.utctime - utc)* 1000.0
1245
1245
1246 self.basicHeaderObj.utc = utc
1246 self.basicHeaderObj.utc = utc
1247 self.basicHeaderObj.miliSecond = milisecond
1247 self.basicHeaderObj.miliSecond = milisecond
1248 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1248 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1249 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1249 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1250 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1250 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1251
1251
1252 def setFirstHeader(self):
1252 def setFirstHeader(self):
1253 """
1253 """
1254 Obtiene una copia del First Header
1254 Obtiene una copia del First Header
1255
1255
1256 Affected:
1256 Affected:
1257
1257
1258 self.basicHeaderObj
1258 self.basicHeaderObj
1259 self.systemHeaderObj
1259 self.systemHeaderObj
1260 self.radarControllerHeaderObj
1260 self.radarControllerHeaderObj
1261 self.processingHeaderObj self.
1261 self.processingHeaderObj self.
1262
1262
1263 Return:
1263 Return:
1264 None
1264 None
1265 """
1265 """
1266
1266
1267 raise ValueError, "No implemented"
1267 raise ValueError, "No implemented"
1268
1268
1269 def __writeFirstHeader(self):
1269 def __writeFirstHeader(self):
1270 """
1270 """
1271 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1271 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1272
1272
1273 Affected:
1273 Affected:
1274 __dataType
1274 __dataType
1275
1275
1276 Return:
1276 Return:
1277 None
1277 None
1278 """
1278 """
1279
1279
1280 # CALCULAR PARAMETROS
1280 # CALCULAR PARAMETROS
1281
1281
1282 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1282 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1283 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1283 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1284
1284
1285 self.basicHeaderObj.write(self.fp)
1285 self.basicHeaderObj.write(self.fp)
1286 self.systemHeaderObj.write(self.fp)
1286 self.systemHeaderObj.write(self.fp)
1287 self.radarControllerHeaderObj.write(self.fp)
1287 self.radarControllerHeaderObj.write(self.fp)
1288 self.processingHeaderObj.write(self.fp)
1288 self.processingHeaderObj.write(self.fp)
1289
1289
1290 self.dtype = self.dataOut.dtype
1290 self.dtype = self.dataOut.dtype
1291
1291
1292 def __setNewBlock(self):
1292 def __setNewBlock(self):
1293 """
1293 """
1294 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1294 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1295
1295
1296 Return:
1296 Return:
1297 0 : si no pudo escribir nada
1297 0 : si no pudo escribir nada
1298 1 : Si escribio el Basic el First Header
1298 1 : Si escribio el Basic el First Header
1299 """
1299 """
1300 if self.fp == None:
1300 if self.fp == None:
1301 self.setNextFile()
1301 self.setNextFile()
1302
1302
1303 if self.flagIsNewFile:
1303 if self.flagIsNewFile:
1304 return 1
1304 return 1
1305
1305
1306 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1306 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1307 self.basicHeaderObj.write(self.fp)
1307 self.basicHeaderObj.write(self.fp)
1308 return 1
1308 return 1
1309
1309
1310 if not( self.setNextFile() ):
1310 if not( self.setNextFile() ):
1311 return 0
1311 return 0
1312
1312
1313 return 1
1313 return 1
1314
1314
1315
1315
1316 def writeNextBlock(self):
1316 def writeNextBlock(self):
1317 """
1317 """
1318 Selecciona el bloque siguiente de datos y los escribe en un file
1318 Selecciona el bloque siguiente de datos y los escribe en un file
1319
1319
1320 Return:
1320 Return:
1321 0 : Si no hizo pudo escribir el bloque de datos
1321 0 : Si no hizo pudo escribir el bloque de datos
1322 1 : Si no pudo escribir el bloque de datos
1322 1 : Si no pudo escribir el bloque de datos
1323 """
1323 """
1324 if not( self.__setNewBlock() ):
1324 if not( self.__setNewBlock() ):
1325 return 0
1325 return 0
1326
1326
1327 self.writeBlock()
1327 self.writeBlock()
1328
1328
1329 return 1
1329 return 1
1330
1330
1331 def setNextFile(self):
1331 def setNextFile(self):
1332 """
1332 """
1333 Determina el siguiente file que sera escrito
1333 Determina el siguiente file que sera escrito
1334
1334
1335 Affected:
1335 Affected:
1336 self.filename
1336 self.filename
1337 self.subfolder
1337 self.subfolder
1338 self.fp
1338 self.fp
1339 self.setFile
1339 self.setFile
1340 self.flagIsNewFile
1340 self.flagIsNewFile
1341
1341
1342 Return:
1342 Return:
1343 0 : Si el archivo no puede ser escrito
1343 0 : Si el archivo no puede ser escrito
1344 1 : Si el archivo esta listo para ser escrito
1344 1 : Si el archivo esta listo para ser escrito
1345 """
1345 """
1346 ext = self.ext
1346 ext = self.ext
1347 path = self.path
1347 path = self.path
1348
1348
1349 if self.fp != None:
1349 if self.fp != None:
1350 self.fp.close()
1350 self.fp.close()
1351
1351
1352 timeTuple = time.localtime( self.dataOut.utctime)
1352 timeTuple = time.localtime( self.dataOut.utctime)
1353 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1353 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1354
1354
1355 fullpath = os.path.join( path, subfolder )
1355 fullpath = os.path.join( path, subfolder )
1356 if not( os.path.exists(fullpath) ):
1356 if not( os.path.exists(fullpath) ):
1357 os.mkdir(fullpath)
1357 os.mkdir(fullpath)
1358 self.setFile = -1 #inicializo mi contador de seteo
1358 self.setFile = -1 #inicializo mi contador de seteo
1359 else:
1359 else:
1360 filesList = os.listdir( fullpath )
1360 filesList = os.listdir( fullpath )
1361 if len( filesList ) > 0:
1361 if len( filesList ) > 0:
1362 filesList = sorted( filesList, key=str.lower )
1362 filesList = sorted( filesList, key=str.lower )
1363 filen = filesList[-1]
1363 filen = filesList[-1]
1364 # el filename debera tener el siguiente formato
1364 # el filename debera tener el siguiente formato
1365 # 0 1234 567 89A BCDE (hex)
1365 # 0 1234 567 89A BCDE (hex)
1366 # x YYYY DDD SSS .ext
1366 # x YYYY DDD SSS .ext
1367 if isNumber( filen[8:11] ):
1367 if isNumber( filen[8:11] ):
1368 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1368 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1369 else:
1369 else:
1370 self.setFile = -1
1370 self.setFile = -1
1371 else:
1371 else:
1372 self.setFile = -1 #inicializo mi contador de seteo
1372 self.setFile = -1 #inicializo mi contador de seteo
1373
1373
1374 setFile = self.setFile
1374 setFile = self.setFile
1375 setFile += 1
1375 setFile += 1
1376
1376
1377 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1377 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1378 timeTuple.tm_year,
1378 timeTuple.tm_year,
1379 timeTuple.tm_yday,
1379 timeTuple.tm_yday,
1380 setFile,
1380 setFile,
1381 ext )
1381 ext )
1382
1382
1383 filename = os.path.join( path, subfolder, filen )
1383 filename = os.path.join( path, subfolder, filen )
1384
1384
1385 fp = open( filename,'wb' )
1385 fp = open( filename,'wb' )
1386
1386
1387 self.blockIndex = 0
1387 self.blockIndex = 0
1388
1388
1389 #guardando atributos
1389 #guardando atributos
1390 self.filename = filename
1390 self.filename = filename
1391 self.subfolder = subfolder
1391 self.subfolder = subfolder
1392 self.fp = fp
1392 self.fp = fp
1393 self.setFile = setFile
1393 self.setFile = setFile
1394 self.flagIsNewFile = 1
1394 self.flagIsNewFile = 1
1395
1395
1396 self.setFirstHeader()
1396 self.setFirstHeader()
1397
1397
1398 print '[Writing] file: %s'%self.filename
1398 print '[Writing] file: %s'%self.filename
1399
1399
1400 self.__writeFirstHeader()
1400 self.__writeFirstHeader()
1401
1401
1402 return 1
1402 return 1
1403
1403
1404 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1404 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1405 """
1405 """
1406 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1406 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1407
1407
1408 Inputs:
1408 Inputs:
1409 path : el path destino en el cual se escribiran los files a crear
1409 path : el path destino en el cual se escribiran los files a crear
1410 format : formato en el cual sera salvado un file
1410 format : formato en el cual sera salvado un file
1411 set : el setebo del file
1411 set : el setebo del file
1412
1412
1413 Return:
1413 Return:
1414 0 : Si no realizo un buen seteo
1414 0 : Si no realizo un buen seteo
1415 1 : Si realizo un buen seteo
1415 1 : Si realizo un buen seteo
1416 """
1416 """
1417
1417
1418 if ext == None:
1418 if ext == None:
1419 ext = self.ext
1419 ext = self.ext
1420
1420
1421 ext = ext.lower()
1421 ext = ext.lower()
1422
1422
1423 self.ext = ext
1423 self.ext = ext
1424
1424
1425 self.path = path
1425 self.path = path
1426
1426
1427 self.setFile = set - 1
1427 self.setFile = set - 1
1428
1428
1429 self.blocksPerFile = blocksPerFile
1429 self.blocksPerFile = blocksPerFile
1430
1430
1431 self.profilesPerBlock = profilesPerBlock
1431 self.profilesPerBlock = profilesPerBlock
1432
1432
1433 self.dataOut = dataOut
1433 self.dataOut = dataOut
1434
1434
1435 if not(self.setNextFile()):
1435 if not(self.setNextFile()):
1436 print "[Writing] There isn't a next file"
1436 print "[Writing] There isn't a next file"
1437 return 0
1437 return 0
1438
1438
1439 self.setBlockDimension()
1439 self.setBlockDimension()
1440
1440
1441 return 1
1441 return 1
1442
1442
1443 def run(self, dataOut, **kwargs):
1443 def run(self, dataOut, **kwargs):
1444
1444
1445 if not(self.isConfig):
1445 if not(self.isConfig):
1446
1446
1447 self.setup(dataOut, **kwargs)
1447 self.setup(dataOut, **kwargs)
1448 self.isConfig = True
1448 self.isConfig = True
1449
1449
1450 self.putData()
1450 self.putData()
1451
1451
General Comments 0
You need to be logged in to leave comments. Login now