##// END OF EJS Templates
schain
RSS
Alexander Valdez -
r563:d1e598150683
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1344 +1,1349
1 '''
1 '''
2
2
3 '''
3 '''
4 import os
4 import os
5 import sys
5 import sys
6 import glob
6 import glob
7 import time
7 import time
8 import numpy
8 import numpy
9 import fnmatch
9 import fnmatch
10 import time, datetime
10 import time, datetime
11 #import h5py
11 #import h5py
12 import traceback
12 import traceback
13
13
14 #try:
14 #try:
15 # import pyfits
15 # import pyfits
16 #except:
16 #except:
17 # print "pyfits module has not been imported, it should be installed to save files in fits format"
17 # print "pyfits module has not been imported, it should be installed to save files in fits format"
18
18
19 #from jrodata import *
19 #from jrodata import *
20 #from jroheaderIO import *
20 #from jroheaderIO import *
21 #from jroprocessing import *
21 #from jroprocessing import *
22
22
23 #import re
23 #import re
24 #from xml.etree.ElementTree import Element, SubElement, ElementTree
24 #from xml.etree.ElementTree import Element, SubElement, ElementTree
25
25
26
26
27 LOCALTIME = True #-18000
27 LOCALTIME = True #-18000
28
28
29 from model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
29 from model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
30
30
31 def isNumber(str):
31 def isNumber(str):
32 """
32 """
33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34
34
35 Excepciones:
35 Excepciones:
36 Si un determinado string no puede ser convertido a numero
36 Si un determinado string no puede ser convertido a numero
37 Input:
37 Input:
38 str, string al cual se le analiza para determinar si convertible a un numero o no
38 str, string al cual se le analiza para determinar si convertible a un numero o no
39
39
40 Return:
40 Return:
41 True : si el string es uno numerico
41 True : si el string es uno numerico
42 False : no es un string numerico
42 False : no es un string numerico
43 """
43 """
44 try:
44 try:
45 float( str )
45 float( str )
46 return True
46 return True
47 except:
47 except:
48 return False
48 return False
49
49
50 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
50 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
51 """
51 """
52 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
52 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
53
53
54 Inputs:
54 Inputs:
55 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
55 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
56
56
57 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
57 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
58 segundos contados desde 01/01/1970.
58 segundos contados desde 01/01/1970.
59 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
59 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
60 segundos contados desde 01/01/1970.
60 segundos contados desde 01/01/1970.
61
61
62 Return:
62 Return:
63 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
63 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
64 fecha especificado, de lo contrario retorna False.
64 fecha especificado, de lo contrario retorna False.
65
65
66 Excepciones:
66 Excepciones:
67 Si el archivo no existe o no puede ser abierto
67 Si el archivo no existe o no puede ser abierto
68 Si la cabecera no puede ser leida.
68 Si la cabecera no puede ser leida.
69
69
70 """
70 """
71 basicHeaderObj = BasicHeader(LOCALTIME)
71 basicHeaderObj = BasicHeader(LOCALTIME)
72
72
73 try:
73 try:
74 fp = open(filename,'rb')
74 fp = open(filename,'rb')
75 except IOError:
75 except IOError:
76 traceback.print_exc()
76 traceback.print_exc()
77 raise IOError, "The file %s can't be opened" %(filename)
77 raise IOError, "The file %s can't be opened" %(filename)
78
78
79 sts = basicHeaderObj.read(fp)
79 sts = basicHeaderObj.read(fp)
80 fp.close()
80 fp.close()
81
81
82 if not(sts):
82 if not(sts):
83 print "Skipping the file %s because it has not a valid header" %(filename)
83 print "Skipping the file %s because it has not a valid header" %(filename)
84 return 0
84 return 0
85
85
86 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
86 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
87 return 0
87 return 0
88
88
89 return 1
89 return 1
90
90
91 def isFileinThisTime(filename, startTime, endTime):
91 def isFileinThisTime(filename, startTime, endTime):
92 """
92 """
93 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
93 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
94
94
95 Inputs:
95 Inputs:
96 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
96 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
97
97
98 startTime : tiempo inicial del rango seleccionado en formato datetime.time
98 startTime : tiempo inicial del rango seleccionado en formato datetime.time
99
99
100 endTime : tiempo final del rango seleccionado en formato datetime.time
100 endTime : tiempo final del rango seleccionado en formato datetime.time
101
101
102 Return:
102 Return:
103 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
103 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
104 fecha especificado, de lo contrario retorna False.
104 fecha especificado, de lo contrario retorna False.
105
105
106 Excepciones:
106 Excepciones:
107 Si el archivo no existe o no puede ser abierto
107 Si el archivo no existe o no puede ser abierto
108 Si la cabecera no puede ser leida.
108 Si la cabecera no puede ser leida.
109
109
110 """
110 """
111
111
112
112
113 try:
113 try:
114 fp = open(filename,'rb')
114 fp = open(filename,'rb')
115 except IOError:
115 except IOError:
116 traceback.print_exc()
116 traceback.print_exc()
117 raise IOError, "The file %s can't be opened" %(filename)
117 raise IOError, "The file %s can't be opened" %(filename)
118
118
119 basicHeaderObj = BasicHeader(LOCALTIME)
119 basicHeaderObj = BasicHeader(LOCALTIME)
120 sts = basicHeaderObj.read(fp)
120 sts = basicHeaderObj.read(fp)
121 fp.close()
121 fp.close()
122
122
123 thisDatetime = basicHeaderObj.datatime
123 thisDatetime = basicHeaderObj.datatime
124 thisTime = thisDatetime.time()
124 thisTime = thisDatetime.time()
125
125
126 if not(sts):
126 if not(sts):
127 print "Skipping the file %s because it has not a valid header" %(filename)
127 print "Skipping the file %s because it has not a valid header" %(filename)
128 return None
128 return None
129
129
130 if not ((startTime <= thisTime) and (endTime > thisTime)):
130 if not ((startTime <= thisTime) and (endTime > thisTime)):
131 return None
131 return None
132
132
133 return thisDatetime
133 return thisDatetime
134
134
135 def getFileFromSet(path, ext, set):
135 def getFileFromSet(path, ext, set):
136 validFilelist = []
136 validFilelist = []
137 fileList = os.listdir(path)
137 fileList = os.listdir(path)
138
138
139 # 0 1234 567 89A BCDE
139 # 0 1234 567 89A BCDE
140 # H YYYY DDD SSS .ext
140 # H YYYY DDD SSS .ext
141
141
142 for thisFile in fileList:
142 for thisFile in fileList:
143 try:
143 try:
144 year = int(thisFile[1:5])
144 year = int(thisFile[1:5])
145 doy = int(thisFile[5:8])
145 doy = int(thisFile[5:8])
146 except:
146 except:
147 continue
147 continue
148
148
149 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
149 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
150 continue
150 continue
151
151
152 validFilelist.append(thisFile)
152 validFilelist.append(thisFile)
153
153
154 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
154 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
155
155
156 if len(myfile)!= 0:
156 if len(myfile)!= 0:
157 return myfile[0]
157 return myfile[0]
158 else:
158 else:
159 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
159 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
160 print 'the filename %s does not exist'%filename
160 print 'the filename %s does not exist'%filename
161 print '...going to the last file: '
161 print '...going to the last file: '
162
162
163 if validFilelist:
163 if validFilelist:
164 validFilelist = sorted( validFilelist, key=str.lower )
164 validFilelist = sorted( validFilelist, key=str.lower )
165 return validFilelist[-1]
165 return validFilelist[-1]
166
166
167 return None
167 return None
168
168
169 def getlastFileFromPath(path, ext):
169 def getlastFileFromPath(path, ext):
170 """
170 """
171 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
171 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
172 al final de la depuracion devuelve el ultimo file de la lista que quedo.
172 al final de la depuracion devuelve el ultimo file de la lista que quedo.
173
173
174 Input:
174 Input:
175 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
175 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
176 ext : extension de los files contenidos en una carpeta
176 ext : extension de los files contenidos en una carpeta
177
177
178 Return:
178 Return:
179 El ultimo file de una determinada carpeta, no se considera el path.
179 El ultimo file de una determinada carpeta, no se considera el path.
180 """
180 """
181 validFilelist = []
181 validFilelist = []
182 fileList = os.listdir(path)
182 fileList = os.listdir(path)
183
183
184 # 0 1234 567 89A BCDE
184 # 0 1234 567 89A BCDE
185 # H YYYY DDD SSS .ext
185 # H YYYY DDD SSS .ext
186
186
187 for thisFile in fileList:
187 for thisFile in fileList:
188
188
189 year = thisFile[1:5]
189 year = thisFile[1:5]
190 if not isNumber(year):
190 if not isNumber(year):
191 continue
191 continue
192
192
193 doy = thisFile[5:8]
193 doy = thisFile[5:8]
194 if not isNumber(doy):
194 if not isNumber(doy):
195 continue
195 continue
196
196
197 year = int(year)
197 year = int(year)
198 doy = int(doy)
198 doy = int(doy)
199
199
200 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
200 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
201 continue
201 continue
202
202
203 validFilelist.append(thisFile)
203 validFilelist.append(thisFile)
204
204
205 if validFilelist:
205 if validFilelist:
206 validFilelist = sorted( validFilelist, key=str.lower )
206 validFilelist = sorted( validFilelist, key=str.lower )
207 return validFilelist[-1]
207 return validFilelist[-1]
208
208
209 return None
209 return None
210
210
211 def checkForRealPath(path, foldercounter, year, doy, set, ext):
211 def checkForRealPath(path, foldercounter, year, doy, set, ext):
212 """
212 """
213 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
213 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
214 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
214 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
215 el path exacto de un determinado file.
215 el path exacto de un determinado file.
216
216
217 Example :
217 Example :
218 nombre correcto del file es .../.../D2009307/P2009307367.ext
218 nombre correcto del file es .../.../D2009307/P2009307367.ext
219
219
220 Entonces la funcion prueba con las siguientes combinaciones
220 Entonces la funcion prueba con las siguientes combinaciones
221 .../.../y2009307367.ext
221 .../.../y2009307367.ext
222 .../.../Y2009307367.ext
222 .../.../Y2009307367.ext
223 .../.../x2009307/y2009307367.ext
223 .../.../x2009307/y2009307367.ext
224 .../.../x2009307/Y2009307367.ext
224 .../.../x2009307/Y2009307367.ext
225 .../.../X2009307/y2009307367.ext
225 .../.../X2009307/y2009307367.ext
226 .../.../X2009307/Y2009307367.ext
226 .../.../X2009307/Y2009307367.ext
227 siendo para este caso, la ultima combinacion de letras, identica al file buscado
227 siendo para este caso, la ultima combinacion de letras, identica al file buscado
228
228
229 Return:
229 Return:
230 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
230 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
231 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
231 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
232 para el filename
232 para el filename
233 """
233 """
234 fullfilename = None
234 fullfilename = None
235 find_flag = False
235 find_flag = False
236 filename = None
236 filename = None
237
237
238 prefixDirList = [None,'d','D']
238 prefixDirList = [None,'d','D']
239 if ext.lower() == ".r": #voltage
239 if ext.lower() == ".r": #voltage
240 prefixFileList = ['d','D']
240 prefixFileList = ['d','D']
241 elif ext.lower() == ".pdata": #spectra
241 elif ext.lower() == ".pdata": #spectra
242 prefixFileList = ['p','P']
242 prefixFileList = ['p','P']
243 else:
243 else:
244 return None, filename
244 return None, filename
245
245
246 #barrido por las combinaciones posibles
246 #barrido por las combinaciones posibles
247 for prefixDir in prefixDirList:
247 for prefixDir in prefixDirList:
248 thispath = path
248 thispath = path
249 if prefixDir != None:
249 if prefixDir != None:
250 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
250 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
251 if foldercounter == 0:
251 if foldercounter == 0:
252 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
252 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
253 else:
253 else:
254 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
254 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
255 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
255 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
256 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
256 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
257 fullfilename = os.path.join( thispath, filename ) #formo el path completo
257 fullfilename = os.path.join( thispath, filename ) #formo el path completo
258
258
259 if os.path.exists( fullfilename ): #verifico que exista
259 if os.path.exists( fullfilename ): #verifico que exista
260 find_flag = True
260 find_flag = True
261 break
261 break
262 if find_flag:
262 if find_flag:
263 break
263 break
264
264
265 if not(find_flag):
265 if not(find_flag):
266 return None, filename
266 return None, filename
267
267
268 return fullfilename, filename
268 return fullfilename, filename
269
269
270 def isDoyFolder(folder):
270 def isDoyFolder(folder):
271 try:
271 try:
272 year = int(folder[1:5])
272 year = int(folder[1:5])
273 except:
273 except:
274 return 0
274 return 0
275
275
276 try:
276 try:
277 doy = int(folder[5:8])
277 doy = int(folder[5:8])
278 except:
278 except:
279 return 0
279 return 0
280
280
281 return 1
281 return 1
282
282
283 class JRODataIO:
283 class JRODataIO:
284
284
285 c = 3E8
285 c = 3E8
286
286
287 isConfig = False
287 isConfig = False
288
288
289 basicHeaderObj = None
289 basicHeaderObj = None
290
290
291 systemHeaderObj = None
291 systemHeaderObj = None
292
292
293 radarControllerHeaderObj = None
293 radarControllerHeaderObj = None
294
294
295 processingHeaderObj = None
295 processingHeaderObj = None
296
296
297 online = 0
297 online = 0
298
298
299 dtype = None
299 dtype = None
300
300
301 pathList = []
301 pathList = []
302
302
303 filenameList = []
303 filenameList = []
304
304
305 filename = None
305 filename = None
306
306
307 ext = None
307 ext = None
308
308
309 flagIsNewFile = 1
309 flagIsNewFile = 1
310
310
311 flagTimeBlock = 0
311 flagTimeBlock = 0
312
312
313 flagIsNewBlock = 0
313 flagIsNewBlock = 0
314
314
315 fp = None
315 fp = None
316
316
317 firstHeaderSize = 0
317 firstHeaderSize = 0
318
318
319 basicHeaderSize = 24
319 basicHeaderSize = 24
320
320
321 versionFile = 1103
321 versionFile = 1103
322
322
323 fileSize = None
323 fileSize = None
324
324
325 # ippSeconds = None
325 # ippSeconds = None
326
326
327 fileSizeByHeader = None
327 fileSizeByHeader = None
328
328
329 fileIndex = None
329 fileIndex = None
330
330
331 profileIndex = None
331 profileIndex = None
332
332
333 blockIndex = None
333 blockIndex = None
334
334
335 nTotalBlocks = None
335 nTotalBlocks = None
336
336
337 maxTimeStep = 30
337 maxTimeStep = 30
338
338
339 lastUTTime = None
339 lastUTTime = None
340
340
341 datablock = None
341 datablock = None
342
342
343 dataOut = None
343 dataOut = None
344
344
345 blocksize = None
345 blocksize = None
346
346
347 getByBlock = False
347 getByBlock = False
348
348
349 def __init__(self):
349 def __init__(self):
350
350
351 raise ValueError, "Not implemented"
351 raise ValueError, "Not implemented"
352
352
353 def run(self):
353 def run(self):
354
354
355 raise ValueError, "Not implemented"
355 raise ValueError, "Not implemented"
356
356
357 class JRODataReader(JRODataIO):
357 class JRODataReader(JRODataIO):
358
358
359 nReadBlocks = 0
359 nReadBlocks = 0
360
360
361 delay = 10 #number of seconds waiting a new file
361 delay = 10 #number of seconds waiting a new file
362
362
363 nTries = 3 #quantity tries
363 nTries = 3 #quantity tries
364
364
365 nFiles = 3 #number of files for searching
365 nFiles = 3 #number of files for searching
366
366
367 path = None
367 path = None
368
368
369 foldercounter = 0
369 foldercounter = 0
370
370
371 flagNoMoreFiles = 0
371 flagNoMoreFiles = 0
372
372
373 datetimeList = []
373 datetimeList = []
374
374
375 __isFirstTimeOnline = 1
375 __isFirstTimeOnline = 1
376
376
377 __printInfo = True
377 __printInfo = True
378
378
379 profileIndex = None
379 profileIndex = None
380
380
381 nTxs = 1
381 nTxs = 1
382
382
383 txIndex = None
383 txIndex = None
384
384
385 def __init__(self):
385 def __init__(self):
386
386
387 """
387 """
388
388
389 """
389 """
390
390
391 raise ValueError, "This method has not been implemented"
391 raise ValueError, "This method has not been implemented"
392
392
393
393
394 def createObjByDefault(self):
394 def createObjByDefault(self):
395 """
395 """
396
396
397 """
397 """
398 raise ValueError, "This method has not been implemented"
398 raise ValueError, "This method has not been implemented"
399
399
400 def getBlockDimension(self):
400 def getBlockDimension(self):
401
401
402 raise ValueError, "No implemented"
402 raise ValueError, "No implemented"
403
403
404 def __searchFilesOffLine(self,
404 def __searchFilesOffLine(self,
405 path,
405 path,
406 startDate,
406 startDate,
407 endDate,
407 endDate,
408 startTime=datetime.time(0,0,0),
408 startTime=datetime.time(0,0,0),
409 endTime=datetime.time(23,59,59),
409 endTime=datetime.time(23,59,59),
410 set=None,
410 set=None,
411 expLabel='',
411 expLabel='',
412 ext='.r',
412 ext='.r',
413 walk=True):
413 walk=True):
414
414
415 pathList = []
415 pathList = []
416
416
417 if not walk:
417 if not walk:
418 #pathList.append(path)
418 #pathList.append(path)
419 multi_path = path.split(',')
419 multi_path = path.split(',')
420 for single_path in multi_path:
420 for single_path in multi_path:
421 pathList.append(single_path)
421 pathList.append(single_path)
422
422
423 else:
423 else:
424 #dirList = []
424 #dirList = []
425 multi_path = path.split(',')
425 multi_path = path.split(',')
426 for single_path in multi_path:
426 for single_path in multi_path:
427 dirList = []
427 dirList = []
428 for thisPath in os.listdir(single_path):
428 for thisPath in os.listdir(single_path):
429 if not os.path.isdir(os.path.join(single_path,thisPath)):
429 if not os.path.isdir(os.path.join(single_path,thisPath)):
430 continue
430 continue
431 if not isDoyFolder(thisPath):
431 if not isDoyFolder(thisPath):
432 continue
432 continue
433
433
434 dirList.append(thisPath)
434 dirList.append(thisPath)
435
435
436 if not(dirList):
436 if not(dirList):
437 return None, None
437 return None, None
438
438
439 thisDate = startDate
439 thisDate = startDate
440
440
441 while(thisDate <= endDate):
441 while(thisDate <= endDate):
442 year = thisDate.timetuple().tm_year
442 year = thisDate.timetuple().tm_year
443 doy = thisDate.timetuple().tm_yday
443 doy = thisDate.timetuple().tm_yday
444
444
445 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
445 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
446 if len(matchlist) == 0:
446 if len(matchlist) == 0:
447 thisDate += datetime.timedelta(1)
447 thisDate += datetime.timedelta(1)
448 continue
448 continue
449 for match in matchlist:
449 for match in matchlist:
450 pathList.append(os.path.join(single_path,match,expLabel))
450 pathList.append(os.path.join(single_path,match,expLabel))
451
451
452 thisDate += datetime.timedelta(1)
452 thisDate += datetime.timedelta(1)
453
453
454 if pathList == []:
454 if pathList == []:
455 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
455 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
456 return None, None
456 return None, None
457
457
458 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
458 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
459
459
460 filenameList = []
460 filenameList = []
461 datetimeList = []
461 datetimeList = []
462 pathDict = {}
462 pathDict = {}
463 filenameList_to_sort = []
463 filenameList_to_sort = []
464
464
465 for i in range(len(pathList)):
465 for i in range(len(pathList)):
466
466
467 thisPath = pathList[i]
467 thisPath = pathList[i]
468
468
469 fileList = glob.glob1(thisPath, "*%s" %ext)
469 fileList = glob.glob1(thisPath, "*%s" %ext)
470 fileList.sort()
470 fileList.sort()
471 pathDict.setdefault(fileList[0])
471 pathDict.setdefault(fileList[0])
472 pathDict[fileList[0]] = i
472 pathDict[fileList[0]] = i
473 filenameList_to_sort.append(fileList[0])
473 filenameList_to_sort.append(fileList[0])
474
474
475 filenameList_to_sort.sort()
475 filenameList_to_sort.sort()
476
476
477 for file in filenameList_to_sort:
477 for file in filenameList_to_sort:
478 thisPath = pathList[pathDict[file]]
478 thisPath = pathList[pathDict[file]]
479
479
480 fileList = glob.glob1(thisPath, "*%s" %ext)
480 fileList = glob.glob1(thisPath, "*%s" %ext)
481 fileList.sort()
481 fileList.sort()
482
482
483 for file in fileList:
483 for file in fileList:
484
484
485 filename = os.path.join(thisPath,file)
485 filename = os.path.join(thisPath,file)
486 thisDatetime = isFileinThisTime(filename, startTime, endTime)
486 thisDatetime = isFileinThisTime(filename, startTime, endTime)
487
487
488 if not(thisDatetime):
488 if not(thisDatetime):
489 continue
489 continue
490
490
491 filenameList.append(filename)
491 filenameList.append(filename)
492 datetimeList.append(thisDatetime)
492 datetimeList.append(thisDatetime)
493
493
494 if not(filenameList):
494 if not(filenameList):
495 print "Any file was found for the time range %s - %s" %(startTime, endTime)
495 print "Any file was found for the time range %s - %s" %(startTime, endTime)
496 return None, None
496 return None, None
497
497
498 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
498 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
499 print
499 print
500
500
501 for i in range(len(filenameList)):
501 for i in range(len(filenameList)):
502 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
502 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
503
503
504 self.filenameList = filenameList
504 self.filenameList = filenameList
505 self.datetimeList = datetimeList
505 self.datetimeList = datetimeList
506
506
507 return pathList, filenameList
507 return pathList, filenameList
508
508
509 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
509 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
510
510
511 """
511 """
512 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
512 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
513 devuelve el archivo encontrado ademas de otros datos.
513 devuelve el archivo encontrado ademas de otros datos.
514
514
515 Input:
515 Input:
516 path : carpeta donde estan contenidos los files que contiene data
516 path : carpeta donde estan contenidos los files que contiene data
517
517
518 expLabel : Nombre del subexperimento (subfolder)
518 expLabel : Nombre del subexperimento (subfolder)
519
519
520 ext : extension de los files
520 ext : extension de los files
521
521
522 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
522 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
523
523
524 Return:
524 Return:
525 directory : eL directorio donde esta el file encontrado
525 directory : eL directorio donde esta el file encontrado
526 filename : el ultimo file de una determinada carpeta
526 filename : el ultimo file de una determinada carpeta
527 year : el anho
527 year : el anho
528 doy : el numero de dia del anho
528 doy : el numero de dia del anho
529 set : el set del archivo
529 set : el set del archivo
530
530
531
531
532 """
532 """
533 dirList = []
533 dirList = []
534
534
535 if not walk:
535 if not walk:
536 fullpath = path
536 fullpath = path
537 foldercounter = 0
537 foldercounter = 0
538 else:
538 else:
539 #Filtra solo los directorios
539 #Filtra solo los directorios
540 for thisPath in os.listdir(path):
540 for thisPath in os.listdir(path):
541 if not os.path.isdir(os.path.join(path,thisPath)):
541 if not os.path.isdir(os.path.join(path,thisPath)):
542 continue
542 continue
543 if not isDoyFolder(thisPath):
543 if not isDoyFolder(thisPath):
544 continue
544 continue
545
545
546 dirList.append(thisPath)
546 dirList.append(thisPath)
547
547
548 if not(dirList):
548 if not(dirList):
549 return None, None, None, None, None, None
549 return None, None, None, None, None, None
550
550
551 dirList = sorted( dirList, key=str.lower )
551 dirList = sorted( dirList, key=str.lower )
552
552
553 doypath = dirList[-1]
553 doypath = dirList[-1]
554 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
554 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
555 fullpath = os.path.join(path, doypath, expLabel)
555 fullpath = os.path.join(path, doypath, expLabel)
556
556
557
557
558 print "%s folder was found: " %(fullpath )
558 print "%s folder was found: " %(fullpath )
559
559
560 if set == None:
560 if set == None:
561 filename = getlastFileFromPath(fullpath, ext)
561 filename = getlastFileFromPath(fullpath, ext)
562 else:
562 else:
563 filename = getFileFromSet(fullpath, ext, set)
563 filename = getFileFromSet(fullpath, ext, set)
564
564
565 if not(filename):
565 if not(filename):
566 return None, None, None, None, None, None
566 return None, None, None, None, None, None
567
567
568 print "%s file was found" %(filename)
568 print "%s file was found" %(filename)
569
569
570 if not(self.__verifyFile(os.path.join(fullpath, filename))):
570 if not(self.__verifyFile(os.path.join(fullpath, filename))):
571 return None, None, None, None, None, None
571 return None, None, None, None, None, None
572
572
573 year = int( filename[1:5] )
573 year = int( filename[1:5] )
574 doy = int( filename[5:8] )
574 doy = int( filename[5:8] )
575 set = int( filename[8:11] )
575 set = int( filename[8:11] )
576
576
577 return fullpath, foldercounter, filename, year, doy, set
577 return fullpath, foldercounter, filename, year, doy, set
578
578
579 def __setNextFileOffline(self):
579 def __setNextFileOffline(self):
580
580
581 idFile = self.fileIndex
581 idFile = self.fileIndex
582
582
583 while (True):
583 while (True):
584 idFile += 1
584 idFile += 1
585 if not(idFile < len(self.filenameList)):
585 if not(idFile < len(self.filenameList)):
586 self.flagNoMoreFiles = 1
586 self.flagNoMoreFiles = 1
587 print "No more Files"
587 print "No more Files"
588 return 0
588 return 0
589
589
590 filename = self.filenameList[idFile]
590 filename = self.filenameList[idFile]
591
591
592 if not(self.__verifyFile(filename)):
592 if not(self.__verifyFile(filename)):
593 continue
593 continue
594
594
595 fileSize = os.path.getsize(filename)
595 fileSize = os.path.getsize(filename)
596 fp = open(filename,'rb')
596 fp = open(filename,'rb')
597 break
597 break
598
598
599 self.flagIsNewFile = 1
599 self.flagIsNewFile = 1
600 self.fileIndex = idFile
600 self.fileIndex = idFile
601 self.filename = filename
601 self.filename = filename
602 self.fileSize = fileSize
602 self.fileSize = fileSize
603 self.fp = fp
603 self.fp = fp
604
604
605 print "Setting the file: %s"%self.filename
605 print "Setting the file: %s"%self.filename
606
606
607 return 1
607 return 1
608
608
609 def __setNextFileOnline(self):
609 def __setNextFileOnline(self):
610 """
610 """
611 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
611 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
612 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
612 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
613 siguientes.
613 siguientes.
614
614
615 Affected:
615 Affected:
616 self.flagIsNewFile
616 self.flagIsNewFile
617 self.filename
617 self.filename
618 self.fileSize
618 self.fileSize
619 self.fp
619 self.fp
620 self.set
620 self.set
621 self.flagNoMoreFiles
621 self.flagNoMoreFiles
622
622
623 Return:
623 Return:
624 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
624 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
625 1 : si el file fue abierto con exito y esta listo a ser leido
625 1 : si el file fue abierto con exito y esta listo a ser leido
626
626
627 Excepciones:
627 Excepciones:
628 Si un determinado file no puede ser abierto
628 Si un determinado file no puede ser abierto
629 """
629 """
630 nFiles = 0
630 nFiles = 0
631 fileOk_flag = False
631 fileOk_flag = False
632 firstTime_flag = True
632 firstTime_flag = True
633
633
634 self.set += 1
634 self.set += 1
635
635
636 if self.set > 999:
636 if self.set > 999:
637 self.set = 0
637 self.set = 0
638 self.foldercounter += 1
638 self.foldercounter += 1
639
639
640 #busca el 1er file disponible
640 #busca el 1er file disponible
641 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
641 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
642 if fullfilename:
642 if fullfilename:
643 if self.__verifyFile(fullfilename, False):
643 if self.__verifyFile(fullfilename, False):
644 fileOk_flag = True
644 fileOk_flag = True
645
645
646 #si no encuentra un file entonces espera y vuelve a buscar
646 #si no encuentra un file entonces espera y vuelve a buscar
647 if not(fileOk_flag):
647 if not(fileOk_flag):
648 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
648 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
649
649
650 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
650 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
651 tries = self.nTries
651 tries = self.nTries
652 else:
652 else:
653 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
653 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
654
654
655 for nTries in range( tries ):
655 for nTries in range( tries ):
656 if firstTime_flag:
656 if firstTime_flag:
657 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
657 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
658 time.sleep( self.delay )
658 time.sleep( self.delay )
659 else:
659 else:
660 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
660 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
661
661
662 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
662 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
663 if fullfilename:
663 if fullfilename:
664 if self.__verifyFile(fullfilename):
664 if self.__verifyFile(fullfilename):
665 fileOk_flag = True
665 fileOk_flag = True
666 break
666 break
667
667
668 if fileOk_flag:
668 if fileOk_flag:
669 break
669 break
670
670
671 firstTime_flag = False
671 firstTime_flag = False
672
672
673 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
673 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
674 self.set += 1
674 self.set += 1
675
675
676 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
676 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
677 self.set = 0
677 self.set = 0
678 self.doy += 1
678 self.doy += 1
679 self.foldercounter = 0
679 self.foldercounter = 0
680
680
681 if fileOk_flag:
681 if fileOk_flag:
682 self.fileSize = os.path.getsize( fullfilename )
682 self.fileSize = os.path.getsize( fullfilename )
683 self.filename = fullfilename
683 self.filename = fullfilename
684 self.flagIsNewFile = 1
684 self.flagIsNewFile = 1
685 if self.fp != None: self.fp.close()
685 if self.fp != None: self.fp.close()
686 self.fp = open(fullfilename, 'rb')
686 self.fp = open(fullfilename, 'rb')
687 self.flagNoMoreFiles = 0
687 self.flagNoMoreFiles = 0
688 print 'Setting the file: %s' % fullfilename
688 print 'Setting the file: %s' % fullfilename
689 else:
689 else:
690 self.fileSize = 0
690 self.fileSize = 0
691 self.filename = None
691 self.filename = None
692 self.flagIsNewFile = 0
692 self.flagIsNewFile = 0
693 self.fp = None
693 self.fp = None
694 self.flagNoMoreFiles = 1
694 self.flagNoMoreFiles = 1
695 print 'No more Files'
695 print 'No more Files'
696
696
697 return fileOk_flag
697 return fileOk_flag
698
698
699 def setNextFile(self):
699 def setNextFile(self):
700 if self.fp != None:
700 if self.fp != None:
701 self.fp.close()
701 self.fp.close()
702
702
703 if self.online:
703 if self.online:
704 newFile = self.__setNextFileOnline()
704 newFile = self.__setNextFileOnline()
705 else:
705 else:
706 newFile = self.__setNextFileOffline()
706 newFile = self.__setNextFileOffline()
707
707
708 if not(newFile):
708 if not(newFile):
709 return 0
709 return 0
710
710
711 self.__readFirstHeader()
711 self.__readFirstHeader()
712 self.nReadBlocks = 0
712 self.nReadBlocks = 0
713 return 1
713 return 1
714
714
715 def __waitNewBlock(self):
715 def __waitNewBlock(self):
716 """
716 """
717 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
717 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
718
718
719 Si el modo de lectura es OffLine siempre retorn 0
719 Si el modo de lectura es OffLine siempre retorn 0
720 """
720 """
721 if not self.online:
721 if not self.online:
722 return 0
722 return 0
723
723
724 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
724 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
725 return 0
725 return 0
726
726
727 currentPointer = self.fp.tell()
727 currentPointer = self.fp.tell()
728
728
729 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
729 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
730
730
731 for nTries in range( self.nTries ):
731 for nTries in range( self.nTries ):
732
732
733 self.fp.close()
733 self.fp.close()
734 self.fp = open( self.filename, 'rb' )
734 self.fp = open( self.filename, 'rb' )
735 self.fp.seek( currentPointer )
735 self.fp.seek( currentPointer )
736
736
737 self.fileSize = os.path.getsize( self.filename )
737 self.fileSize = os.path.getsize( self.filename )
738 currentSize = self.fileSize - currentPointer
738 currentSize = self.fileSize - currentPointer
739
739
740 if ( currentSize >= neededSize ):
740 if ( currentSize >= neededSize ):
741 self.basicHeaderObj.read(self.fp)
741 self.basicHeaderObj.read(self.fp)
742 return 1
742 return 1
743
743
744 if self.fileSize == self.fileSizeByHeader:
744 if self.fileSize == self.fileSizeByHeader:
745 # self.flagEoF = True
745 # self.flagEoF = True
746 return 0
746 return 0
747
747
748 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
748 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
749 time.sleep( self.delay )
749 time.sleep( self.delay )
750
750
751
751
752 return 0
752 return 0
753
753
754 def waitDataBlock(self,pointer_location):
754 def waitDataBlock(self,pointer_location):
755
755
756 currentPointer = pointer_location
756 currentPointer = pointer_location
757
757
758 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
758 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
759
759
760 for nTries in range( self.nTries ):
760 for nTries in range( self.nTries ):
761 self.fp.close()
761 self.fp.close()
762 self.fp = open( self.filename, 'rb' )
762 self.fp = open( self.filename, 'rb' )
763 self.fp.seek( currentPointer )
763 self.fp.seek( currentPointer )
764
764
765 self.fileSize = os.path.getsize( self.filename )
765 self.fileSize = os.path.getsize( self.filename )
766 currentSize = self.fileSize - currentPointer
766 currentSize = self.fileSize - currentPointer
767
767
768 if ( currentSize >= neededSize ):
768 if ( currentSize >= neededSize ):
769 return 1
769 return 1
770
770
771 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
771 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
772 time.sleep( self.delay )
772 time.sleep( self.delay )
773
773
774 return 0
774 return 0
775
775
776 def __jumpToLastBlock(self):
776 def __jumpToLastBlock(self):
777
777
778 if not(self.__isFirstTimeOnline):
778 if not(self.__isFirstTimeOnline):
779 return
779 return
780
780
781 csize = self.fileSize - self.fp.tell()
781 csize = self.fileSize - self.fp.tell()
782 blocksize = self.processingHeaderObj.blockSize
782 blocksize = self.processingHeaderObj.blockSize
783
783
784 #salta el primer bloque de datos
784 #salta el primer bloque de datos
785 if csize > self.processingHeaderObj.blockSize:
785 if csize > self.processingHeaderObj.blockSize:
786 self.fp.seek(self.fp.tell() + blocksize)
786 self.fp.seek(self.fp.tell() + blocksize)
787 else:
787 else:
788 return
788 return
789
789
790 csize = self.fileSize - self.fp.tell()
790 csize = self.fileSize - self.fp.tell()
791 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
791 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
792 while True:
792 while True:
793
793
794 if self.fp.tell()<self.fileSize:
794 if self.fp.tell()<self.fileSize:
795 self.fp.seek(self.fp.tell() + neededsize)
795 self.fp.seek(self.fp.tell() + neededsize)
796 else:
796 else:
797 self.fp.seek(self.fp.tell() - neededsize)
797 self.fp.seek(self.fp.tell() - neededsize)
798 break
798 break
799
799
800 # csize = self.fileSize - self.fp.tell()
800 # csize = self.fileSize - self.fp.tell()
801 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
801 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
802 # factor = int(csize/neededsize)
802 # factor = int(csize/neededsize)
803 # if factor > 0:
803 # if factor > 0:
804 # self.fp.seek(self.fp.tell() + factor*neededsize)
804 # self.fp.seek(self.fp.tell() + factor*neededsize)
805
805
806 self.flagIsNewFile = 0
806 self.flagIsNewFile = 0
807 self.__isFirstTimeOnline = 0
807 self.__isFirstTimeOnline = 0
808
808
809 def __setNewBlock(self):
809 def __setNewBlock(self):
810
810
811 if self.fp == None:
811 if self.fp == None:
812 return 0
812 return 0
813
813
814 if self.online:
814 if self.online:
815 self.__jumpToLastBlock()
815 self.__jumpToLastBlock()
816
816
817 if self.flagIsNewFile:
817 if self.flagIsNewFile:
818 return 1
818 return 1
819
819
820 self.lastUTTime = self.basicHeaderObj.utc
820 self.lastUTTime = self.basicHeaderObj.utc
821 currentSize = self.fileSize - self.fp.tell()
821 currentSize = self.fileSize - self.fp.tell()
822 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
822 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
823
823
824 if (currentSize >= neededSize):
824 if (currentSize >= neededSize):
825 self.basicHeaderObj.read(self.fp)
825 self.basicHeaderObj.read(self.fp)
826 return 1
826 return 1
827
827
828 if self.__waitNewBlock():
828 if self.__waitNewBlock():
829 return 1
829 return 1
830
830
831 if not(self.setNextFile()):
831 if not(self.setNextFile()):
832 return 0
832 return 0
833
833
834 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
834 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
835
835
836 self.flagTimeBlock = 0
836 self.flagTimeBlock = 0
837
837
838 if deltaTime > self.maxTimeStep:
838 if deltaTime > self.maxTimeStep:
839 self.flagTimeBlock = 1
839 self.flagTimeBlock = 1
840
840
841 return 1
841 return 1
842
842
843 def readNextBlock(self):
843 def readNextBlock(self):
844 if not(self.__setNewBlock()):
844 if not(self.__setNewBlock()):
845 return 0
845 return 0
846
846
847 if not(self.readBlock()):
847 if not(self.readBlock()):
848 return 0
848 return 0
849
849
850 return 1
850 return 1
851
851
852 def __readFirstHeader(self):
852 def __readFirstHeader(self):
853
853
854 self.basicHeaderObj.read(self.fp)
854 self.basicHeaderObj.read(self.fp)
855 self.systemHeaderObj.read(self.fp)
855 self.systemHeaderObj.read(self.fp)
856 self.radarControllerHeaderObj.read(self.fp)
856 self.radarControllerHeaderObj.read(self.fp)
857 self.processingHeaderObj.read(self.fp)
857 self.processingHeaderObj.read(self.fp)
858
858
859 self.firstHeaderSize = self.basicHeaderObj.size
859 self.firstHeaderSize = self.basicHeaderObj.size
860
860
861 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
861 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
862 if datatype == 0:
862 if datatype == 0:
863 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
863 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
864 elif datatype == 1:
864 elif datatype == 1:
865 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
865 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
866 elif datatype == 2:
866 elif datatype == 2:
867 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
867 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
868 elif datatype == 3:
868 elif datatype == 3:
869 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
869 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
870 elif datatype == 4:
870 elif datatype == 4:
871 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
871 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
872 elif datatype == 5:
872 elif datatype == 5:
873 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
873 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
874 else:
874 else:
875 raise ValueError, 'Data type was not defined'
875 raise ValueError, 'Data type was not defined'
876
876
877 self.dtype = datatype_str
877 self.dtype = datatype_str
878 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
878 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
879 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
879 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
880 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
880 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
881 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
881 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
882 self.getBlockDimension()
882 self.getBlockDimension()
883
883
884 def __verifyFile(self, filename, msgFlag=True):
884 def __verifyFile(self, filename, msgFlag=True):
885 msg = None
885 msg = None
886 try:
886 try:
887 fp = open(filename, 'rb')
887 fp = open(filename, 'rb')
888 currentPosition = fp.tell()
888 currentPosition = fp.tell()
889 except IOError:
889 except IOError:
890 traceback.print_exc()
890 traceback.print_exc()
891 if msgFlag:
891 if msgFlag:
892 print "The file %s can't be opened" % (filename)
892 print "The file %s can't be opened" % (filename)
893 return False
893 return False
894
894
895 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
895 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
896
896
897 if neededSize == 0:
897 if neededSize == 0:
898 basicHeaderObj = BasicHeader(LOCALTIME)
898 basicHeaderObj = BasicHeader(LOCALTIME)
899 systemHeaderObj = SystemHeader()
899 systemHeaderObj = SystemHeader()
900 radarControllerHeaderObj = RadarControllerHeader()
900 radarControllerHeaderObj = RadarControllerHeader()
901 processingHeaderObj = ProcessingHeader()
901 processingHeaderObj = ProcessingHeader()
902
902
903 try:
903 try:
904 if not( basicHeaderObj.read(fp) ): raise IOError
904 if not( basicHeaderObj.read(fp) ): raise IOError
905 if not( systemHeaderObj.read(fp) ): raise IOError
905 if not( systemHeaderObj.read(fp) ): raise IOError
906 if not( radarControllerHeaderObj.read(fp) ): raise IOError
906 if not( radarControllerHeaderObj.read(fp) ): raise IOError
907 if not( processingHeaderObj.read(fp) ): raise IOError
907 if not( processingHeaderObj.read(fp) ): raise IOError
908 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
908 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
909
909
910 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
910 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
911
911
912 except IOError:
912 except IOError:
913 traceback.print_exc()
913 traceback.print_exc()
914 if msgFlag:
914 if msgFlag:
915 print "\tThe file %s is empty or it hasn't enough data" % filename
915 print "\tThe file %s is empty or it hasn't enough data" % filename
916
916
917 fp.close()
917 fp.close()
918 return False
918 return False
919 else:
919 else:
920 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
920 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
921
921
922 fp.close()
922 fp.close()
923 fileSize = os.path.getsize(filename)
923 fileSize = os.path.getsize(filename)
924 currentSize = fileSize - currentPosition
924 currentSize = fileSize - currentPosition
925 if currentSize < neededSize:
925 if currentSize < neededSize:
926 if msgFlag and (msg != None):
926 if msgFlag and (msg != None):
927 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
927 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
928 return False
928 return False
929
929
930 return True
930 return True
931
931
932 def setup(self,
932 def setup(self,
933 path=None,
933 path=None,
934 startDate=None,
934 startDate=None,
935 endDate=None,
935 endDate=None,
936 startTime=datetime.time(0,0,0),
936 startTime=datetime.time(0,0,0),
937 endTime=datetime.time(23,59,59),
937 endTime=datetime.time(23,59,59),
938 set=None,
938 set=None,
939 expLabel = "",
939 expLabel = "",
940 ext = None,
940 ext = None,
941 online = False,
941 online = False,
942 delay = 60,
942 delay = 60,
943 walk = True,
943 walk = True,
944 getblock = False,
944 getblock = False,
945 nTxs = 1):
945 nTxs = 1):
946
946
947 if path == None:
947 if path == None:
948 raise ValueError, "The path is not valid"
948 raise ValueError, "The path is not valid"
949
949
950 if ext == None:
950 if ext == None:
951 ext = self.ext
951 ext = self.ext
952
952
953 if online:
953 if online:
954 print "Searching files in online mode..."
954 print "Searching files in online mode..."
955
955
956 for nTries in range( self.nTries ):
956 for nTries in range( self.nTries ):
957 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
957 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
958
958
959 if fullpath:
959 if fullpath:
960 break
960 break
961
961
962 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
962 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
963 time.sleep( self.delay )
963 time.sleep( self.delay )
964
964
965 if not(fullpath):
965 if not(fullpath):
966 print "There 'isn't valied files in %s" % path
966 print "There 'isn't valied files in %s" % path
967 return None
967 return None
968
968
969 self.year = year
969 self.year = year
970 self.doy = doy
970 self.doy = doy
971 self.set = set - 1
971 self.set = set - 1
972 self.path = path
972 self.path = path
973 self.foldercounter = foldercounter
973 self.foldercounter = foldercounter
974 last_set = None
974 last_set = None
975
975
976 else:
976 else:
977 print "Searching files in offline mode ..."
977 print "Searching files in offline mode ..."
978 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
978 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
979 startTime=startTime, endTime=endTime,
979 startTime=startTime, endTime=endTime,
980 set=set, expLabel=expLabel, ext=ext,
980 set=set, expLabel=expLabel, ext=ext,
981 walk=walk)
981 walk=walk)
982
982
983 if not(pathList):
983 if not(pathList):
984 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
984 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
985 datetime.datetime.combine(startDate,startTime).ctime(),
985 datetime.datetime.combine(startDate,startTime).ctime(),
986 datetime.datetime.combine(endDate,endTime).ctime())
986 datetime.datetime.combine(endDate,endTime).ctime())
987
987
988 sys.exit(-1)
988 sys.exit(-1)
989
989
990
990
991 self.fileIndex = -1
991 self.fileIndex = -1
992 self.pathList = pathList
992 self.pathList = pathList
993 self.filenameList = filenameList
993 self.filenameList = filenameList
994 file_name = os.path.basename(filenameList[-1])
994 file_name = os.path.basename(filenameList[-1])
995 basename, ext = os.path.splitext(file_name)
995 basename, ext = os.path.splitext(file_name)
996 last_set = int(basename[-3:])
996 last_set = int(basename[-3:])
997
997
998 self.online = online
998 self.online = online
999 self.delay = delay
999 self.delay = delay
1000 ext = ext.lower()
1000 ext = ext.lower()
1001 self.ext = ext
1001 self.ext = ext
1002 self.getByBlock = getblock
1002 self.getByBlock = getblock
1003 self.nTxs = int(nTxs)
1003 self.nTxs = int(nTxs)
1004
1004
1005 if not(self.setNextFile()):
1005 if not(self.setNextFile()):
1006 if (startDate!=None) and (endDate!=None):
1006 if (startDate!=None) and (endDate!=None):
1007 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1007 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1008 elif startDate != None:
1008 elif startDate != None:
1009 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1009 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1010 else:
1010 else:
1011 print "No files"
1011 print "No files"
1012
1012
1013 sys.exit(-1)
1013 sys.exit(-1)
1014
1014
1015 # self.updateDataHeader()
1015 # self.updateDataHeader()
1016 if last_set != None:
1016 if last_set != None:
1017 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1017 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1018 return
1018 return
1019
1019
1020 def getBasicHeader(self):
1020 def getBasicHeader(self):
1021
1021
1022 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1022 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1023
1023
1024 self.dataOut.flagTimeBlock = self.flagTimeBlock
1024 self.dataOut.flagTimeBlock = self.flagTimeBlock
1025
1025
1026 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1026 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1027
1027
1028 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1028 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1029
1029
1030 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1030 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1031
1031
1032 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1032 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1033
1033
1034 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1035
1036 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1037
1038
1034 def getFirstHeader(self):
1039 def getFirstHeader(self):
1035
1040
1036 raise ValueError, "This method has not been implemented"
1041 raise ValueError, "This method has not been implemented"
1037
1042
1038 def getData(self):
1043 def getData(self):
1039
1044
1040 raise ValueError, "This method has not been implemented"
1045 raise ValueError, "This method has not been implemented"
1041
1046
1042 def hasNotDataInBuffer(self):
1047 def hasNotDataInBuffer(self):
1043
1048
1044 raise ValueError, "This method has not been implemented"
1049 raise ValueError, "This method has not been implemented"
1045
1050
1046 def readBlock(self):
1051 def readBlock(self):
1047
1052
1048 raise ValueError, "This method has not been implemented"
1053 raise ValueError, "This method has not been implemented"
1049
1054
1050 def isEndProcess(self):
1055 def isEndProcess(self):
1051
1056
1052 return self.flagNoMoreFiles
1057 return self.flagNoMoreFiles
1053
1058
1054 def printReadBlocks(self):
1059 def printReadBlocks(self):
1055
1060
1056 print "Number of read blocks per file %04d" %self.nReadBlocks
1061 print "Number of read blocks per file %04d" %self.nReadBlocks
1057
1062
1058 def printTotalBlocks(self):
1063 def printTotalBlocks(self):
1059
1064
1060 print "Number of read blocks %04d" %self.nTotalBlocks
1065 print "Number of read blocks %04d" %self.nTotalBlocks
1061
1066
1062 def printNumberOfBlock(self):
1067 def printNumberOfBlock(self):
1063
1068
1064 if self.flagIsNewBlock:
1069 if self.flagIsNewBlock:
1065 print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1070 print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1066 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1071 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1067
1072
1068 def printInfo(self):
1073 def printInfo(self):
1069
1074
1070 if self.__printInfo == False:
1075 if self.__printInfo == False:
1071 return
1076 return
1072
1077
1073 self.basicHeaderObj.printInfo()
1078 self.basicHeaderObj.printInfo()
1074 self.systemHeaderObj.printInfo()
1079 self.systemHeaderObj.printInfo()
1075 self.radarControllerHeaderObj.printInfo()
1080 self.radarControllerHeaderObj.printInfo()
1076 self.processingHeaderObj.printInfo()
1081 self.processingHeaderObj.printInfo()
1077
1082
1078 self.__printInfo = False
1083 self.__printInfo = False
1079
1084
1080
1085
1081 def run(self, **kwargs):
1086 def run(self, **kwargs):
1082
1087
1083 if not(self.isConfig):
1088 if not(self.isConfig):
1084
1089
1085 # self.dataOut = dataOut
1090 # self.dataOut = dataOut
1086 self.setup(**kwargs)
1091 self.setup(**kwargs)
1087 self.isConfig = True
1092 self.isConfig = True
1088
1093
1089 self.getData()
1094 self.getData()
1090
1095
1091 class JRODataWriter(JRODataIO):
1096 class JRODataWriter(JRODataIO):
1092
1097
1093 """
1098 """
1094 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1099 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1095 de los datos siempre se realiza por bloques.
1100 de los datos siempre se realiza por bloques.
1096 """
1101 """
1097
1102
1098 blockIndex = 0
1103 blockIndex = 0
1099
1104
1100 path = None
1105 path = None
1101
1106
1102 setFile = None
1107 setFile = None
1103
1108
1104 profilesPerBlock = None
1109 profilesPerBlock = None
1105
1110
1106 blocksPerFile = None
1111 blocksPerFile = None
1107
1112
1108 nWriteBlocks = 0
1113 nWriteBlocks = 0
1109
1114
1110 def __init__(self, dataOut=None):
1115 def __init__(self, dataOut=None):
1111 raise ValueError, "Not implemented"
1116 raise ValueError, "Not implemented"
1112
1117
1113
1118
1114 def hasAllDataInBuffer(self):
1119 def hasAllDataInBuffer(self):
1115 raise ValueError, "Not implemented"
1120 raise ValueError, "Not implemented"
1116
1121
1117
1122
1118 def setBlockDimension(self):
1123 def setBlockDimension(self):
1119 raise ValueError, "Not implemented"
1124 raise ValueError, "Not implemented"
1120
1125
1121
1126
1122 def writeBlock(self):
1127 def writeBlock(self):
1123 raise ValueError, "No implemented"
1128 raise ValueError, "No implemented"
1124
1129
1125
1130
1126 def putData(self):
1131 def putData(self):
1127 raise ValueError, "No implemented"
1132 raise ValueError, "No implemented"
1128
1133
1129
1134
1130 def setBasicHeader(self):
1135 def setBasicHeader(self):
1131
1136
1132 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1137 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1133 self.basicHeaderObj.version = self.versionFile
1138 self.basicHeaderObj.version = self.versionFile
1134 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1139 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1135
1140
1136 utc = numpy.floor(self.dataOut.utctime)
1141 utc = numpy.floor(self.dataOut.utctime)
1137 milisecond = (self.dataOut.utctime - utc)* 1000.0
1142 milisecond = (self.dataOut.utctime - utc)* 1000.0
1138
1143
1139 self.basicHeaderObj.utc = utc
1144 self.basicHeaderObj.utc = utc
1140 self.basicHeaderObj.miliSecond = milisecond
1145 self.basicHeaderObj.miliSecond = milisecond
1141 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1146 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1142 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1147 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1143 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1148 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1144
1149
1145 def setFirstHeader(self):
1150 def setFirstHeader(self):
1146 """
1151 """
1147 Obtiene una copia del First Header
1152 Obtiene una copia del First Header
1148
1153
1149 Affected:
1154 Affected:
1150
1155
1151 self.basicHeaderObj
1156 self.basicHeaderObj
1152 self.systemHeaderObj
1157 self.systemHeaderObj
1153 self.radarControllerHeaderObj
1158 self.radarControllerHeaderObj
1154 self.processingHeaderObj self.
1159 self.processingHeaderObj self.
1155
1160
1156 Return:
1161 Return:
1157 None
1162 None
1158 """
1163 """
1159
1164
1160 raise ValueError, "No implemented"
1165 raise ValueError, "No implemented"
1161
1166
1162 def __writeFirstHeader(self):
1167 def __writeFirstHeader(self):
1163 """
1168 """
1164 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1169 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1165
1170
1166 Affected:
1171 Affected:
1167 __dataType
1172 __dataType
1168
1173
1169 Return:
1174 Return:
1170 None
1175 None
1171 """
1176 """
1172
1177
1173 # CALCULAR PARAMETROS
1178 # CALCULAR PARAMETROS
1174
1179
1175 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1180 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1176 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1181 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1177
1182
1178 self.basicHeaderObj.write(self.fp)
1183 self.basicHeaderObj.write(self.fp)
1179 self.systemHeaderObj.write(self.fp)
1184 self.systemHeaderObj.write(self.fp)
1180 self.radarControllerHeaderObj.write(self.fp)
1185 self.radarControllerHeaderObj.write(self.fp)
1181 self.processingHeaderObj.write(self.fp)
1186 self.processingHeaderObj.write(self.fp)
1182
1187
1183 self.dtype = self.dataOut.dtype
1188 self.dtype = self.dataOut.dtype
1184
1189
1185 def __setNewBlock(self):
1190 def __setNewBlock(self):
1186 """
1191 """
1187 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1192 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1188
1193
1189 Return:
1194 Return:
1190 0 : si no pudo escribir nada
1195 0 : si no pudo escribir nada
1191 1 : Si escribio el Basic el First Header
1196 1 : Si escribio el Basic el First Header
1192 """
1197 """
1193 if self.fp == None:
1198 if self.fp == None:
1194 self.setNextFile()
1199 self.setNextFile()
1195
1200
1196 if self.flagIsNewFile:
1201 if self.flagIsNewFile:
1197 return 1
1202 return 1
1198
1203
1199 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1204 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1200 self.basicHeaderObj.write(self.fp)
1205 self.basicHeaderObj.write(self.fp)
1201 return 1
1206 return 1
1202
1207
1203 if not( self.setNextFile() ):
1208 if not( self.setNextFile() ):
1204 return 0
1209 return 0
1205
1210
1206 return 1
1211 return 1
1207
1212
1208
1213
1209 def writeNextBlock(self):
1214 def writeNextBlock(self):
1210 """
1215 """
1211 Selecciona el bloque siguiente de datos y los escribe en un file
1216 Selecciona el bloque siguiente de datos y los escribe en un file
1212
1217
1213 Return:
1218 Return:
1214 0 : Si no hizo pudo escribir el bloque de datos
1219 0 : Si no hizo pudo escribir el bloque de datos
1215 1 : Si no pudo escribir el bloque de datos
1220 1 : Si no pudo escribir el bloque de datos
1216 """
1221 """
1217 if not( self.__setNewBlock() ):
1222 if not( self.__setNewBlock() ):
1218 return 0
1223 return 0
1219
1224
1220 self.writeBlock()
1225 self.writeBlock()
1221
1226
1222 return 1
1227 return 1
1223
1228
1224 def setNextFile(self):
1229 def setNextFile(self):
1225 """
1230 """
1226 Determina el siguiente file que sera escrito
1231 Determina el siguiente file que sera escrito
1227
1232
1228 Affected:
1233 Affected:
1229 self.filename
1234 self.filename
1230 self.subfolder
1235 self.subfolder
1231 self.fp
1236 self.fp
1232 self.setFile
1237 self.setFile
1233 self.flagIsNewFile
1238 self.flagIsNewFile
1234
1239
1235 Return:
1240 Return:
1236 0 : Si el archivo no puede ser escrito
1241 0 : Si el archivo no puede ser escrito
1237 1 : Si el archivo esta listo para ser escrito
1242 1 : Si el archivo esta listo para ser escrito
1238 """
1243 """
1239 ext = self.ext
1244 ext = self.ext
1240 path = self.path
1245 path = self.path
1241
1246
1242 if self.fp != None:
1247 if self.fp != None:
1243 self.fp.close()
1248 self.fp.close()
1244
1249
1245 timeTuple = time.localtime( self.dataOut.utctime)
1250 timeTuple = time.localtime( self.dataOut.utctime)
1246 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1251 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1247
1252
1248 fullpath = os.path.join( path, subfolder )
1253 fullpath = os.path.join( path, subfolder )
1249 if not( os.path.exists(fullpath) ):
1254 if not( os.path.exists(fullpath) ):
1250 os.mkdir(fullpath)
1255 os.mkdir(fullpath)
1251 self.setFile = -1 #inicializo mi contador de seteo
1256 self.setFile = -1 #inicializo mi contador de seteo
1252 else:
1257 else:
1253 filesList = os.listdir( fullpath )
1258 filesList = os.listdir( fullpath )
1254 if len( filesList ) > 0:
1259 if len( filesList ) > 0:
1255 filesList = sorted( filesList, key=str.lower )
1260 filesList = sorted( filesList, key=str.lower )
1256 filen = filesList[-1]
1261 filen = filesList[-1]
1257 # el filename debera tener el siguiente formato
1262 # el filename debera tener el siguiente formato
1258 # 0 1234 567 89A BCDE (hex)
1263 # 0 1234 567 89A BCDE (hex)
1259 # x YYYY DDD SSS .ext
1264 # x YYYY DDD SSS .ext
1260 if isNumber( filen[8:11] ):
1265 if isNumber( filen[8:11] ):
1261 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1266 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1262 else:
1267 else:
1263 self.setFile = -1
1268 self.setFile = -1
1264 else:
1269 else:
1265 self.setFile = -1 #inicializo mi contador de seteo
1270 self.setFile = -1 #inicializo mi contador de seteo
1266
1271
1267 setFile = self.setFile
1272 setFile = self.setFile
1268 setFile += 1
1273 setFile += 1
1269
1274
1270 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1275 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1271 timeTuple.tm_year,
1276 timeTuple.tm_year,
1272 timeTuple.tm_yday,
1277 timeTuple.tm_yday,
1273 setFile,
1278 setFile,
1274 ext )
1279 ext )
1275
1280
1276 filename = os.path.join( path, subfolder, file )
1281 filename = os.path.join( path, subfolder, file )
1277
1282
1278 fp = open( filename,'wb' )
1283 fp = open( filename,'wb' )
1279
1284
1280 self.blockIndex = 0
1285 self.blockIndex = 0
1281
1286
1282 #guardando atributos
1287 #guardando atributos
1283 self.filename = filename
1288 self.filename = filename
1284 self.subfolder = subfolder
1289 self.subfolder = subfolder
1285 self.fp = fp
1290 self.fp = fp
1286 self.setFile = setFile
1291 self.setFile = setFile
1287 self.flagIsNewFile = 1
1292 self.flagIsNewFile = 1
1288
1293
1289 self.setFirstHeader()
1294 self.setFirstHeader()
1290
1295
1291 print 'Writing the file: %s'%self.filename
1296 print 'Writing the file: %s'%self.filename
1292
1297
1293 self.__writeFirstHeader()
1298 self.__writeFirstHeader()
1294
1299
1295 return 1
1300 return 1
1296
1301
1297 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1302 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1298 """
1303 """
1299 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1304 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1300
1305
1301 Inputs:
1306 Inputs:
1302 path : el path destino en el cual se escribiran los files a crear
1307 path : el path destino en el cual se escribiran los files a crear
1303 format : formato en el cual sera salvado un file
1308 format : formato en el cual sera salvado un file
1304 set : el setebo del file
1309 set : el setebo del file
1305
1310
1306 Return:
1311 Return:
1307 0 : Si no realizo un buen seteo
1312 0 : Si no realizo un buen seteo
1308 1 : Si realizo un buen seteo
1313 1 : Si realizo un buen seteo
1309 """
1314 """
1310
1315
1311 if ext == None:
1316 if ext == None:
1312 ext = self.ext
1317 ext = self.ext
1313
1318
1314 ext = ext.lower()
1319 ext = ext.lower()
1315
1320
1316 self.ext = ext
1321 self.ext = ext
1317
1322
1318 self.path = path
1323 self.path = path
1319
1324
1320 self.setFile = set - 1
1325 self.setFile = set - 1
1321
1326
1322 self.blocksPerFile = blocksPerFile
1327 self.blocksPerFile = blocksPerFile
1323
1328
1324 self.profilesPerBlock = profilesPerBlock
1329 self.profilesPerBlock = profilesPerBlock
1325
1330
1326 self.dataOut = dataOut
1331 self.dataOut = dataOut
1327
1332
1328 if not(self.setNextFile()):
1333 if not(self.setNextFile()):
1329 print "There isn't a next file"
1334 print "There isn't a next file"
1330 return 0
1335 return 0
1331
1336
1332 self.setBlockDimension()
1337 self.setBlockDimension()
1333
1338
1334 return 1
1339 return 1
1335
1340
1336 def run(self, dataOut, **kwargs):
1341 def run(self, dataOut, **kwargs):
1337
1342
1338 if not(self.isConfig):
1343 if not(self.isConfig):
1339
1344
1340 self.setup(dataOut, **kwargs)
1345 self.setup(dataOut, **kwargs)
1341 self.isConfig = True
1346 self.isConfig = True
1342
1347
1343 self.putData()
1348 self.putData()
1344
1349
General Comments 0
You need to be logged in to leave comments. Login now