##// END OF EJS Templates
schain-jc
RSS
add multiSchain (@jchavez)
jespinoza -
r892:30b3788062ca
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,82
1 import argparse
2
3 from schainpy.controller import Project, multiSchain
4
5 desc = "HF_EXAMPLE"
6
7 def fiber(cursor, skip, q, dt):
8
9 controllerObj = Project()
10
11 controllerObj.setup(id='191', name='test01', description=desc)
12
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
14 path='/data/workspace/data/julia/',
15 startDate=dt,
16 endDate=dt,
17 startTime="00:00:00",
18 endTime="23:59:59",
19 online=0,
20 #set=1426485881,
21 delay=10,
22 walk=1,
23 queue=q,
24 cursor=cursor,
25 skip=skip,
26 #timezone=-5*3600
27 )
28
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
30 #
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
32 # opObj11 = procUnitConfObj2.addParameter(name='pairsList', value='(0,1)', format='pairslist')
33 #
34 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
35
36 # opObj11 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
37
38 #
39 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
40 # opObj11.addParameter(name='id', value='1000', format='int')
41 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
42 # opObj11.addParameter(name='channelList', value='0', format='intlist')
43 # opObj11.addParameter(name='zmin', value='-120', format='float')
44 # opObj11.addParameter(name='zmax', value='-70', format='float')
45 # opObj11.addParameter(name='save', value='1', format='int')
46 # opObj11.addParameter(name='figpath', value=figpath, format='str')
47
48 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
49 # opObj11.addParameter(name='id', value='2000', format='int')
50 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca', format='str')
51 # opObj11.addParameter(name='showprofile', value='0', format='int')
52 # opObj11.addParameter(name='channelList', value='0', format='intlist')
53 # # opObj11.addParameter(name='xmin', value='0', format='float')
54 # opObj11.addParameter(name='xmin', value='0', format='float')
55 # opObj11.addParameter(name='xmax', value='24', format='float')
56 #
57 # opObj11.addParameter(name='zmin', value='-110', format='float')
58 # opObj11.addParameter(name='zmax', value='-70', format='float')
59 # opObj11.addParameter(name='save', value='0', format='int')
60 # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
61
62 opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
63 opObj12.addParameter(name='zeromq', value=1, format='int')
64
65 # print "Escribiendo el archivo XML"
66 # controllerObj.writeXml(filename)
67 # print "Leyendo el archivo XML"
68 # controllerObj.readXml(filename)
69
70 controllerObj.createObjects()
71 controllerObj.connectObjects()
72
73 # timeit.timeit('controllerObj.run()', number=2)
74
75 controllerObj.run()
76
77
78 if __name__ == '__main__':
79 parser = argparse.ArgumentParser(description='Set number of parallel processes')
80 parser.add_argument('--nProcess', default=2, type=int)
81 args = parser.parse_args()
82 multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/20')
Show file before | Show file after | Hide comments
@@ -7,6 +7,8 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 from multiprocessing import Process, Queue, cpu_count
11
10 12 import schainpy
11 13 import schainpy.admin
12 14
@@ -23,6 +25,51 def prettify(elem):
23 25 reparsed = minidom.parseString(rough_string)
24 26 return reparsed.toprettyxml(indent=" ")
25 27
28 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None):
29 skip = 0
30 cursor = 0
31 nFiles = None
32 processes = []
33
34 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
35 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
36 days = (dt2 - dt1).days
37 print days
38 for day in range(days+1):
39 skip = 0
40 cursor = 0
41 q = Queue()
42 processes = []
43 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
44 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
45 firstProcess.start()
46 nFiles = q.get()
47 firstProcess.terminate()
48 skip = int(math.ceil(nFiles/nProcess))
49 try:
50 while True:
51 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
52 processes[cursor].start()
53 if nFiles < cursor*skip:
54 break
55 cursor += 1
56 except KeyboardInterrupt:
57 for process in processes:
58 process.terminate()
59 process.join()
60 for process in processes:
61 process.join()
62 #process.terminate()
63 sleep(3)
64
65 try:
66 while True:
67 pass
68 except KeyboardInterrupt:
69 for process in processes:
70 process.terminate()
71 process.join()
72
26 73 class ParameterConf():
27 74
28 75 id = None
@@ -51,6 +98,9 class ParameterConf():
51 98
52 99 return self.__formated_value
53 100
101 if format == 'obj':
102 return value
103
54 104 if format == 'str':
55 105 self.__formated_value = str(value)
56 106 return self.__formated_value
@@ -171,7 +221,10 class ParameterConf():
171 221
172 222 self.id = str(id)
173 223 self.name = name
174 self.value = str(value)
224 if format == 'obj':
225 self.value = value
226 else:
227 self.value = str(value)
175 228 self.format = str.lower(format)
176 229
177 230 self.getValue()
@@ -698,7 +751,7 class ReadUnitConf(ProcUnitConf):
698 751
699 752 return self.ELEMENTNAME
700 753
701 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
754 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
702 755
703 756 #Compatible with old signal chain version
704 757 if datatype==None and name==None:
@@ -725,7 +778,7 class ReadUnitConf(ProcUnitConf):
725 778
726 779 self.inputId = '0'
727 780 self.parentId = parentId
728
781 self.queue = queue
729 782 self.addRunOperation(**kwargs)
730 783
731 784 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
Show file before | Show file after | Hide comments
@@ -84,7 +84,8 class PlotData(Operation, Process):
84 84 print 'plotting...{}'.format(self.CODE)
85 85
86 86 self.plot()
87 self.figure.suptitle('{} {}'.format(self.title, self.CODE.upper()))
87 self.figure.suptitle('{} {} - Date:{}'.format(self.title, self.CODE.upper(),
88 datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')))
88 89
89 90 if self.save:
90 91 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
@@ -234,17 +235,17 class PlotSpectraData(PlotData):
234 235 ax.ax_profile.set_ylim(self.ymin, self.ymax)
235 236 ax.ax_profile.set_xlabel('dB')
236 237 ax.ax_profile.grid(b=True, axis='x')
238 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
239 color="k", linestyle="dashed", lw=2)[0]
237 240 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
238 noise = 10*numpy.log10(self.data['rti'][self.max_time][n]/self.dataOut.normFactor)
239 ax.ax_profile.vlines(noise, self.ymin, self.ymax, colors="k", linestyle="dashed", lw=2)
240 241 else:
241 242 ax.plot.set_array(z[n].T.ravel())
242 ax.set_title('{} {}'.format(self.titles[n],
243 datetime.datetime.utcfromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
244 size=8)
245 243 if self.showprofile:
246 244 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
245 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
247 246
247 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
248 size=8)
248 249
249 250 class PlotRTIData(PlotData):
250 251
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (966 lines changed) Show them Hide them
@@ -17,7 +17,7 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20
20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
@@ -27,11 +27,11 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 Excepciones:
30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
@@ -45,271 +45,271 def isNumber(cad):
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56
56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60
60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64
64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73
73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76
76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80
80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83
83
84 84 return 1
85 85
86 86 def isTimeInRange(thisTime, startTime, endTime):
87
87
88 88 if endTime >= startTime:
89 89 if (thisTime < startTime) or (thisTime > endTime):
90 90 return 0
91
91
92 92 return 1
93 93 else:
94 94 if (thisTime < startTime) and (thisTime > endTime):
95 95 return 0
96
96
97 97 return 1
98
98
99 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 100 """
101 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102
102
103 103 Inputs:
104 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105
105
106 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107
107
108 108 endDate : fecha final del rango seleccionado en formato datetime.date
109
109
110 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111
111
112 112 endTime : tiempo final del rango seleccionado en formato datetime.time
113
113
114 114 Return:
115 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 116 fecha especificado, de lo contrario retorna False.
117
117
118 118 Excepciones:
119 119 Si el archivo no existe o no puede ser abierto
120 120 Si la cabecera no puede ser leida.
121
121
122 122 """
123
124
123
124
125 125 try:
126 126 fp = open(filename,'rb')
127 127 except IOError:
128 128 print "The file %s can't be opened" %(filename)
129 129 return None
130
130
131 131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 132 systemHeaderObj = SystemHeader()
133 133 radarControllerHeaderObj = RadarControllerHeader()
134 134 processingHeaderObj = ProcessingHeader()
135
135
136 136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137
137
138 138 sts = firstBasicHeaderObj.read(fp)
139
139
140 140 if not(sts):
141 141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 142 return None
143
143
144 144 if not systemHeaderObj.read(fp):
145 145 return None
146
146
147 147 if not radarControllerHeaderObj.read(fp):
148 148 return None
149
149
150 150 if not processingHeaderObj.read(fp):
151 151 return None
152
152
153 153 filesize = os.path.getsize(filename)
154
154
155 155 offset = processingHeaderObj.blockSize + 24 #header size
156
156
157 157 if filesize <= offset:
158 158 print "[Reading] %s: This file has not enough data" %filename
159 159 return None
160
160
161 161 fp.seek(-offset, 2)
162
162
163 163 sts = lastBasicHeaderObj.read(fp)
164
164
165 165 fp.close()
166
166
167 167 thisDatetime = lastBasicHeaderObj.datatime
168 168 thisTime_last_block = thisDatetime.time()
169
169
170 170 thisDatetime = firstBasicHeaderObj.datatime
171 171 thisDate = thisDatetime.date()
172 172 thisTime_first_block = thisDatetime.time()
173
173
174 174 #General case
175 175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 176 #-----------o----------------------------o-----------
177 177 # startTime endTime
178
178
179 179 if endTime >= startTime:
180 180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 181 return None
182
182
183 183 return thisDatetime
184
184
185 185 #If endTime < startTime then endTime belongs to the next day
186
187
186
187
188 188 #<<<<<<<<<<<o o>>>>>>>>>>>
189 189 #-----------o----------------------------o-----------
190 190 # endTime startTime
191
191
192 192 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 193 return None
194
194
195 195 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 196 return None
197
197
198 198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 199 return None
200
200
201 201 return thisDatetime
202 202
203 203 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 204 """
205 205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206
206
207 207 Inputs:
208 208 folder : nombre completo del directorio.
209 209 Su formato deberia ser "/path_root/?YYYYDDD"
210
210
211 211 siendo:
212 212 YYYY : Anio (ejemplo 2015)
213 213 DDD : Dia del anio (ejemplo 305)
214
214
215 215 startDate : fecha inicial del rango seleccionado en formato datetime.date
216
216
217 217 endDate : fecha final del rango seleccionado en formato datetime.date
218
218
219 219 Return:
220 220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 221 fecha especificado, de lo contrario retorna False.
222 222 Excepciones:
223 223 Si el directorio no tiene el formato adecuado
224 224 """
225
225
226 226 basename = os.path.basename(folder)
227
227
228 228 if not isRadarFolder(basename):
229 229 print "The folder %s has not the rigth format" %folder
230 230 return 0
231
231
232 232 if startDate and endDate:
233 233 thisDate = getDateFromRadarFolder(basename)
234
234
235 235 if thisDate < startDate:
236 236 return 0
237
237
238 238 if thisDate > endDate:
239 239 return 0
240
240
241 241 return 1
242 242
243 243 def isFileInDateRange(filename, startDate=None, endDate=None):
244 244 """
245 245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246
246
247 247 Inputs:
248 248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249
249
250 250 Su formato deberia ser "?YYYYDDDsss"
251
251
252 252 siendo:
253 253 YYYY : Anio (ejemplo 2015)
254 254 DDD : Dia del anio (ejemplo 305)
255 255 sss : set
256
256
257 257 startDate : fecha inicial del rango seleccionado en formato datetime.date
258
258
259 259 endDate : fecha final del rango seleccionado en formato datetime.date
260
260
261 261 Return:
262 262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 263 fecha especificado, de lo contrario retorna False.
264 264 Excepciones:
265 265 Si el archivo no tiene el formato adecuado
266 266 """
267
267
268 268 basename = os.path.basename(filename)
269
269
270 270 if not isRadarFile(basename):
271 271 print "The filename %s has not the rigth format" %filename
272 272 return 0
273
273
274 274 if startDate and endDate:
275 275 thisDate = getDateFromRadarFile(basename)
276
276
277 277 if thisDate < startDate:
278 278 return 0
279
279
280 280 if thisDate > endDate:
281 281 return 0
282
282
283 283 return 1
284 284
285 285 def getFileFromSet(path, ext, set):
286 286 validFilelist = []
287 287 fileList = os.listdir(path)
288
288
289 289 # 0 1234 567 89A BCDE
290 290 # H YYYY DDD SSS .ext
291
291
292 292 for thisFile in fileList:
293 293 try:
294 294 year = int(thisFile[1:5])
295 295 doy = int(thisFile[5:8])
296 296 except:
297 297 continue
298
298
299 299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 300 continue
301
301
302 302 validFilelist.append(thisFile)
303 303
304 304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305
305
306 306 if len(myfile)!= 0:
307 307 return myfile[0]
308 308 else:
309 309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 310 print 'the filename %s does not exist'%filename
311 311 print '...going to the last file: '
312
312
313 313 if validFilelist:
314 314 validFilelist = sorted( validFilelist, key=str.lower )
315 315 return validFilelist[-1]
@@ -319,37 +319,37 def getFileFromSet(path, ext, set):
319 319 def getlastFileFromPath(path, ext):
320 320 """
321 321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323
324 Input:
322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323
324 Input:
325 325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 ext : extension de los files contenidos en una carpeta
327
326 ext : extension de los files contenidos en una carpeta
327
328 328 Return:
329 329 El ultimo file de una determinada carpeta, no se considera el path.
330 330 """
331 331 validFilelist = []
332 332 fileList = os.listdir(path)
333
333
334 334 # 0 1234 567 89A BCDE
335 335 # H YYYY DDD SSS .ext
336
336
337 337 for thisFile in fileList:
338
338
339 339 year = thisFile[1:5]
340 340 if not isNumber(year):
341 341 continue
342
342
343 343 doy = thisFile[5:8]
344 344 if not isNumber(doy):
345 345 continue
346
346
347 347 year = int(year)
348 348 doy = int(doy)
349
349
350 350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 351 continue
352
352
353 353 validFilelist.append(thisFile)
354 354
355 355 if validFilelist:
@@ -363,10 +363,10 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 365 el path exacto de un determinado file.
366
366
367 367 Example :
368 368 nombre correcto del file es .../.../D2009307/P2009307367.ext
369
369
370 370 Entonces la funcion prueba con las siguientes combinaciones
371 371 .../.../y2009307367.ext
372 372 .../.../Y2009307367.ext
@@ -374,17 +374,17 def checkForRealPath(path, foldercounter, year, doy, set, ext):
374 374 .../.../x2009307/Y2009307367.ext
375 375 .../.../X2009307/y2009307367.ext
376 376 .../.../X2009307/Y2009307367.ext
377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378
377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378
379 379 Return:
380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 para el filename
380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 para el filename
383 383 """
384 384 fullfilename = None
385 385 find_flag = False
386 386 filename = None
387
387
388 388 prefixDirList = [None,'d','D']
389 389 if ext.lower() == ".r": #voltage
390 390 prefixFileList = ['d','D']
@@ -392,20 +392,20 def checkForRealPath(path, foldercounter, year, doy, set, ext):
392 392 prefixFileList = ['p','P']
393 393 else:
394 394 return None, filename
395
396 #barrido por las combinaciones posibles
395
396 #barrido por las combinaciones posibles
397 397 for prefixDir in prefixDirList:
398 398 thispath = path
399 399 if prefixDir != None:
400 400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 401 if foldercounter == 0:
402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 403 else:
404 404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408
408
409 409 if os.path.exists( fullfilename ): #verifico que exista
410 410 find_flag = True
411 411 break
@@ -423,7 +423,7 def isRadarFolder(folder):
423 423 doy = int(folder[5:8])
424 424 except:
425 425 return 0
426
426
427 427 return 1
428 428
429 429 def isRadarFile(file):
@@ -433,7 +433,7 def isRadarFile(file):
433 433 set = int(file[8:11])
434 434 except:
435 435 return 0
436
436
437 437 return 1
438 438
439 439 def getDateFromRadarFile(file):
@@ -456,138 +456,138 def getDateFromRadarFolder(folder):
456 456
457 457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 458 return thisDate
459
459
460 460 class JRODataIO:
461
461
462 462 c = 3E8
463
463
464 464 isConfig = False
465
465
466 466 basicHeaderObj = None
467
467
468 468 systemHeaderObj = None
469
469
470 470 radarControllerHeaderObj = None
471
471
472 472 processingHeaderObj = None
473
473
474 474 dtype = None
475
475
476 476 pathList = []
477
477
478 478 filenameList = []
479
479
480 480 filename = None
481
481
482 482 ext = None
483
483
484 484 flagIsNewFile = 1
485
485
486 486 flagDiscontinuousBlock = 0
487
487
488 488 flagIsNewBlock = 0
489
489
490 490 fp = None
491
491
492 492 firstHeaderSize = 0
493
493
494 494 basicHeaderSize = 24
495
495
496 496 versionFile = 1103
497
497
498 498 fileSize = None
499
499
500 500 # ippSeconds = None
501
501
502 502 fileSizeByHeader = None
503
503
504 504 fileIndex = None
505
505
506 506 profileIndex = None
507
507
508 508 blockIndex = None
509
509
510 510 nTotalBlocks = None
511
511
512 512 maxTimeStep = 30
513
513
514 514 lastUTTime = None
515
515
516 516 datablock = None
517
517
518 518 dataOut = None
519
519
520 520 blocksize = None
521
521
522 522 getByBlock = False
523
523
524 524 def __init__(self):
525
525
526 526 raise NotImplementedError
527
527
528 528 def run(self):
529
529
530 530 raise NotImplementedError
531 531
532 532 def getDtypeWidth(self):
533
533
534 534 dtype_index = get_dtype_index(self.dtype)
535 535 dtype_width = get_dtype_width(dtype_index)
536
536
537 537 return dtype_width
538
538
539 539 class JRODataReader(JRODataIO):
540
541
540
541
542 542 online = 0
543
543
544 544 realtime = 0
545
545
546 546 nReadBlocks = 0
547
547
548 548 delay = 10 #number of seconds waiting a new file
549
549
550 550 nTries = 3 #quantity tries
551
551
552 552 nFiles = 3 #number of files for searching
553
553
554 554 path = None
555
555
556 556 foldercounter = 0
557
557
558 558 flagNoMoreFiles = 0
559
559
560 560 datetimeList = []
561
561
562 562 __isFirstTimeOnline = 1
563
563
564 564 __printInfo = True
565
565
566 566 profileIndex = None
567
567
568 568 nTxs = 1
569
569
570 570 txIndex = None
571
571
572 572 #Added--------------------
573
573
574 574 selBlocksize = None
575
575
576 576 selBlocktime = None
577
578
577
578
579 579 def __init__(self):
580
580
581 581 """
582 582 This class is used to find data files
583
583
584 584 Example:
585 585 reader = JRODataReader()
586 586 fileList = reader.findDataFiles()
587
587
588 588 """
589 589 pass
590
590
591 591
592 592 def createObjByDefault(self):
593 593 """
@@ -596,7 +596,7 class JRODataReader(JRODataIO):
596 596 raise NotImplementedError
597 597
598 598 def getBlockDimension(self):
599
599
600 600 raise NotImplementedError
601 601
602 602 def __searchFilesOffLine(self,
@@ -608,76 +608,93 class JRODataReader(JRODataIO):
608 608 set=None,
609 609 expLabel='',
610 610 ext='.r',
611 queue=None,
612 cursor=None,
613 skip=None,
611 614 walk=True):
612
615
613 616 self.filenameList = []
614 617 self.datetimeList = []
615
618
616 619 pathList = []
617
620
618 621 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
619
622
620 623 if dateList == []:
621 624 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
622 625 return None, None
623
626
624 627 if len(dateList) > 1:
625 628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
626 629 else:
627 630 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
628
631
629 632 filenameList = []
630 633 datetimeList = []
631
634
632 635 for thisPath in pathList:
633 636 # thisPath = pathList[pathDict[file]]
634
637
635 638 fileList = glob.glob1(thisPath, "*%s" %ext)
636 639 fileList.sort()
637
638 for file in fileList:
639
640
641 skippedFileList = []
642
643 if cursor is not None and skip is not None:
644 # if cursor*skip > len(fileList):
645 if skip == 0:
646 if queue is not None:
647 queue.put(len(fileList))
648 skippedFileList = []
649 else:
650 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
651
652 else:
653 skippedFileList = fileList
654
655 for file in skippedFileList:
656
640 657 filename = os.path.join(thisPath,file)
641
658
642 659 if not isFileInDateRange(filename, startDate, endDate):
643 660 continue
644
661
645 662 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
646
663
647 664 if not(thisDatetime):
648 665 continue
649
666
650 667 filenameList.append(filename)
651 668 datetimeList.append(thisDatetime)
652
669
653 670 if not(filenameList):
654 671 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
655 672 return None, None
656
673
657 674 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
658 675 print
659
676
660 677 for i in range(len(filenameList)):
661 678 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
662 679
663 680 self.filenameList = filenameList
664 681 self.datetimeList = datetimeList
665
682
666 683 return pathList, filenameList
667 684
668 685 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
669
686
670 687 """
671 688 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
672 689 devuelve el archivo encontrado ademas de otros datos.
673
674 Input:
690
691 Input:
675 692 path : carpeta donde estan contenidos los files que contiene data
676
693
677 694 expLabel : Nombre del subexperimento (subfolder)
678
695
679 696 ext : extension de los files
680
697
681 698 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
682 699
683 700 Return:
@@ -686,14 +703,14 class JRODataReader(JRODataIO):
686 703 year : el anho
687 704 doy : el numero de dia del anho
688 705 set : el set del archivo
689
690
706
707
691 708 """
692 709 if not os.path.isdir(path):
693 710 return None, None, None, None, None, None
694
711
695 712 dirList = []
696
713
697 714 if not walk:
698 715 fullpath = path
699 716 foldercounter = 0
@@ -704,19 +721,19 class JRODataReader(JRODataIO):
704 721 continue
705 722 if not isRadarFolder(thisPath):
706 723 continue
707
724
708 725 dirList.append(thisPath)
709
726
710 727 if not(dirList):
711 728 return None, None, None, None, None, None
712
729
713 730 dirList = sorted( dirList, key=str.lower )
714
731
715 732 doypath = dirList[-1]
716 733 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
717 734 fullpath = os.path.join(path, doypath, expLabel)
718
719
735
736
720 737 print "[Reading] %s folder was found: " %(fullpath )
721 738
722 739 if set == None:
@@ -726,20 +743,20 class JRODataReader(JRODataIO):
726 743
727 744 if not(filename):
728 745 return None, None, None, None, None, None
729
746
730 747 print "[Reading] %s file was found" %(filename)
731
748
732 749 if not(self.__verifyFile(os.path.join(fullpath, filename))):
733 750 return None, None, None, None, None, None
734 751
735 752 year = int( filename[1:5] )
736 753 doy = int( filename[5:8] )
737 set = int( filename[8:11] )
738
754 set = int( filename[8:11] )
755
739 756 return fullpath, foldercounter, filename, year, doy, set
740
757
741 758 def __setNextFileOffline(self):
742
759
743 760 idFile = self.fileIndex
744 761
745 762 while (True):
@@ -772,9 +789,9 class JRODataReader(JRODataIO):
772 789 """
773 790 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
774 791 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
775 siguientes.
776
777 Affected:
792 siguientes.
793
794 Affected:
778 795 self.flagIsNewFile
779 796 self.filename
780 797 self.fileSize
@@ -782,23 +799,23 class JRODataReader(JRODataIO):
782 799 self.set
783 800 self.flagNoMoreFiles
784 801
785 Return:
802 Return:
786 803 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
787 804 1 : si el file fue abierto con exito y esta listo a ser leido
788
789 Excepciones:
805
806 Excepciones:
790 807 Si un determinado file no puede ser abierto
791 808 """
792 809 nFiles = 0
793 fileOk_flag = False
810 fileOk_flag = False
794 811 firstTime_flag = True
795 812
796 813 self.set += 1
797
814
798 815 if self.set > 999:
799 816 self.set = 0
800 self.foldercounter += 1
801
817 self.foldercounter += 1
818
802 819 #busca el 1er file disponible
803 820 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
804 821 if fullfilename:
@@ -806,27 +823,27 class JRODataReader(JRODataIO):
806 823 fileOk_flag = True
807 824
808 825 #si no encuentra un file entonces espera y vuelve a buscar
809 if not(fileOk_flag):
826 if not(fileOk_flag):
810 827 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
811 828
812 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
829 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
813 830 tries = self.nTries
814 831 else:
815 832 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
816
817 for nTries in range( tries ):
833
834 for nTries in range( tries ):
818 835 if firstTime_flag:
819 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
836 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
820 837 sleep( self.delay )
821 838 else:
822 839 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
823
840
824 841 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 842 if fullfilename:
826 843 if self.__verifyFile(fullfilename):
827 844 fileOk_flag = True
828 845 break
829
846
830 847 if fileOk_flag:
831 848 break
832 849
@@ -834,7 +851,7 class JRODataReader(JRODataIO):
834 851
835 852 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
836 853 self.set += 1
837
854
838 855 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
839 856 self.set = 0
840 857 self.doy += 1
@@ -844,7 +861,7 class JRODataReader(JRODataIO):
844 861 self.fileSize = os.path.getsize( fullfilename )
845 862 self.filename = fullfilename
846 863 self.flagIsNewFile = 1
847 if self.fp != None: self.fp.close()
864 if self.fp != None: self.fp.close()
848 865 self.fp = open(fullfilename, 'rb')
849 866 self.flagNoMoreFiles = 0
850 867 # print '[Reading] Setting the file: %s' % fullfilename
@@ -857,7 +874,7 class JRODataReader(JRODataIO):
857 874 # print '[Reading] No more files to read'
858 875
859 876 return fileOk_flag
860
877
861 878 def setNextFile(self):
862 879 if self.fp != None:
863 880 self.fp.close()
@@ -872,29 +889,29 class JRODataReader(JRODataIO):
872 889 return 0
873 890
874 891 print '[Reading] Setting the file: %s' % self.filename
875
892
876 893 self.__readFirstHeader()
877 894 self.nReadBlocks = 0
878 895 return 1
879 896
880 897 def __waitNewBlock(self):
881 898 """
882 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
883
899 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
900
884 901 Si el modo de lectura es OffLine siempre retorn 0
885 902 """
886 903 if not self.online:
887 904 return 0
888
905
889 906 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
890 907 return 0
891
908
892 909 currentPointer = self.fp.tell()
893
910
894 911 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
895
912
896 913 for nTries in range( self.nTries ):
897
914
898 915 self.fp.close()
899 916 self.fp = open( self.filename, 'rb' )
900 917 self.fp.seek( currentPointer )
@@ -905,99 +922,99 class JRODataReader(JRODataIO):
905 922 if ( currentSize >= neededSize ):
906 923 self.basicHeaderObj.read(self.fp)
907 924 return 1
908
925
909 926 if self.fileSize == self.fileSizeByHeader:
910 927 # self.flagEoF = True
911 928 return 0
912
929
913 930 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
914 931 sleep( self.delay )
915
916
917 return 0
932
933
934 return 0
918 935
919 936 def waitDataBlock(self,pointer_location):
920
937
921 938 currentPointer = pointer_location
922
939
923 940 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
924
941
925 942 for nTries in range( self.nTries ):
926 943 self.fp.close()
927 944 self.fp = open( self.filename, 'rb' )
928 945 self.fp.seek( currentPointer )
929
946
930 947 self.fileSize = os.path.getsize( self.filename )
931 948 currentSize = self.fileSize - currentPointer
932
949
933 950 if ( currentSize >= neededSize ):
934 951 return 1
935
952
936 953 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
937 954 sleep( self.delay )
938
955
939 956 return 0
940 957
941 958 def __jumpToLastBlock(self):
942
959
943 960 if not(self.__isFirstTimeOnline):
944 961 return
945
962
946 963 csize = self.fileSize - self.fp.tell()
947 964 blocksize = self.processingHeaderObj.blockSize
948
965
949 966 #salta el primer bloque de datos
950 967 if csize > self.processingHeaderObj.blockSize:
951 968 self.fp.seek(self.fp.tell() + blocksize)
952 969 else:
953 970 return
954
971
955 972 csize = self.fileSize - self.fp.tell()
956 973 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
957 974 while True:
958
975
959 976 if self.fp.tell()<self.fileSize:
960 977 self.fp.seek(self.fp.tell() + neededsize)
961 978 else:
962 979 self.fp.seek(self.fp.tell() - neededsize)
963 980 break
964
981
965 982 # csize = self.fileSize - self.fp.tell()
966 983 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
967 984 # factor = int(csize/neededsize)
968 985 # if factor > 0:
969 986 # self.fp.seek(self.fp.tell() + factor*neededsize)
970
987
971 988 self.flagIsNewFile = 0
972 989 self.__isFirstTimeOnline = 0
973 990
974 991 def __setNewBlock(self):
975
992
976 993 if self.fp == None:
977 994 return 0
978
995
979 996 # if self.online:
980 997 # self.__jumpToLastBlock()
981
998
982 999 if self.flagIsNewFile:
983 1000 self.lastUTTime = self.basicHeaderObj.utc
984 1001 return 1
985
1002
986 1003 if self.realtime:
987 1004 self.flagDiscontinuousBlock = 1
988 1005 if not(self.setNextFile()):
989 1006 return 0
990 1007 else:
991 1008 return 1
992
1009
993 1010 currentSize = self.fileSize - self.fp.tell()
994 1011 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995
1012
996 1013 if (currentSize >= neededSize):
997 1014 self.basicHeaderObj.read(self.fp)
998 1015 self.lastUTTime = self.basicHeaderObj.utc
999 1016 return 1
1000
1017
1001 1018 if self.__waitNewBlock():
1002 1019 self.lastUTTime = self.basicHeaderObj.utc
1003 1020 return 1
@@ -1007,7 +1024,7 class JRODataReader(JRODataIO):
1007 1024
1008 1025 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1009 1026 self.lastUTTime = self.basicHeaderObj.utc
1010
1027
1011 1028 self.flagDiscontinuousBlock = 0
1012 1029
1013 1030 if deltaTime > self.maxTimeStep:
@@ -1016,33 +1033,33 class JRODataReader(JRODataIO):
1016 1033 return 1
1017 1034
1018 1035 def readNextBlock(self):
1019
1036
1020 1037 #Skip block out of startTime and endTime
1021 1038 while True:
1022 1039 if not(self.__setNewBlock()):
1023 1040 return 0
1024
1041
1025 1042 if not(self.readBlock()):
1026 1043 return 0
1027
1044
1028 1045 self.getBasicHeader()
1029
1046
1030 1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1031
1048
1032 1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1033 1050 self.processingHeaderObj.dataBlocksPerFile,
1034 1051 self.dataOut.datatime.ctime())
1035 1052 continue
1036
1053
1037 1054 break
1038
1055
1039 1056 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1040 1057 self.processingHeaderObj.dataBlocksPerFile,
1041 1058 self.dataOut.datatime.ctime())
1042 1059 return 1
1043 1060
1044 1061 def __readFirstHeader(self):
1045
1062
1046 1063 self.basicHeaderObj.read(self.fp)
1047 1064 self.systemHeaderObj.read(self.fp)
1048 1065 self.radarControllerHeaderObj.read(self.fp)
@@ -1072,20 +1089,20 class JRODataReader(JRODataIO):
1072 1089 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1073 1090 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1074 1091 self.getBlockDimension()
1075
1092
1076 1093 def __verifyFile(self, filename, msgFlag=True):
1077
1094
1078 1095 msg = None
1079
1096
1080 1097 try:
1081 1098 fp = open(filename, 'rb')
1082 1099 except IOError:
1083
1100
1084 1101 if msgFlag:
1085 1102 print "[Reading] File %s can't be opened" % (filename)
1086
1103
1087 1104 return False
1088
1105
1089 1106 currentPosition = fp.tell()
1090 1107 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1091 1108
@@ -1094,32 +1111,32 class JRODataReader(JRODataIO):
1094 1111 systemHeaderObj = SystemHeader()
1095 1112 radarControllerHeaderObj = RadarControllerHeader()
1096 1113 processingHeaderObj = ProcessingHeader()
1097
1114
1098 1115 if not( basicHeaderObj.read(fp) ):
1099 1116 fp.close()
1100 1117 return False
1101
1118
1102 1119 if not( systemHeaderObj.read(fp) ):
1103 1120 fp.close()
1104 1121 return False
1105
1122
1106 1123 if not( radarControllerHeaderObj.read(fp) ):
1107 1124 fp.close()
1108 1125 return False
1109
1126
1110 1127 if not( processingHeaderObj.read(fp) ):
1111 1128 fp.close()
1112 1129 return False
1113
1130
1114 1131 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1115 1132 else:
1116 1133 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1117 1134
1118 1135 fp.close()
1119
1136
1120 1137 fileSize = os.path.getsize(filename)
1121 1138 currentSize = fileSize - currentPosition
1122
1139
1123 1140 if currentSize < neededSize:
1124 1141 if msgFlag and (msg != None):
1125 1142 print msg
@@ -1128,97 +1145,97 class JRODataReader(JRODataIO):
1128 1145 return True
1129 1146
1130 1147 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1131
1148
1132 1149 path_empty = True
1133
1150
1134 1151 dateList = []
1135 1152 pathList = []
1136
1153
1137 1154 multi_path = path.split(',')
1138
1155
1139 1156 if not walk:
1140
1157
1141 1158 for single_path in multi_path:
1142
1159
1143 1160 if not os.path.isdir(single_path):
1144 1161 continue
1145
1162
1146 1163 fileList = glob.glob1(single_path, "*"+ext)
1147
1164
1148 1165 if not fileList:
1149 1166 continue
1150
1167
1151 1168 path_empty = False
1152
1169
1153 1170 fileList.sort()
1154
1171
1155 1172 for thisFile in fileList:
1156
1173
1157 1174 if not os.path.isfile(os.path.join(single_path, thisFile)):
1158 1175 continue
1159
1176
1160 1177 if not isRadarFile(thisFile):
1161 1178 continue
1162
1179
1163 1180 if not isFileInDateRange(thisFile, startDate, endDate):
1164 1181 continue
1165
1182
1166 1183 thisDate = getDateFromRadarFile(thisFile)
1167
1184
1168 1185 if thisDate in dateList:
1169 1186 continue
1170
1187
1171 1188 dateList.append(thisDate)
1172 1189 pathList.append(single_path)
1173
1190
1174 1191 else:
1175 1192 for single_path in multi_path:
1176
1193
1177 1194 if not os.path.isdir(single_path):
1178 1195 continue
1179
1196
1180 1197 dirList = []
1181
1198
1182 1199 for thisPath in os.listdir(single_path):
1183
1200
1184 1201 if not os.path.isdir(os.path.join(single_path,thisPath)):
1185 1202 continue
1186
1203
1187 1204 if not isRadarFolder(thisPath):
1188 1205 continue
1189
1206
1190 1207 if not isFolderInDateRange(thisPath, startDate, endDate):
1191 1208 continue
1192
1209
1193 1210 dirList.append(thisPath)
1194
1211
1195 1212 if not dirList:
1196 1213 continue
1197
1214
1198 1215 dirList.sort()
1199
1216
1200 1217 for thisDir in dirList:
1201
1218
1202 1219 datapath = os.path.join(single_path, thisDir, expLabel)
1203 1220 fileList = glob.glob1(datapath, "*"+ext)
1204
1221
1205 1222 if not fileList:
1206 1223 continue
1207
1224
1208 1225 path_empty = False
1209
1226
1210 1227 thisDate = getDateFromRadarFolder(thisDir)
1211
1228
1212 1229 pathList.append(datapath)
1213 1230 dateList.append(thisDate)
1214
1231
1215 1232 dateList.sort()
1216
1233
1217 1234 if walk:
1218 1235 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1219 1236 else:
1220 1237 pattern_path = multi_path[0]
1221
1238
1222 1239 if path_empty:
1223 1240 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1224 1241 else:
@@ -1227,18 +1244,18 class JRODataReader(JRODataIO):
1227 1244
1228 1245 if include_path:
1229 1246 return dateList, pathList
1230
1247
1231 1248 return dateList
1232
1249
1233 1250 def setup(self,
1234 1251 path=None,
1235 startDate=None,
1236 endDate=None,
1237 startTime=datetime.time(0,0,0),
1238 endTime=datetime.time(23,59,59),
1239 set=None,
1240 expLabel = "",
1241 ext = None,
1252 startDate=None,
1253 endDate=None,
1254 startTime=datetime.time(0,0,0),
1255 endTime=datetime.time(23,59,59),
1256 set=None,
1257 expLabel = "",
1258 ext = None,
1242 1259 online = False,
1243 1260 delay = 60,
1244 1261 walk = True,
@@ -1246,7 +1263,10 class JRODataReader(JRODataIO):
1246 1263 nTxs = 1,
1247 1264 realtime=False,
1248 1265 blocksize=None,
1249 blocktime=None):
1266 blocktime=None,
1267 queue=None,
1268 skip=None,
1269 cursor=None):
1250 1270
1251 1271 if path == None:
1252 1272 raise ValueError, "[Reading] The path is not valid"
@@ -1255,54 +1275,55 class JRODataReader(JRODataIO):
1255 1275 ext = self.ext
1256 1276
1257 1277 if online:
1258 print "[Reading] Searching files in online mode..."
1259
1278 print "[Reading] Searching files in online mode..."
1279
1260 1280 for nTries in range( self.nTries ):
1261 1281 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1262
1282
1263 1283 if fullpath:
1264 1284 break
1265
1285
1266 1286 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1267 1287 sleep( self.delay )
1268
1288
1269 1289 if not(fullpath):
1270 1290 print "[Reading] There 'isn't any valid file in %s" % path
1271 1291 return
1272
1292
1273 1293 self.year = year
1274 1294 self.doy = doy
1275 1295 self.set = set - 1
1276 1296 self.path = path
1277 1297 self.foldercounter = foldercounter
1278 1298 last_set = None
1279
1299
1280 1300 else:
1281 1301 print "[Reading] Searching files in offline mode ..."
1282 1302 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1283 1303 startTime=startTime, endTime=endTime,
1284 1304 set=set, expLabel=expLabel, ext=ext,
1285 walk=walk)
1286
1305 walk=walk, cursor=cursor,
1306 skip=skip, queue=queue)
1307
1287 1308 if not(pathList):
1288 1309 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1289 1310 # datetime.datetime.combine(startDate,startTime).ctime(),
1290 1311 # datetime.datetime.combine(endDate,endTime).ctime())
1291
1312
1292 1313 # sys.exit(-1)
1293
1314
1294 1315 self.fileIndex = -1
1295 1316 self.pathList = []
1296 1317 self.filenameList = []
1297 1318 return
1298
1319
1299 1320 self.fileIndex = -1
1300 1321 self.pathList = pathList
1301 1322 self.filenameList = filenameList
1302 1323 file_name = os.path.basename(filenameList[-1])
1303 1324 basename, ext = os.path.splitext(file_name)
1304 1325 last_set = int(basename[-3:])
1305
1326
1306 1327 self.online = online
1307 1328 self.realtime = realtime
1308 1329 self.delay = delay
@@ -1312,12 +1333,12 class JRODataReader(JRODataIO):
1312 1333 self.nTxs = nTxs
1313 1334 self.startTime = startTime
1314 1335 self.endTime = endTime
1315
1336
1316 1337 #Added-----------------
1317 1338 self.selBlocksize = blocksize
1318 1339 self.selBlocktime = blocktime
1319
1320
1340
1341
1321 1342 if not(self.setNextFile()):
1322 1343 if (startDate!=None) and (endDate!=None):
1323 1344 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
@@ -1325,116 +1346,116 class JRODataReader(JRODataIO):
1325 1346 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1326 1347 else:
1327 1348 print "[Reading] No files"
1328
1349
1329 1350 self.fileIndex = -1
1330 1351 self.pathList = []
1331 1352 self.filenameList = []
1332 1353 return
1333 1354
1334 1355 # self.getBasicHeader()
1335
1356
1336 1357 if last_set != None:
1337 1358 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1338 1359 return
1339 1360
1340 1361 def getBasicHeader(self):
1341
1362
1342 1363 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1343
1364
1344 1365 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1345
1366
1346 1367 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1347
1368
1348 1369 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1349
1370
1350 1371 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1351
1372
1352 1373 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1353
1374
1354 1375 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1355
1376
1356 1377 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1357
1358
1378
1379
1359 1380 def getFirstHeader(self):
1360
1381
1361 1382 raise NotImplementedError
1362
1383
1363 1384 def getData(self):
1364
1385
1365 1386 raise NotImplementedError
1366 1387
1367 1388 def hasNotDataInBuffer(self):
1368
1389
1369 1390 raise NotImplementedError
1370 1391
1371 1392 def readBlock(self):
1372
1393
1373 1394 raise NotImplementedError
1374
1395
1375 1396 def isEndProcess(self):
1376
1397
1377 1398 return self.flagNoMoreFiles
1378
1399
1379 1400 def printReadBlocks(self):
1380
1401
1381 1402 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1382
1403
1383 1404 def printTotalBlocks(self):
1384
1405
1385 1406 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1386 1407
1387 1408 def printNumberOfBlock(self):
1388
1409
1389 1410 if self.flagIsNewBlock:
1390 1411 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1391 1412 self.processingHeaderObj.dataBlocksPerFile,
1392 1413 self.dataOut.datatime.ctime())
1393 1414
1394 1415 def printInfo(self):
1395
1416
1396 1417 if self.__printInfo == False:
1397 1418 return
1398
1419
1399 1420 self.basicHeaderObj.printInfo()
1400 1421 self.systemHeaderObj.printInfo()
1401 1422 self.radarControllerHeaderObj.printInfo()
1402 1423 self.processingHeaderObj.printInfo()
1403
1424
1404 1425 self.__printInfo = False
1405
1406
1426
1427
1407 1428 def run(self, **kwargs):
1408
1429
1409 1430 if not(self.isConfig):
1410
1431
1411 1432 # self.dataOut = dataOut
1412 1433 self.setup(**kwargs)
1413 1434 self.isConfig = True
1414
1435
1415 1436 self.getData()
1416 1437
1417 1438 class JRODataWriter(JRODataIO):
1418 1439
1419 """
1440 """
1420 1441 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1421 de los datos siempre se realiza por bloques.
1442 de los datos siempre se realiza por bloques.
1422 1443 """
1423
1444
1424 1445 blockIndex = 0
1425
1446
1426 1447 path = None
1427
1448
1428 1449 setFile = None
1429
1450
1430 1451 profilesPerBlock = None
1431
1452
1432 1453 blocksPerFile = None
1433
1454
1434 1455 nWriteBlocks = 0
1435
1456
1436 1457 fileDate = None
1437
1458
1438 1459 def __init__(self, dataOut=None):
1439 1460 raise NotImplementedError
1440 1461
@@ -1446,7 +1467,7 class JRODataWriter(JRODataIO):
1446 1467 def setBlockDimension(self):
1447 1468 raise NotImplementedError
1448 1469
1449
1470
1450 1471 def writeBlock(self):
1451 1472 raise NotImplementedError
1452 1473
@@ -1454,140 +1475,140 class JRODataWriter(JRODataIO):
1454 1475 def putData(self):
1455 1476 raise NotImplementedError
1456 1477
1457
1478
1458 1479 def getProcessFlags(self):
1459
1480
1460 1481 processFlags = 0
1461
1482
1462 1483 dtype_index = get_dtype_index(self.dtype)
1463 1484 procflag_dtype = get_procflag_dtype(dtype_index)
1464
1485
1465 1486 processFlags += procflag_dtype
1466
1487
1467 1488 if self.dataOut.flagDecodeData:
1468 1489 processFlags += PROCFLAG.DECODE_DATA
1469
1490
1470 1491 if self.dataOut.flagDeflipData:
1471 1492 processFlags += PROCFLAG.DEFLIP_DATA
1472
1493
1473 1494 if self.dataOut.code is not None:
1474 1495 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1475
1496
1476 1497 if self.dataOut.nCohInt > 1:
1477 1498 processFlags += PROCFLAG.COHERENT_INTEGRATION
1478
1499
1479 1500 if self.dataOut.type == "Spectra":
1480 1501 if self.dataOut.nIncohInt > 1:
1481 1502 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1482
1503
1483 1504 if self.dataOut.data_dc is not None:
1484 1505 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1485
1506
1486 1507 if self.dataOut.flagShiftFFT:
1487 1508 processFlags += PROCFLAG.SHIFT_FFT_DATA
1488
1509
1489 1510 return processFlags
1490
1511
1491 1512 def setBasicHeader(self):
1492
1513
1493 1514 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1494 1515 self.basicHeaderObj.version = self.versionFile
1495 1516 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1496
1517
1497 1518 utc = numpy.floor(self.dataOut.utctime)
1498 1519 milisecond = (self.dataOut.utctime - utc)* 1000.0
1499
1520
1500 1521 self.basicHeaderObj.utc = utc
1501 1522 self.basicHeaderObj.miliSecond = milisecond
1502 1523 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1503 1524 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1504 1525 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1505
1526
1506 1527 def setFirstHeader(self):
1507 1528 """
1508 1529 Obtiene una copia del First Header
1509
1530
1510 1531 Affected:
1511
1532
1512 1533 self.basicHeaderObj
1513 1534 self.systemHeaderObj
1514 1535 self.radarControllerHeaderObj
1515 1536 self.processingHeaderObj self.
1516
1537
1517 1538 Return:
1518 1539 None
1519 1540 """
1520
1541
1521 1542 raise NotImplementedError
1522
1543
1523 1544 def __writeFirstHeader(self):
1524 1545 """
1525 1546 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1526
1547
1527 1548 Affected:
1528 1549 __dataType
1529
1550
1530 1551 Return:
1531 1552 None
1532 1553 """
1533
1554
1534 1555 # CALCULAR PARAMETROS
1535
1556
1536 1557 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1537 1558 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1538
1559
1539 1560 self.basicHeaderObj.write(self.fp)
1540 1561 self.systemHeaderObj.write(self.fp)
1541 1562 self.radarControllerHeaderObj.write(self.fp)
1542 1563 self.processingHeaderObj.write(self.fp)
1543
1564
1544 1565 def __setNewBlock(self):
1545 1566 """
1546 1567 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1547
1568
1548 1569 Return:
1549 1570 0 : si no pudo escribir nada
1550 1571 1 : Si escribio el Basic el First Header
1551 """
1572 """
1552 1573 if self.fp == None:
1553 1574 self.setNextFile()
1554
1575
1555 1576 if self.flagIsNewFile:
1556 1577 return 1
1557
1578
1558 1579 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1559 1580 self.basicHeaderObj.write(self.fp)
1560 1581 return 1
1561
1582
1562 1583 if not( self.setNextFile() ):
1563 1584 return 0
1564
1585
1565 1586 return 1
1566 1587
1567 1588
1568 1589 def writeNextBlock(self):
1569 1590 """
1570 1591 Selecciona el bloque siguiente de datos y los escribe en un file
1571
1572 Return:
1573 0 : Si no hizo pudo escribir el bloque de datos
1592
1593 Return:
1594 0 : Si no hizo pudo escribir el bloque de datos
1574 1595 1 : Si no pudo escribir el bloque de datos
1575 1596 """
1576 1597 if not( self.__setNewBlock() ):
1577 1598 return 0
1578
1599
1579 1600 self.writeBlock()
1580
1601
1581 1602 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1582 1603 self.processingHeaderObj.dataBlocksPerFile)
1583
1584 return 1
1604
1605 return 1
1585 1606
1586 1607 def setNextFile(self):
1587 """
1608 """
1588 1609 Determina el siguiente file que sera escrito
1589 1610
1590 Affected:
1611 Affected:
1591 1612 self.filename
1592 1613 self.subfolder
1593 1614 self.fp
@@ -1600,16 +1621,16 class JRODataWriter(JRODataIO):
1600 1621 """
1601 1622 ext = self.ext
1602 1623 path = self.path
1603
1624
1604 1625 if self.fp != None:
1605 1626 self.fp.close()
1606
1627
1607 1628 timeTuple = time.localtime( self.dataOut.utctime)
1608 1629 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1609 1630
1610 1631 fullpath = os.path.join( path, subfolder )
1611 1632 setFile = self.setFile
1612
1633
1613 1634 if not( os.path.exists(fullpath) ):
1614 1635 os.mkdir(fullpath)
1615 1636 setFile = -1 #inicializo mi contador de seteo
@@ -1623,49 +1644,49 class JRODataWriter(JRODataIO):
1623 1644 # x YYYY DDD SSS .ext
1624 1645 if isNumber( filen[8:11] ):
1625 1646 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1626 else:
1647 else:
1627 1648 setFile = -1
1628 1649 else:
1629 1650 setFile = -1 #inicializo mi contador de seteo
1630
1651
1631 1652 setFile += 1
1632
1653
1633 1654 #If this is a new day it resets some values
1634 1655 if self.dataOut.datatime.date() > self.fileDate:
1635 1656 setFile = 0
1636 1657 self.nTotalBlocks = 0
1637
1658
1638 1659 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1639 1660
1640 1661 filename = os.path.join( path, subfolder, filen )
1641 1662
1642 1663 fp = open( filename,'wb' )
1643
1664
1644 1665 self.blockIndex = 0
1645
1646 #guardando atributos
1666
1667 #guardando atributos
1647 1668 self.filename = filename
1648 1669 self.subfolder = subfolder
1649 1670 self.fp = fp
1650 1671 self.setFile = setFile
1651 1672 self.flagIsNewFile = 1
1652 1673 self.fileDate = self.dataOut.datatime.date()
1653
1674
1654 1675 self.setFirstHeader()
1655
1676
1656 1677 print '[Writing] Opening file: %s'%self.filename
1657
1678
1658 1679 self.__writeFirstHeader()
1659
1680
1660 1681 return 1
1661
1682
1662 1683 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1663 1684 """
1664 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1665
1685 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1686
1666 1687 Inputs:
1667 1688 path : directory where data will be saved
1668 profilesPerBlock : number of profiles per block
1689 profilesPerBlock : number of profiles per block
1669 1690 set : initial file set
1670 1691 datatype : An integer number that defines data type:
1671 1692 0 : int8 (1 byte)
@@ -1674,50 +1695,49 class JRODataWriter(JRODataIO):
1674 1695 3 : int64 (8 bytes)
1675 1696 4 : float32 (4 bytes)
1676 1697 5 : double64 (8 bytes)
1677
1698
1678 1699 Return:
1679 1700 0 : Si no realizo un buen seteo
1680 1 : Si realizo un buen seteo
1701 1 : Si realizo un buen seteo
1681 1702 """
1682
1703
1683 1704 if ext == None:
1684 1705 ext = self.ext
1685
1706
1686 1707 self.ext = ext.lower()
1687
1708
1688 1709 self.path = path
1689
1710
1690 1711 if set is None:
1691 1712 self.setFile = -1
1692 1713 else:
1693 1714 self.setFile = set - 1
1694
1715
1695 1716 self.blocksPerFile = blocksPerFile
1696
1717
1697 1718 self.profilesPerBlock = profilesPerBlock
1698
1719
1699 1720 self.dataOut = dataOut
1700 1721 self.fileDate = self.dataOut.datatime.date()
1701 1722 #By default
1702 1723 self.dtype = self.dataOut.dtype
1703
1724
1704 1725 if datatype is not None:
1705 1726 self.dtype = get_numpy_dtype(datatype)
1706
1727
1707 1728 if not(self.setNextFile()):
1708 1729 print "[Writing] There isn't a next file"
1709 1730 return 0
1710
1731
1711 1732 self.setBlockDimension()
1712
1733
1713 1734 return 1
1714
1735
1715 1736 def run(self, dataOut, **kwargs):
1716
1737
1717 1738 if not(self.isConfig):
1718
1739
1719 1740 self.setup(dataOut, **kwargs)
1720 1741 self.isConfig = True
1721
1722 self.putData()
1723 1742
1743 self.putData()
Show file before | Show file after | Hide comments
@@ -279,7 +279,7 class ReceiverData(ProcessingUnit, Process):
279 279 self.data['times'] = []
280 280 for plottype in self.plottypes:
281 281 self.data[plottype] = {}
282
282 self.data['noise'] = {}
283 283 self.isConfig = True
284 284
285 285 def event_monitor(self, monitor):
@@ -325,8 +325,8 class ReceiverData(ProcessingUnit, Process):
325 325
326 326 if plottype == 'spc':
327 327 z = self.dataOut.data_spc/self.dataOut.normFactor
328 zdB = 10*numpy.log10(z)
329 self.data[plottype] = zdB
328 self.data[plottype] = 10*numpy.log10(z)
329 self.data['noise'][t] = 10*numpy.log10(self.dataOut.getNoise()/self.dataOut.normFactor)
330 330 if plottype == 'rti':
331 331 self.data[plottype][t] = self.dataOut.getPower()
332 332 if plottype == 'snr':
General Comments 0
You need to be logged in to leave comments. Login now