##// END OF EJS Templates
schain-jc
RSS
cambios xmax xmin
José Chávez -
r1004:3c7daf19daec merge
parent child
Show More
Side by Side Unified
Context file:
r1004:3c7daf19daec Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1324 +1,1329
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, Queue, cpu_count
13 from profilehooks import profile, coverage
13
14
14 import schainpy
15 import schainpy
15 import schainpy.admin
16 import schainpy.admin
16
17
17 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 from xml.dom import minidom
19 from xml.dom import minidom
19
20
20 from schainpy.model import *
21 from schainpy.model import *
21 from time import sleep
22 from time import sleep
22
23
24
25
23 def prettify(elem):
26 def prettify(elem):
24 """Return a pretty-printed XML string for the Element.
27 """Return a pretty-printed XML string for the Element.
25 """
28 """
26 rough_string = tostring(elem, 'utf-8')
29 rough_string = tostring(elem, 'utf-8')
27 reparsed = minidom.parseString(rough_string)
30 reparsed = minidom.parseString(rough_string)
28 return reparsed.toprettyxml(indent=" ")
31 return reparsed.toprettyxml(indent=" ")
29
32
30 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
31 skip = 0
34 skip = 0
32 cursor = 0
35 cursor = 0
33 nFiles = None
36 nFiles = None
34 processes = []
37 processes = []
35 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
36 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
37 days = (dt2 - dt1).days
40 days = (dt2 - dt1).days
38
41
39 for day in range(days+1):
42 for day in range(days+1):
40 skip = 0
43 skip = 0
41 cursor = 0
44 cursor = 0
42 q = Queue()
45 q = Queue()
43 processes = []
46 processes = []
44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
46 firstProcess.start()
49 firstProcess.start()
47 if by_day:
50 if by_day:
48 continue
51 continue
49 nFiles = q.get()
52 nFiles = q.get()
50 firstProcess.terminate()
53 firstProcess.terminate()
51 skip = int(math.ceil(nFiles/nProcess))
54 skip = int(math.ceil(nFiles/nProcess))
52 while True:
55 while True:
53 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
56 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
54 processes[cursor].start()
57 processes[cursor].start()
55 if nFiles < cursor*skip:
58 if nFiles < cursor*skip:
56 break
59 break
57 cursor += 1
60 cursor += 1
58
61
59 def beforeExit(exctype, value, trace):
62 def beforeExit(exctype, value, trace):
60 for process in processes:
63 for process in processes:
61 process.terminate()
64 process.terminate()
62 process.join()
65 process.join()
63 print traceback.print_tb(trace)
66 print traceback.print_tb(trace)
64
67
65 sys.excepthook = beforeExit
68 sys.excepthook = beforeExit
66
69
67 for process in processes:
70 for process in processes:
68 process.join()
71 process.join()
69 process.terminate()
72 process.terminate()
70 time.sleep(3)
73 time.sleep(3)
71
74
72 class ParameterConf():
75 class ParameterConf():
73
76
74 id = None
77 id = None
75 name = None
78 name = None
76 value = None
79 value = None
77 format = None
80 format = None
78
81
79 __formated_value = None
82 __formated_value = None
80
83
81 ELEMENTNAME = 'Parameter'
84 ELEMENTNAME = 'Parameter'
82
85
83 def __init__(self):
86 def __init__(self):
84
87
85 self.format = 'str'
88 self.format = 'str'
86
89
87 def getElementName(self):
90 def getElementName(self):
88
91
89 return self.ELEMENTNAME
92 return self.ELEMENTNAME
90
93
91 def getValue(self):
94 def getValue(self):
92
95
93 value = self.value
96 value = self.value
94 format = self.format
97 format = self.format
95
98
96 if self.__formated_value != None:
99 if self.__formated_value != None:
97
100
98 return self.__formated_value
101 return self.__formated_value
99
102
100 if format == 'obj':
103 if format == 'obj':
101 return value
104 return value
102
105
103 if format == 'str':
106 if format == 'str':
104 self.__formated_value = str(value)
107 self.__formated_value = str(value)
105 return self.__formated_value
108 return self.__formated_value
106
109
107 if value == '':
110 if value == '':
108 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, "%s: This parameter value is empty" %self.name
109
112
110 if format == 'list':
113 if format == 'list':
111 strList = value.split(',')
114 strList = value.split(',')
112
115
113 self.__formated_value = strList
116 self.__formated_value = strList
114
117
115 return self.__formated_value
118 return self.__formated_value
116
119
117 if format == 'intlist':
120 if format == 'intlist':
118 """
121 """
119 Example:
122 Example:
120 value = (0,1,2)
123 value = (0,1,2)
121 """
124 """
122
125
123 new_value = ast.literal_eval(value)
126 new_value = ast.literal_eval(value)
124
127
125 if type(new_value) not in (tuple, list):
128 if type(new_value) not in (tuple, list):
126 new_value = [int(new_value)]
129 new_value = [int(new_value)]
127
130
128 self.__formated_value = new_value
131 self.__formated_value = new_value
129
132
130 return self.__formated_value
133 return self.__formated_value
131
134
132 if format == 'floatlist':
135 if format == 'floatlist':
133 """
136 """
134 Example:
137 Example:
135 value = (0.5, 1.4, 2.7)
138 value = (0.5, 1.4, 2.7)
136 """
139 """
137
140
138 new_value = ast.literal_eval(value)
141 new_value = ast.literal_eval(value)
139
142
140 if type(new_value) not in (tuple, list):
143 if type(new_value) not in (tuple, list):
141 new_value = [float(new_value)]
144 new_value = [float(new_value)]
142
145
143 self.__formated_value = new_value
146 self.__formated_value = new_value
144
147
145 return self.__formated_value
148 return self.__formated_value
146
149
147 if format == 'date':
150 if format == 'date':
148 strList = value.split('/')
151 strList = value.split('/')
149 intList = [int(x) for x in strList]
152 intList = [int(x) for x in strList]
150 date = datetime.date(intList[0], intList[1], intList[2])
153 date = datetime.date(intList[0], intList[1], intList[2])
151
154
152 self.__formated_value = date
155 self.__formated_value = date
153
156
154 return self.__formated_value
157 return self.__formated_value
155
158
156 if format == 'time':
159 if format == 'time':
157 strList = value.split(':')
160 strList = value.split(':')
158 intList = [int(x) for x in strList]
161 intList = [int(x) for x in strList]
159 time = datetime.time(intList[0], intList[1], intList[2])
162 time = datetime.time(intList[0], intList[1], intList[2])
160
163
161 self.__formated_value = time
164 self.__formated_value = time
162
165
163 return self.__formated_value
166 return self.__formated_value
164
167
165 if format == 'pairslist':
168 if format == 'pairslist':
166 """
169 """
167 Example:
170 Example:
168 value = (0,1),(1,2)
171 value = (0,1),(1,2)
169 """
172 """
170
173
171 new_value = ast.literal_eval(value)
174 new_value = ast.literal_eval(value)
172
175
173 if type(new_value) not in (tuple, list):
176 if type(new_value) not in (tuple, list):
174 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, "%s has to be a tuple or list of pairs" %value
175
178
176 if type(new_value[0]) not in (tuple, list):
179 if type(new_value[0]) not in (tuple, list):
177 if len(new_value) != 2:
180 if len(new_value) != 2:
178 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
179 new_value = [new_value]
182 new_value = [new_value]
180
183
181 for thisPair in new_value:
184 for thisPair in new_value:
182 if len(thisPair) != 2:
185 if len(thisPair) != 2:
183 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, "%s has to be a tuple or list of pairs" %value
184
187
185 self.__formated_value = new_value
188 self.__formated_value = new_value
186
189
187 return self.__formated_value
190 return self.__formated_value
188
191
189 if format == 'multilist':
192 if format == 'multilist':
190 """
193 """
191 Example:
194 Example:
192 value = (0,1,2),(3,4,5)
195 value = (0,1,2),(3,4,5)
193 """
196 """
194 multiList = ast.literal_eval(value)
197 multiList = ast.literal_eval(value)
195
198
196 if type(multiList[0]) == int:
199 if type(multiList[0]) == int:
197 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval("(" + value + ")")
198
201
199 self.__formated_value = multiList
202 self.__formated_value = multiList
200
203
201 return self.__formated_value
204 return self.__formated_value
202
205
203 if format == 'bool':
206 if format == 'bool':
204 value = int(value)
207 value = int(value)
205
208
206 if format == 'int':
209 if format == 'int':
207 value = float(value)
210 value = float(value)
208
211
209 format_func = eval(format)
212 format_func = eval(format)
210
213
211 self.__formated_value = format_func(value)
214 self.__formated_value = format_func(value)
212
215
213 return self.__formated_value
216 return self.__formated_value
214
217
215 def updateId(self, new_id):
218 def updateId(self, new_id):
216
219
217 self.id = str(new_id)
220 self.id = str(new_id)
218
221
219 def setup(self, id, name, value, format='str'):
222 def setup(self, id, name, value, format='str'):
220 self.id = str(id)
223 self.id = str(id)
221 self.name = name
224 self.name = name
222 if format == 'obj':
225 if format == 'obj':
223 self.value = value
226 self.value = value
224 else:
227 else:
225 self.value = str(value)
228 self.value = str(value)
226 self.format = str.lower(format)
229 self.format = str.lower(format)
227
230
228 self.getValue()
231 self.getValue()
229
232
230 return 1
233 return 1
231
234
232 def update(self, name, value, format='str'):
235 def update(self, name, value, format='str'):
233
236
234 self.name = name
237 self.name = name
235 self.value = str(value)
238 self.value = str(value)
236 self.format = format
239 self.format = format
237
240
238 def makeXml(self, opElement):
241 def makeXml(self, opElement):
239 if self.name not in ('queue',):
242 if self.name not in ('queue',):
240 parmElement = SubElement(opElement, self.ELEMENTNAME)
243 parmElement = SubElement(opElement, self.ELEMENTNAME)
241 parmElement.set('id', str(self.id))
244 parmElement.set('id', str(self.id))
242 parmElement.set('name', self.name)
245 parmElement.set('name', self.name)
243 parmElement.set('value', self.value)
246 parmElement.set('value', self.value)
244 parmElement.set('format', self.format)
247 parmElement.set('format', self.format)
245
248
246 def readXml(self, parmElement):
249 def readXml(self, parmElement):
247
250
248 self.id = parmElement.get('id')
251 self.id = parmElement.get('id')
249 self.name = parmElement.get('name')
252 self.name = parmElement.get('name')
250 self.value = parmElement.get('value')
253 self.value = parmElement.get('value')
251 self.format = str.lower(parmElement.get('format'))
254 self.format = str.lower(parmElement.get('format'))
252
255
253 #Compatible with old signal chain version
256 #Compatible with old signal chain version
254 if self.format == 'int' and self.name == 'idfigure':
257 if self.format == 'int' and self.name == 'idfigure':
255 self.name = 'id'
258 self.name = 'id'
256
259
257 def printattr(self):
260 def printattr(self):
258
261
259 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
260
263
261 class OperationConf():
264 class OperationConf():
262
265
263 id = None
266 id = None
264 name = None
267 name = None
265 priority = None
268 priority = None
266 type = None
269 type = None
267
270
268 parmConfObjList = []
271 parmConfObjList = []
269
272
270 ELEMENTNAME = 'Operation'
273 ELEMENTNAME = 'Operation'
271
274
272 def __init__(self):
275 def __init__(self):
273
276
274 self.id = '0'
277 self.id = '0'
275 self.name = None
278 self.name = None
276 self.priority = None
279 self.priority = None
277 self.type = 'self'
280 self.type = 'self'
278
281
279
282
280 def __getNewId(self):
283 def __getNewId(self):
281
284
282 return int(self.id)*10 + len(self.parmConfObjList) + 1
285 return int(self.id)*10 + len(self.parmConfObjList) + 1
283
286
284 def updateId(self, new_id):
287 def updateId(self, new_id):
285
288
286 self.id = str(new_id)
289 self.id = str(new_id)
287
290
288 n = 1
291 n = 1
289 for parmObj in self.parmConfObjList:
292 for parmObj in self.parmConfObjList:
290
293
291 idParm = str(int(new_id)*10 + n)
294 idParm = str(int(new_id)*10 + n)
292 parmObj.updateId(idParm)
295 parmObj.updateId(idParm)
293
296
294 n += 1
297 n += 1
295
298
296 def getElementName(self):
299 def getElementName(self):
297
300
298 return self.ELEMENTNAME
301 return self.ELEMENTNAME
299
302
300 def getParameterObjList(self):
303 def getParameterObjList(self):
301
304
302 return self.parmConfObjList
305 return self.parmConfObjList
303
306
304 def getParameterObj(self, parameterName):
307 def getParameterObj(self, parameterName):
305
308
306 for parmConfObj in self.parmConfObjList:
309 for parmConfObj in self.parmConfObjList:
307
310
308 if parmConfObj.name != parameterName:
311 if parmConfObj.name != parameterName:
309 continue
312 continue
310
313
311 return parmConfObj
314 return parmConfObj
312
315
313 return None
316 return None
314
317
315 def getParameterObjfromValue(self, parameterValue):
318 def getParameterObjfromValue(self, parameterValue):
316
319
317 for parmConfObj in self.parmConfObjList:
320 for parmConfObj in self.parmConfObjList:
318
321
319 if parmConfObj.getValue() != parameterValue:
322 if parmConfObj.getValue() != parameterValue:
320 continue
323 continue
321
324
322 return parmConfObj.getValue()
325 return parmConfObj.getValue()
323
326
324 return None
327 return None
325
328
326 def getParameterValue(self, parameterName):
329 def getParameterValue(self, parameterName):
327
330
328 parameterObj = self.getParameterObj(parameterName)
331 parameterObj = self.getParameterObj(parameterName)
329
332
330 # if not parameterObj:
333 # if not parameterObj:
331 # return None
334 # return None
332
335
333 value = parameterObj.getValue()
336 value = parameterObj.getValue()
334
337
335 return value
338 return value
336
339
337
340
338 def getKwargs(self):
341 def getKwargs(self):
339
342
340 kwargs = {}
343 kwargs = {}
341
344
342 for parmConfObj in self.parmConfObjList:
345 for parmConfObj in self.parmConfObjList:
343 if self.name == 'run' and parmConfObj.name == 'datatype':
346 if self.name == 'run' and parmConfObj.name == 'datatype':
344 continue
347 continue
345
348
346 kwargs[parmConfObj.name] = parmConfObj.getValue()
349 kwargs[parmConfObj.name] = parmConfObj.getValue()
347
350
348 return kwargs
351 return kwargs
349
352
350 def setup(self, id, name, priority, type):
353 def setup(self, id, name, priority, type):
351
354
352 self.id = str(id)
355 self.id = str(id)
353 self.name = name
356 self.name = name
354 self.type = type
357 self.type = type
355 self.priority = priority
358 self.priority = priority
356
359
357 self.parmConfObjList = []
360 self.parmConfObjList = []
358
361
359 def removeParameters(self):
362 def removeParameters(self):
360
363
361 for obj in self.parmConfObjList:
364 for obj in self.parmConfObjList:
362 del obj
365 del obj
363
366
364 self.parmConfObjList = []
367 self.parmConfObjList = []
365
368
366 def addParameter(self, name, value, format='str'):
369 def addParameter(self, name, value, format='str'):
367
370
368 id = self.__getNewId()
371 id = self.__getNewId()
369
372
370 parmConfObj = ParameterConf()
373 parmConfObj = ParameterConf()
371 if not parmConfObj.setup(id, name, value, format):
374 if not parmConfObj.setup(id, name, value, format):
372 return None
375 return None
373
376
374 self.parmConfObjList.append(parmConfObj)
377 self.parmConfObjList.append(parmConfObj)
375
378
376 return parmConfObj
379 return parmConfObj
377
380
378 def changeParameter(self, name, value, format='str'):
381 def changeParameter(self, name, value, format='str'):
379
382
380 parmConfObj = self.getParameterObj(name)
383 parmConfObj = self.getParameterObj(name)
381 parmConfObj.update(name, value, format)
384 parmConfObj.update(name, value, format)
382
385
383 return parmConfObj
386 return parmConfObj
384
387
385 def makeXml(self, procUnitElement):
388 def makeXml(self, procUnitElement):
386
389
387 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
390 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
388 opElement.set('id', str(self.id))
391 opElement.set('id', str(self.id))
389 opElement.set('name', self.name)
392 opElement.set('name', self.name)
390 opElement.set('type', self.type)
393 opElement.set('type', self.type)
391 opElement.set('priority', str(self.priority))
394 opElement.set('priority', str(self.priority))
392
395
393 for parmConfObj in self.parmConfObjList:
396 for parmConfObj in self.parmConfObjList:
394 parmConfObj.makeXml(opElement)
397 parmConfObj.makeXml(opElement)
395
398
396 def readXml(self, opElement):
399 def readXml(self, opElement):
397
400
398 self.id = opElement.get('id')
401 self.id = opElement.get('id')
399 self.name = opElement.get('name')
402 self.name = opElement.get('name')
400 self.type = opElement.get('type')
403 self.type = opElement.get('type')
401 self.priority = opElement.get('priority')
404 self.priority = opElement.get('priority')
402
405
403 #Compatible with old signal chain version
406 #Compatible with old signal chain version
404 #Use of 'run' method instead 'init'
407 #Use of 'run' method instead 'init'
405 if self.type == 'self' and self.name == 'init':
408 if self.type == 'self' and self.name == 'init':
406 self.name = 'run'
409 self.name = 'run'
407
410
408 self.parmConfObjList = []
411 self.parmConfObjList = []
409
412
410 parmElementList = opElement.iter(ParameterConf().getElementName())
413 parmElementList = opElement.iter(ParameterConf().getElementName())
411
414
412 for parmElement in parmElementList:
415 for parmElement in parmElementList:
413 parmConfObj = ParameterConf()
416 parmConfObj = ParameterConf()
414 parmConfObj.readXml(parmElement)
417 parmConfObj.readXml(parmElement)
415
418
416 #Compatible with old signal chain version
419 #Compatible with old signal chain version
417 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
420 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
418 if self.type != 'self' and self.name == 'Plot':
421 if self.type != 'self' and self.name == 'Plot':
419 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
422 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
420 self.name = parmConfObj.value
423 self.name = parmConfObj.value
421 continue
424 continue
422
425
423 self.parmConfObjList.append(parmConfObj)
426 self.parmConfObjList.append(parmConfObj)
424
427
425 def printattr(self):
428 def printattr(self):
426
429
427 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
430 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
428 self.id,
431 self.id,
429 self.name,
432 self.name,
430 self.type,
433 self.type,
431 self.priority)
434 self.priority)
432
435
433 for parmConfObj in self.parmConfObjList:
436 for parmConfObj in self.parmConfObjList:
434 parmConfObj.printattr()
437 parmConfObj.printattr()
435
438
436 def createObject(self, plotter_queue=None):
439 def createObject(self, plotter_queue=None):
437
440
438
441
439 if self.type == 'self':
442 if self.type == 'self':
440 raise ValueError, "This operation type cannot be created"
443 raise ValueError, "This operation type cannot be created"
441
444
442 if self.type == 'plotter':
445 if self.type == 'plotter':
443 #Plotter(plotter_name)
446 #Plotter(plotter_name)
444 if not plotter_queue:
447 if not plotter_queue:
445 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
448 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
446
449
447 opObj = Plotter(self.name, plotter_queue)
450 opObj = Plotter(self.name, plotter_queue)
448
451
449 if self.type == 'external' or self.type == 'other':
452 if self.type == 'external' or self.type == 'other':
450
453
451 className = eval(self.name)
454 className = eval(self.name)
452 kwargs = self.getKwargs()
455 kwargs = self.getKwargs()
453
456
454 opObj = className(**kwargs)
457 opObj = className(**kwargs)
455
458
456 return opObj
459 return opObj
457
460
458
461
459 class ProcUnitConf():
462 class ProcUnitConf():
460
463
461 id = None
464 id = None
462 name = None
465 name = None
463 datatype = None
466 datatype = None
464 inputId = None
467 inputId = None
465 parentId = None
468 parentId = None
466
469
467 opConfObjList = []
470 opConfObjList = []
468
471
469 procUnitObj = None
472 procUnitObj = None
470 opObjList = []
473 opObjList = []
471
474
472 ELEMENTNAME = 'ProcUnit'
475 ELEMENTNAME = 'ProcUnit'
473
476
474 def __init__(self):
477 def __init__(self):
475
478
476 self.id = None
479 self.id = None
477 self.datatype = None
480 self.datatype = None
478 self.name = None
481 self.name = None
479 self.inputId = None
482 self.inputId = None
480
483
481 self.opConfObjList = []
484 self.opConfObjList = []
482
485
483 self.procUnitObj = None
486 self.procUnitObj = None
484 self.opObjDict = {}
487 self.opObjDict = {}
485
488
486 def __getPriority(self):
489 def __getPriority(self):
487
490
488 return len(self.opConfObjList)+1
491 return len(self.opConfObjList)+1
489
492
490 def __getNewId(self):
493 def __getNewId(self):
491
494
492 return int(self.id)*10 + len(self.opConfObjList) + 1
495 return int(self.id)*10 + len(self.opConfObjList) + 1
493
496
494 def getElementName(self):
497 def getElementName(self):
495
498
496 return self.ELEMENTNAME
499 return self.ELEMENTNAME
497
500
498 def getId(self):
501 def getId(self):
499
502
500 return self.id
503 return self.id
501
504
502 def updateId(self, new_id, parentId=parentId):
505 def updateId(self, new_id, parentId=parentId):
503
506
504
507
505 new_id = int(parentId)*10 + (int(self.id) % 10)
508 new_id = int(parentId)*10 + (int(self.id) % 10)
506 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
509 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
507
510
508 #If this proc unit has not inputs
511 #If this proc unit has not inputs
509 if self.inputId == '0':
512 if self.inputId == '0':
510 new_inputId = 0
513 new_inputId = 0
511
514
512 n = 1
515 n = 1
513 for opConfObj in self.opConfObjList:
516 for opConfObj in self.opConfObjList:
514
517
515 idOp = str(int(new_id)*10 + n)
518 idOp = str(int(new_id)*10 + n)
516 opConfObj.updateId(idOp)
519 opConfObj.updateId(idOp)
517
520
518 n += 1
521 n += 1
519
522
520 self.parentId = str(parentId)
523 self.parentId = str(parentId)
521 self.id = str(new_id)
524 self.id = str(new_id)
522 self.inputId = str(new_inputId)
525 self.inputId = str(new_inputId)
523
526
524
527
525 def getInputId(self):
528 def getInputId(self):
526
529
527 return self.inputId
530 return self.inputId
528
531
529 def getOperationObjList(self):
532 def getOperationObjList(self):
530
533
531 return self.opConfObjList
534 return self.opConfObjList
532
535
533 def getOperationObj(self, name=None):
536 def getOperationObj(self, name=None):
534
537
535 for opConfObj in self.opConfObjList:
538 for opConfObj in self.opConfObjList:
536
539
537 if opConfObj.name != name:
540 if opConfObj.name != name:
538 continue
541 continue
539
542
540 return opConfObj
543 return opConfObj
541
544
542 return None
545 return None
543
546
544 def getOpObjfromParamValue(self, value=None):
547 def getOpObjfromParamValue(self, value=None):
545
548
546 for opConfObj in self.opConfObjList:
549 for opConfObj in self.opConfObjList:
547 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
550 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
548 continue
551 continue
549 return opConfObj
552 return opConfObj
550 return None
553 return None
551
554
552 def getProcUnitObj(self):
555 def getProcUnitObj(self):
553
556
554 return self.procUnitObj
557 return self.procUnitObj
555
558
556 def setup(self, id, name, datatype, inputId, parentId=None):
559 def setup(self, id, name, datatype, inputId, parentId=None):
557
560
558 #Compatible with old signal chain version
561 #Compatible with old signal chain version
559 if datatype==None and name==None:
562 if datatype==None and name==None:
560 raise ValueError, "datatype or name should be defined"
563 raise ValueError, "datatype or name should be defined"
561
564
562 if name==None:
565 if name==None:
563 if 'Proc' in datatype:
566 if 'Proc' in datatype:
564 name = datatype
567 name = datatype
565 else:
568 else:
566 name = '%sProc' %(datatype)
569 name = '%sProc' %(datatype)
567
570
568 if datatype==None:
571 if datatype==None:
569 datatype = name.replace('Proc','')
572 datatype = name.replace('Proc','')
570
573
571 self.id = str(id)
574 self.id = str(id)
572 self.name = name
575 self.name = name
573 self.datatype = datatype
576 self.datatype = datatype
574 self.inputId = inputId
577 self.inputId = inputId
575 self.parentId = parentId
578 self.parentId = parentId
576
579
577 self.opConfObjList = []
580 self.opConfObjList = []
578
581
579 self.addOperation(name='run', optype='self')
582 self.addOperation(name='run', optype='self')
580
583
581 def removeOperations(self):
584 def removeOperations(self):
582
585
583 for obj in self.opConfObjList:
586 for obj in self.opConfObjList:
584 del obj
587 del obj
585
588
586 self.opConfObjList = []
589 self.opConfObjList = []
587 self.addOperation(name='run')
590 self.addOperation(name='run')
588
591
589 def addParameter(self, **kwargs):
592 def addParameter(self, **kwargs):
590 '''
593 '''
591 Add parameters to "run" operation
594 Add parameters to "run" operation
592 '''
595 '''
593 opObj = self.opConfObjList[0]
596 opObj = self.opConfObjList[0]
594
597
595 opObj.addParameter(**kwargs)
598 opObj.addParameter(**kwargs)
596
599
597 return opObj
600 return opObj
598
601
599 def addOperation(self, name, optype='self'):
602 def addOperation(self, name, optype='self'):
600
603
601 id = self.__getNewId()
604 id = self.__getNewId()
602 priority = self.__getPriority()
605 priority = self.__getPriority()
603
606
604 opConfObj = OperationConf()
607 opConfObj = OperationConf()
605 opConfObj.setup(id, name=name, priority=priority, type=optype)
608 opConfObj.setup(id, name=name, priority=priority, type=optype)
606
609
607 self.opConfObjList.append(opConfObj)
610 self.opConfObjList.append(opConfObj)
608
611
609 return opConfObj
612 return opConfObj
610
613
611 def makeXml(self, projectElement):
614 def makeXml(self, projectElement):
612
615
613 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
616 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
614 procUnitElement.set('id', str(self.id))
617 procUnitElement.set('id', str(self.id))
615 procUnitElement.set('name', self.name)
618 procUnitElement.set('name', self.name)
616 procUnitElement.set('datatype', self.datatype)
619 procUnitElement.set('datatype', self.datatype)
617 procUnitElement.set('inputId', str(self.inputId))
620 procUnitElement.set('inputId', str(self.inputId))
618
621
619 for opConfObj in self.opConfObjList:
622 for opConfObj in self.opConfObjList:
620 opConfObj.makeXml(procUnitElement)
623 opConfObj.makeXml(procUnitElement)
621
624
622 def readXml(self, upElement):
625 def readXml(self, upElement):
623
626
624 self.id = upElement.get('id')
627 self.id = upElement.get('id')
625 self.name = upElement.get('name')
628 self.name = upElement.get('name')
626 self.datatype = upElement.get('datatype')
629 self.datatype = upElement.get('datatype')
627 self.inputId = upElement.get('inputId')
630 self.inputId = upElement.get('inputId')
628
631
629 if self.ELEMENTNAME == "ReadUnit":
632 if self.ELEMENTNAME == "ReadUnit":
630 self.datatype = self.datatype.replace("Reader", "")
633 self.datatype = self.datatype.replace("Reader", "")
631
634
632 if self.ELEMENTNAME == "ProcUnit":
635 if self.ELEMENTNAME == "ProcUnit":
633 self.datatype = self.datatype.replace("Proc", "")
636 self.datatype = self.datatype.replace("Proc", "")
634
637
635 if self.inputId == 'None':
638 if self.inputId == 'None':
636 self.inputId = '0'
639 self.inputId = '0'
637
640
638 self.opConfObjList = []
641 self.opConfObjList = []
639
642
640 opElementList = upElement.iter(OperationConf().getElementName())
643 opElementList = upElement.iter(OperationConf().getElementName())
641
644
642 for opElement in opElementList:
645 for opElement in opElementList:
643 opConfObj = OperationConf()
646 opConfObj = OperationConf()
644 opConfObj.readXml(opElement)
647 opConfObj.readXml(opElement)
645 self.opConfObjList.append(opConfObj)
648 self.opConfObjList.append(opConfObj)
646
649
647 def printattr(self):
650 def printattr(self):
648
651
649 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
652 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
650 self.id,
653 self.id,
651 self.name,
654 self.name,
652 self.datatype,
655 self.datatype,
653 self.inputId)
656 self.inputId)
654
657
655 for opConfObj in self.opConfObjList:
658 for opConfObj in self.opConfObjList:
656 opConfObj.printattr()
659 opConfObj.printattr()
657
660
658
661
659 def getKwargs(self):
662 def getKwargs(self):
660
663
661 opObj = self.opConfObjList[0]
664 opObj = self.opConfObjList[0]
662 kwargs = opObj.getKwargs()
665 kwargs = opObj.getKwargs()
663
666
664 return kwargs
667 return kwargs
665
668
666 def createObjects(self, plotter_queue=None):
669 def createObjects(self, plotter_queue=None):
667
670
668 className = eval(self.name)
671 className = eval(self.name)
669 kwargs = self.getKwargs()
672 kwargs = self.getKwargs()
670 procUnitObj = className(**kwargs)
673 procUnitObj = className(**kwargs)
671
674
672 for opConfObj in self.opConfObjList:
675 for opConfObj in self.opConfObjList:
673
676
674 if opConfObj.type=='self' and self.name=='run':
677 if opConfObj.type=='self' and self.name=='run':
675 continue
678 continue
676 elif opConfObj.type=='self':
679 elif opConfObj.type=='self':
677 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
680 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
678 continue
681 continue
679
682
680 opObj = opConfObj.createObject(plotter_queue)
683 opObj = opConfObj.createObject(plotter_queue)
681
684
682 self.opObjDict[opConfObj.id] = opObj
685 self.opObjDict[opConfObj.id] = opObj
683
686
684 procUnitObj.addOperation(opObj, opConfObj.id)
687 procUnitObj.addOperation(opObj, opConfObj.id)
685
688
686 self.procUnitObj = procUnitObj
689 self.procUnitObj = procUnitObj
687
690
688 return procUnitObj
691 return procUnitObj
689
692
693 ## @profile
690 def run(self):
694 def run(self):
691
695
692 is_ok = False
696 is_ok = False
693
697
694 for opConfObj in self.opConfObjList:
698 for opConfObj in self.opConfObjList:
695
699
696 kwargs = {}
700 kwargs = {}
697 for parmConfObj in opConfObj.getParameterObjList():
701 for parmConfObj in opConfObj.getParameterObjList():
698 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
702 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
699 continue
703 continue
700
704
701 kwargs[parmConfObj.name] = parmConfObj.getValue()
705 kwargs[parmConfObj.name] = parmConfObj.getValue()
702
706
703 #ini = time.time()
707 #ini = time.time()
704
708
705 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
709 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
706 sts = self.procUnitObj.call(opType = opConfObj.type,
710 sts = self.procUnitObj.call(opType = opConfObj.type,
707 opName = opConfObj.name,
711 opName = opConfObj.name,
708 opId = opConfObj.id,
712 opId = opConfObj.id,
709 )
713 )
710
714
711 # total_time = time.time() - ini
715 # total_time = time.time() - ini
712 #
716 #
713 # if total_time > 0.002:
717 # if total_time > 0.002:
714 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
718 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
715
719
716 is_ok = is_ok or sts
720 is_ok = is_ok or sts
717
721
718 return is_ok
722 return is_ok
719
723
720 def close(self):
724 def close(self):
721
725
722 for opConfObj in self.opConfObjList:
726 for opConfObj in self.opConfObjList:
723 if opConfObj.type == 'self':
727 if opConfObj.type == 'self':
724 continue
728 continue
725
729
726 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
730 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
727 opObj.close()
731 opObj.close()
728
732
729 self.procUnitObj.close()
733 self.procUnitObj.close()
730
734
731 return
735 return
732
736
733 class ReadUnitConf(ProcUnitConf):
737 class ReadUnitConf(ProcUnitConf):
734
738
735 path = None
739 path = None
736 startDate = None
740 startDate = None
737 endDate = None
741 endDate = None
738 startTime = None
742 startTime = None
739 endTime = None
743 endTime = None
740
744
741 ELEMENTNAME = 'ReadUnit'
745 ELEMENTNAME = 'ReadUnit'
742
746
743 def __init__(self):
747 def __init__(self):
744
748
745 self.id = None
749 self.id = None
746 self.datatype = None
750 self.datatype = None
747 self.name = None
751 self.name = None
748 self.inputId = None
752 self.inputId = None
749
753
750 self.parentId = None
754 self.parentId = None
751
755
752 self.opConfObjList = []
756 self.opConfObjList = []
753 self.opObjList = []
757 self.opObjList = []
754
758
755 def getElementName(self):
759 def getElementName(self):
756
760
757 return self.ELEMENTNAME
761 return self.ELEMENTNAME
758
762
759 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
763 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
760 endTime="", parentId=None, queue=None, server=None, **kwargs):
764 endTime="", parentId=None, queue=None, server=None, **kwargs):
761
765
762 #Compatible with old signal chain version
766 #Compatible with old signal chain version
763 if datatype==None and name==None:
767 if datatype==None and name==None:
764 raise ValueError, "datatype or name should be defined"
768 raise ValueError, "datatype or name should be defined"
765
769
766 if name==None:
770 if name==None:
767 if 'Reader' in datatype:
771 if 'Reader' in datatype:
768 name = datatype
772 name = datatype
769 else:
773 else:
770 name = '%sReader' %(datatype)
774 name = '%sReader' %(datatype)
771 if datatype==None:
775 if datatype==None:
772 datatype = name.replace('Reader','')
776 datatype = name.replace('Reader','')
773
777
774 self.id = id
778 self.id = id
775 self.name = name
779 self.name = name
776 self.datatype = datatype
780 self.datatype = datatype
777 if path != '':
781 if path != '':
778 self.path = os.path.abspath(path)
782 self.path = os.path.abspath(path)
779 self.startDate = startDate
783 self.startDate = startDate
780 self.endDate = endDate
784 self.endDate = endDate
781 self.startTime = startTime
785 self.startTime = startTime
782 self.endTime = endTime
786 self.endTime = endTime
783
787
784 self.inputId = '0'
788 self.inputId = '0'
785 self.parentId = parentId
789 self.parentId = parentId
786 self.queue = queue
790 self.queue = queue
787 self.server = server
791 self.server = server
788 self.addRunOperation(**kwargs)
792 self.addRunOperation(**kwargs)
789
793
790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
794 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
791
795
792 #Compatible with old signal chain version
796 #Compatible with old signal chain version
793 if datatype==None and name==None:
797 if datatype==None and name==None:
794 raise ValueError, "datatype or name should be defined"
798 raise ValueError, "datatype or name should be defined"
795
799
796 if name==None:
800 if name==None:
797 if 'Reader' in datatype:
801 if 'Reader' in datatype:
798 name = datatype
802 name = datatype
799 else:
803 else:
800 name = '%sReader' %(datatype)
804 name = '%sReader' %(datatype)
801
805
802 if datatype==None:
806 if datatype==None:
803 datatype = name.replace('Reader','')
807 datatype = name.replace('Reader','')
804
808
805 self.datatype = datatype
809 self.datatype = datatype
806 self.name = name
810 self.name = name
807 self.path = path
811 self.path = path
808 self.startDate = startDate
812 self.startDate = startDate
809 self.endDate = endDate
813 self.endDate = endDate
810 self.startTime = startTime
814 self.startTime = startTime
811 self.endTime = endTime
815 self.endTime = endTime
812
816
813 self.inputId = '0'
817 self.inputId = '0'
814 self.parentId = parentId
818 self.parentId = parentId
815
819
816 self.updateRunOperation(**kwargs)
820 self.updateRunOperation(**kwargs)
817
821
818 def removeOperations(self):
822 def removeOperations(self):
819
823
820 for obj in self.opConfObjList:
824 for obj in self.opConfObjList:
821 del obj
825 del obj
822
826
823 self.opConfObjList = []
827 self.opConfObjList = []
824
828
825 def addRunOperation(self, **kwargs):
829 def addRunOperation(self, **kwargs):
826
830
827 opObj = self.addOperation(name = 'run', optype = 'self')
831 opObj = self.addOperation(name = 'run', optype = 'self')
828
832
829 if self.server is None:
833 if self.server is None:
830 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
834 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
831 opObj.addParameter(name='path' , value=self.path, format='str')
835 opObj.addParameter(name='path' , value=self.path, format='str')
832 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
836 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
833 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
837 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
834 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
838 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
835 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
839 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
836 opObj.addParameter(name='queue' , value=self.queue, format='obj')
840 opObj.addParameter(name='queue' , value=self.queue, format='obj')
837 for key, value in kwargs.items():
841 for key, value in kwargs.items():
838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
839 else:
843 else:
840 opObj.addParameter(name='server' , value=self.server, format='str')
844 opObj.addParameter(name='server' , value=self.server, format='str')
841
845
842
846
843 return opObj
847 return opObj
844
848
845 def updateRunOperation(self, **kwargs):
849 def updateRunOperation(self, **kwargs):
846
850
847 opObj = self.getOperationObj(name = 'run')
851 opObj = self.getOperationObj(name = 'run')
848 opObj.removeParameters()
852 opObj.removeParameters()
849
853
850 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
854 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
851 opObj.addParameter(name='path' , value=self.path, format='str')
855 opObj.addParameter(name='path' , value=self.path, format='str')
852 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
856 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
853 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
857 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
854 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
858 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
855 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
859 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
856
860
857 for key, value in kwargs.items():
861 for key, value in kwargs.items():
858 opObj.addParameter(name=key, value=value, format=type(value).__name__)
862 opObj.addParameter(name=key, value=value, format=type(value).__name__)
859
863
860 return opObj
864 return opObj
861
865
862 # def makeXml(self, projectElement):
866 # def makeXml(self, projectElement):
863 #
867 #
864 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
868 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
865 # procUnitElement.set('id', str(self.id))
869 # procUnitElement.set('id', str(self.id))
866 # procUnitElement.set('name', self.name)
870 # procUnitElement.set('name', self.name)
867 # procUnitElement.set('datatype', self.datatype)
871 # procUnitElement.set('datatype', self.datatype)
868 # procUnitElement.set('inputId', str(self.inputId))
872 # procUnitElement.set('inputId', str(self.inputId))
869 #
873 #
870 # for opConfObj in self.opConfObjList:
874 # for opConfObj in self.opConfObjList:
871 # opConfObj.makeXml(procUnitElement)
875 # opConfObj.makeXml(procUnitElement)
872
876
873 def readXml(self, upElement):
877 def readXml(self, upElement):
874
878
875 self.id = upElement.get('id')
879 self.id = upElement.get('id')
876 self.name = upElement.get('name')
880 self.name = upElement.get('name')
877 self.datatype = upElement.get('datatype')
881 self.datatype = upElement.get('datatype')
878 self.inputId = upElement.get('inputId')
882 self.inputId = upElement.get('inputId')
879
883
880 if self.ELEMENTNAME == "ReadUnit":
884 if self.ELEMENTNAME == "ReadUnit":
881 self.datatype = self.datatype.replace("Reader", "")
885 self.datatype = self.datatype.replace("Reader", "")
882
886
883 if self.inputId == 'None':
887 if self.inputId == 'None':
884 self.inputId = '0'
888 self.inputId = '0'
885
889
886 self.opConfObjList = []
890 self.opConfObjList = []
887
891
888 opElementList = upElement.iter(OperationConf().getElementName())
892 opElementList = upElement.iter(OperationConf().getElementName())
889
893
890 for opElement in opElementList:
894 for opElement in opElementList:
891 opConfObj = OperationConf()
895 opConfObj = OperationConf()
892 opConfObj.readXml(opElement)
896 opConfObj.readXml(opElement)
893 self.opConfObjList.append(opConfObj)
897 self.opConfObjList.append(opConfObj)
894
898
895 if opConfObj.name == 'run':
899 if opConfObj.name == 'run':
896 self.path = opConfObj.getParameterValue('path')
900 self.path = opConfObj.getParameterValue('path')
897 self.startDate = opConfObj.getParameterValue('startDate')
901 self.startDate = opConfObj.getParameterValue('startDate')
898 self.endDate = opConfObj.getParameterValue('endDate')
902 self.endDate = opConfObj.getParameterValue('endDate')
899 self.startTime = opConfObj.getParameterValue('startTime')
903 self.startTime = opConfObj.getParameterValue('startTime')
900 self.endTime = opConfObj.getParameterValue('endTime')
904 self.endTime = opConfObj.getParameterValue('endTime')
901
905
902 class Project():
906 class Project():
903
907
904 id = None
908 id = None
905 name = None
909 name = None
906 description = None
910 description = None
907 filename = None
911 filename = None
908
912
909 procUnitConfObjDict = None
913 procUnitConfObjDict = None
910
914
911 ELEMENTNAME = 'Project'
915 ELEMENTNAME = 'Project'
912
916
913 plotterQueue = None
917 plotterQueue = None
914
918
915 def __init__(self, plotter_queue=None):
919 def __init__(self, plotter_queue=None):
916
920
917 self.id = None
921 self.id = None
918 self.name = None
922 self.name = None
919 self.description = None
923 self.description = None
920
924
921 self.plotterQueue = plotter_queue
925 self.plotterQueue = plotter_queue
922
926
923 self.procUnitConfObjDict = {}
927 self.procUnitConfObjDict = {}
924
928
925 def __getNewId(self):
929 def __getNewId(self):
926
930
927 idList = self.procUnitConfObjDict.keys()
931 idList = self.procUnitConfObjDict.keys()
928
932
929 id = int(self.id)*10
933 id = int(self.id)*10
930
934
931 while True:
935 while True:
932 id += 1
936 id += 1
933
937
934 if str(id) in idList:
938 if str(id) in idList:
935 continue
939 continue
936
940
937 break
941 break
938
942
939 return str(id)
943 return str(id)
940
944
941 def getElementName(self):
945 def getElementName(self):
942
946
943 return self.ELEMENTNAME
947 return self.ELEMENTNAME
944
948
945 def getId(self):
949 def getId(self):
946
950
947 return self.id
951 return self.id
948
952
949 def updateId(self, new_id):
953 def updateId(self, new_id):
950
954
951 self.id = str(new_id)
955 self.id = str(new_id)
952
956
953 keyList = self.procUnitConfObjDict.keys()
957 keyList = self.procUnitConfObjDict.keys()
954 keyList.sort()
958 keyList.sort()
955
959
956 n = 1
960 n = 1
957 newProcUnitConfObjDict = {}
961 newProcUnitConfObjDict = {}
958
962
959 for procKey in keyList:
963 for procKey in keyList:
960
964
961 procUnitConfObj = self.procUnitConfObjDict[procKey]
965 procUnitConfObj = self.procUnitConfObjDict[procKey]
962 idProcUnit = str(int(self.id)*10 + n)
966 idProcUnit = str(int(self.id)*10 + n)
963 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
967 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
964
968
965 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
969 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
966 n += 1
970 n += 1
967
971
968 self.procUnitConfObjDict = newProcUnitConfObjDict
972 self.procUnitConfObjDict = newProcUnitConfObjDict
969
973
970 def setup(self, id, name, description):
974 def setup(self, id, name, description):
971
975
972 self.id = str(id)
976 self.id = str(id)
973 self.name = name
977 self.name = name
974 self.description = description
978 self.description = description
975
979
976 def update(self, name, description):
980 def update(self, name, description):
977
981
978 self.name = name
982 self.name = name
979 self.description = description
983 self.description = description
980
984
981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
985 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
982
986
983 if id is None:
987 if id is None:
984 idReadUnit = self.__getNewId()
988 idReadUnit = self.__getNewId()
985 else:
989 else:
986 idReadUnit = str(id)
990 idReadUnit = str(id)
987
991
988 readUnitConfObj = ReadUnitConf()
992 readUnitConfObj = ReadUnitConf()
989 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
993 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
990
994
991 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
995 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
992
996
993 return readUnitConfObj
997 return readUnitConfObj
994
998
995 def addProcUnit(self, inputId='0', datatype=None, name=None):
999 def addProcUnit(self, inputId='0', datatype=None, name=None):
996
1000
997 idProcUnit = self.__getNewId()
1001 idProcUnit = self.__getNewId()
998
1002
999 procUnitConfObj = ProcUnitConf()
1003 procUnitConfObj = ProcUnitConf()
1000 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1004 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1001
1005
1002 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1003
1007
1004 return procUnitConfObj
1008 return procUnitConfObj
1005
1009
1006 def removeProcUnit(self, id):
1010 def removeProcUnit(self, id):
1007
1011
1008 if id in self.procUnitConfObjDict.keys():
1012 if id in self.procUnitConfObjDict.keys():
1009 self.procUnitConfObjDict.pop(id)
1013 self.procUnitConfObjDict.pop(id)
1010
1014
1011 def getReadUnitId(self):
1015 def getReadUnitId(self):
1012
1016
1013 readUnitConfObj = self.getReadUnitObj()
1017 readUnitConfObj = self.getReadUnitObj()
1014
1018
1015 return readUnitConfObj.id
1019 return readUnitConfObj.id
1016
1020
1017 def getReadUnitObj(self):
1021 def getReadUnitObj(self):
1018
1022
1019 for obj in self.procUnitConfObjDict.values():
1023 for obj in self.procUnitConfObjDict.values():
1020 if obj.getElementName() == "ReadUnit":
1024 if obj.getElementName() == "ReadUnit":
1021 return obj
1025 return obj
1022
1026
1023 return None
1027 return None
1024
1028
1025 def getProcUnitObj(self, id=None, name=None):
1029 def getProcUnitObj(self, id=None, name=None):
1026
1030
1027 if id != None:
1031 if id != None:
1028 return self.procUnitConfObjDict[id]
1032 return self.procUnitConfObjDict[id]
1029
1033
1030 if name != None:
1034 if name != None:
1031 return self.getProcUnitObjByName(name)
1035 return self.getProcUnitObjByName(name)
1032
1036
1033 return None
1037 return None
1034
1038
1035 def getProcUnitObjByName(self, name):
1039 def getProcUnitObjByName(self, name):
1036
1040
1037 for obj in self.procUnitConfObjDict.values():
1041 for obj in self.procUnitConfObjDict.values():
1038 if obj.name == name:
1042 if obj.name == name:
1039 return obj
1043 return obj
1040
1044
1041 return None
1045 return None
1042
1046
1043 def procUnitItems(self):
1047 def procUnitItems(self):
1044
1048
1045 return self.procUnitConfObjDict.items()
1049 return self.procUnitConfObjDict.items()
1046
1050
1047 def makeXml(self):
1051 def makeXml(self):
1048
1052
1049 projectElement = Element('Project')
1053 projectElement = Element('Project')
1050 projectElement.set('id', str(self.id))
1054 projectElement.set('id', str(self.id))
1051 projectElement.set('name', self.name)
1055 projectElement.set('name', self.name)
1052 projectElement.set('description', self.description)
1056 projectElement.set('description', self.description)
1053
1057
1054 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 for procUnitConfObj in self.procUnitConfObjDict.values():
1055 procUnitConfObj.makeXml(projectElement)
1059 procUnitConfObj.makeXml(projectElement)
1056
1060
1057 self.projectElement = projectElement
1061 self.projectElement = projectElement
1058
1062
1059 def writeXml(self, filename=None):
1063 def writeXml(self, filename=None):
1060
1064
1061 if filename == None:
1065 if filename == None:
1062 if self.filename:
1066 if self.filename:
1063 filename = self.filename
1067 filename = self.filename
1064 else:
1068 else:
1065 filename = "schain.xml"
1069 filename = "schain.xml"
1066
1070
1067 if not filename:
1071 if not filename:
1068 print "filename has not been defined. Use setFilename(filename) for do it."
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1069 return 0
1073 return 0
1070
1074
1071 abs_file = os.path.abspath(filename)
1075 abs_file = os.path.abspath(filename)
1072
1076
1073 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 if not os.access(os.path.dirname(abs_file), os.W_OK):
1074 print "No write permission on %s" %os.path.dirname(abs_file)
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1075 return 0
1079 return 0
1076
1080
1077 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1078 print "File %s already exists and it could not be overwriten" %abs_file
1082 print "File %s already exists and it could not be overwriten" %abs_file
1079 return 0
1083 return 0
1080
1084
1081 self.makeXml()
1085 self.makeXml()
1082
1086
1083 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 ElementTree(self.projectElement).write(abs_file, method='xml')
1084
1088
1085 self.filename = abs_file
1089 self.filename = abs_file
1086
1090
1087 return 1
1091 return 1
1088
1092
1089 def readXml(self, filename = None):
1093 def readXml(self, filename = None):
1090
1094
1091 if not filename:
1095 if not filename:
1092 print "filename is not defined"
1096 print "filename is not defined"
1093 return 0
1097 return 0
1094
1098
1095 abs_file = os.path.abspath(filename)
1099 abs_file = os.path.abspath(filename)
1096
1100
1097 if not os.path.isfile(abs_file):
1101 if not os.path.isfile(abs_file):
1098 print "%s file does not exist" %abs_file
1102 print "%s file does not exist" %abs_file
1099 return 0
1103 return 0
1100
1104
1101 self.projectElement = None
1105 self.projectElement = None
1102 self.procUnitConfObjDict = {}
1106 self.procUnitConfObjDict = {}
1103
1107
1104 try:
1108 try:
1105 self.projectElement = ElementTree().parse(abs_file)
1109 self.projectElement = ElementTree().parse(abs_file)
1106 except:
1110 except:
1107 print "Error reading %s, verify file format" %filename
1111 print "Error reading %s, verify file format" %filename
1108 return 0
1112 return 0
1109
1113
1110 self.project = self.projectElement.tag
1114 self.project = self.projectElement.tag
1111
1115
1112 self.id = self.projectElement.get('id')
1116 self.id = self.projectElement.get('id')
1113 self.name = self.projectElement.get('name')
1117 self.name = self.projectElement.get('name')
1114 self.description = self.projectElement.get('description')
1118 self.description = self.projectElement.get('description')
1115
1119
1116 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1120 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1117
1121
1118 for readUnitElement in readUnitElementList:
1122 for readUnitElement in readUnitElementList:
1119 readUnitConfObj = ReadUnitConf()
1123 readUnitConfObj = ReadUnitConf()
1120 readUnitConfObj.readXml(readUnitElement)
1124 readUnitConfObj.readXml(readUnitElement)
1121
1125
1122 if readUnitConfObj.parentId == None:
1126 if readUnitConfObj.parentId == None:
1123 readUnitConfObj.parentId = self.id
1127 readUnitConfObj.parentId = self.id
1124
1128
1125 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1126
1130
1127 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1131 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1128
1132
1129 for procUnitElement in procUnitElementList:
1133 for procUnitElement in procUnitElementList:
1130 procUnitConfObj = ProcUnitConf()
1134 procUnitConfObj = ProcUnitConf()
1131 procUnitConfObj.readXml(procUnitElement)
1135 procUnitConfObj.readXml(procUnitElement)
1132
1136
1133 if procUnitConfObj.parentId == None:
1137 if procUnitConfObj.parentId == None:
1134 procUnitConfObj.parentId = self.id
1138 procUnitConfObj.parentId = self.id
1135
1139
1136 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1137
1141
1138 self.filename = abs_file
1142 self.filename = abs_file
1139
1143
1140 return 1
1144 return 1
1141
1145
1142 def printattr(self):
1146 def printattr(self):
1143
1147
1144 print "Project[%s]: name = %s, description = %s" %(self.id,
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1145 self.name,
1149 self.name,
1146 self.description)
1150 self.description)
1147
1151
1148 for procUnitConfObj in self.procUnitConfObjDict.values():
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1149 procUnitConfObj.printattr()
1153 procUnitConfObj.printattr()
1150
1154
1151 def createObjects(self):
1155 def createObjects(self):
1152
1156
1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 procUnitConfObj.createObjects(self.plotterQueue)
1158 procUnitConfObj.createObjects(self.plotterQueue)
1155
1159
1156 def __connect(self, objIN, thisObj):
1160 def __connect(self, objIN, thisObj):
1157
1161
1158 thisObj.setInput(objIN.getOutputObj())
1162 thisObj.setInput(objIN.getOutputObj())
1159
1163
1160 def connectObjects(self):
1164 def connectObjects(self):
1161
1165
1162 for thisPUConfObj in self.procUnitConfObjDict.values():
1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1163
1167
1164 inputId = thisPUConfObj.getInputId()
1168 inputId = thisPUConfObj.getInputId()
1165
1169
1166 if int(inputId) == 0:
1170 if int(inputId) == 0:
1167 continue
1171 continue
1168
1172
1169 #Get input object
1173 #Get input object
1170 puConfINObj = self.procUnitConfObjDict[inputId]
1174 puConfINObj = self.procUnitConfObjDict[inputId]
1171 puObjIN = puConfINObj.getProcUnitObj()
1175 puObjIN = puConfINObj.getProcUnitObj()
1172
1176
1173 #Get current object
1177 #Get current object
1174 thisPUObj = thisPUConfObj.getProcUnitObj()
1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1175
1179
1176 self.__connect(puObjIN, thisPUObj)
1180 self.__connect(puObjIN, thisPUObj)
1177
1181
1178 def __handleError(self, procUnitConfObj, send_email=True):
1182 def __handleError(self, procUnitConfObj, send_email=True):
1179
1183
1180 import socket
1184 import socket
1181
1185
1182 err = traceback.format_exception(sys.exc_info()[0],
1186 err = traceback.format_exception(sys.exc_info()[0],
1183 sys.exc_info()[1],
1187 sys.exc_info()[1],
1184 sys.exc_info()[2])
1188 sys.exc_info()[2])
1185
1189
1186 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1187 print "***** %s" %err[-1]
1191 print "***** %s" %err[-1]
1188
1192
1189 message = "".join(err)
1193 message = "".join(err)
1190
1194
1191 sys.stderr.write(message)
1195 sys.stderr.write(message)
1192
1196
1193 if not send_email:
1197 if not send_email:
1194 return
1198 return
1195
1199
1196 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1197
1201
1198 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1199 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1200 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1201 subtitle += "Configuration file: %s\n" %self.filename
1205 subtitle += "Configuration file: %s\n" %self.filename
1202 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1203
1207
1204 readUnitConfObj = self.getReadUnitObj()
1208 readUnitConfObj = self.getReadUnitObj()
1205 if readUnitConfObj:
1209 if readUnitConfObj:
1206 subtitle += "\nInput parameters:\n"
1210 subtitle += "\nInput parameters:\n"
1207 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1208 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1209 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1210 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1211 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1212 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1213
1217
1214 adminObj = schainpy.admin.SchainNotify()
1218 adminObj = schainpy.admin.SchainNotify()
1215 adminObj.sendAlert(message=message,
1219 adminObj.sendAlert(message=message,
1216 subject=subject,
1220 subject=subject,
1217 subtitle=subtitle,
1221 subtitle=subtitle,
1218 filename=self.filename)
1222 filename=self.filename)
1219
1223
1220 def isPaused(self):
1224 def isPaused(self):
1221 return 0
1225 return 0
1222
1226
1223 def isStopped(self):
1227 def isStopped(self):
1224 return 0
1228 return 0
1225
1229
1226 def runController(self):
1230 def runController(self):
1227 """
1231 """
1228 returns 0 when this process has been stopped, 1 otherwise
1232 returns 0 when this process has been stopped, 1 otherwise
1229 """
1233 """
1230
1234
1231 if self.isPaused():
1235 if self.isPaused():
1232 print "Process suspended"
1236 print "Process suspended"
1233
1237
1234 while True:
1238 while True:
1235 sleep(0.1)
1239 sleep(0.1)
1236
1240
1237 if not self.isPaused():
1241 if not self.isPaused():
1238 break
1242 break
1239
1243
1240 if self.isStopped():
1244 if self.isStopped():
1241 break
1245 break
1242
1246
1243 print "Process reinitialized"
1247 print "Process reinitialized"
1244
1248
1245 if self.isStopped():
1249 if self.isStopped():
1246 print "Process stopped"
1250 print "Process stopped"
1247 return 0
1251 return 0
1248
1252
1249 return 1
1253 return 1
1250
1254
1251 def setFilename(self, filename):
1255 def setFilename(self, filename):
1252
1256
1253 self.filename = filename
1257 self.filename = filename
1254
1258
1255 def setPlotterQueue(self, plotter_queue):
1259 def setPlotterQueue(self, plotter_queue):
1256
1260
1257 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1258
1262
1259 def getPlotterQueue(self):
1263 def getPlotterQueue(self):
1260
1264
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1262
1266
1263 def useExternalPlotter(self):
1267 def useExternalPlotter(self):
1264
1268
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1266
1270
1271
1267 def run(self):
1272 def run(self):
1268
1273
1269 print
1274 print
1270 print "*"*60
1275 print "*"*60
1271 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1276 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1272 print "*"*60
1277 print "*"*60
1273 print
1278 print
1274
1279
1275 keyList = self.procUnitConfObjDict.keys()
1280 keyList = self.procUnitConfObjDict.keys()
1276 keyList.sort()
1281 keyList.sort()
1277
1282
1278 while(True):
1283 while(True):
1279
1284
1280 is_ok = False
1285 is_ok = False
1281
1286
1282 for procKey in keyList:
1287 for procKey in keyList:
1283 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1288 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1284
1289
1285 procUnitConfObj = self.procUnitConfObjDict[procKey]
1290 procUnitConfObj = self.procUnitConfObjDict[procKey]
1286
1291
1287 try:
1292 try:
1288 sts = procUnitConfObj.run()
1293 sts = procUnitConfObj.run()
1289 is_ok = is_ok or sts
1294 is_ok = is_ok or sts
1290 except KeyboardInterrupt:
1295 except KeyboardInterrupt:
1291 is_ok = False
1296 is_ok = False
1292 break
1297 break
1293 except ValueError, e:
1298 except ValueError, e:
1294 sleep(0.5)
1299 sleep(0.5)
1295 self.__handleError(procUnitConfObj, send_email=True)
1300 self.__handleError(procUnitConfObj, send_email=True)
1296 is_ok = False
1301 is_ok = False
1297 break
1302 break
1298 except:
1303 except:
1299 sleep(0.5)
1304 sleep(0.5)
1300 self.__handleError(procUnitConfObj)
1305 self.__handleError(procUnitConfObj)
1301 is_ok = False
1306 is_ok = False
1302 break
1307 break
1303
1308
1304 #If every process unit finished so end process
1309 #If every process unit finished so end process
1305 if not(is_ok):
1310 if not(is_ok):
1306 # print "Every process unit have finished"
1311 # print "Every process unit have finished"
1307 break
1312 break
1308
1313
1309 if not self.runController():
1314 if not self.runController():
1310 break
1315 break
1311
1316
1312 #Closing every process
1317 #Closing every process
1313 for procKey in keyList:
1318 for procKey in keyList:
1314 procUnitConfObj = self.procUnitConfObjDict[procKey]
1319 procUnitConfObj = self.procUnitConfObjDict[procKey]
1315 procUnitConfObj.close()
1320 procUnitConfObj.close()
1316
1321
1317 print "Process finished"
1322 print "Process finished"
1318
1323
1319 def start(self, filename=None):
1324 def start(self, filename=None):
1320
1325
1321 self.writeXml(filename)
1326 self.writeXml(filename)
1322 self.createObjects()
1327 self.createObjects()
1323 self.connectObjects()
1328 self.connectObjects()
1324 self.run()
1329 self.run()
Show file before | Show file after | Hide comments
@@ -1,12 +1,12
1 #from schainpy.model.data.jrodata import *
1 #from schainpy.model.data.jrodata import *
2 # from schainpy.model.io.jrodataIO import *
2 # from schainpy.model.io.jrodataIO import *
3 # from schainpy.model.proc.jroprocessing import *
3 # from schainpy.model.proc.jroprocessing import *
4 # from schainpy.model.graphics.jroplot import *
4 # from schainpy.model.graphics.jroplot import *
5 # from schainpy.model.utils.jroutils import *
5 # from schainpy.model.utils.jroutils import *
6 # from schainpy.serializer import *
6 # from schainpy.serializer import *
7
7
8 from data import *
8 from data import *
9 from io import *
9 from io import *
10 from proc import *
10 from proc import *
11 from graphics import *
11 #from graphics import *
12 from utils import *
12 from utils import *
Show file before | Show file after | Hide comments
@@ -1,775 +1,783
1
1
2 import os
2 import os
3 import zmq
3 import zmq
4 import time
4 import time
5 import numpy
5 import numpy
6 import datetime
6 import datetime
7 import numpy as np
7 import numpy as np
8 import matplotlib
8 import matplotlib
9 matplotlib.use('TkAgg')
9 matplotlib.use('TkAgg')
10 import matplotlib.pyplot as plt
10 import matplotlib.pyplot as plt
11 from mpl_toolkits.axes_grid1 import make_axes_locatable
11 from mpl_toolkits.axes_grid1 import make_axes_locatable
12 from matplotlib.ticker import FuncFormatter, LinearLocator
12 from matplotlib.ticker import FuncFormatter, LinearLocator
13 from multiprocessing import Process
13 from multiprocessing import Process
14
14
15 from schainpy.model.proc.jroproc_base import Operation
15 from schainpy.model.proc.jroproc_base import Operation
16
16
17 plt.ion()
17 plt.ion()
18
18
19 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
19 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
20 fromtimestamp = lambda x, mintime : (datetime.datetime.utcfromtimestamp(mintime).replace(hour=(x + 5), minute=0) - d1970).total_seconds()
21
20
22
21 d1970 = datetime.datetime(1970,1,1)
23 d1970 = datetime.datetime(1970,1,1)
22
24
23 class PlotData(Operation, Process):
25 class PlotData(Operation, Process):
24
26
25 CODE = 'Figure'
27 CODE = 'Figure'
26 colormap = 'jro'
28 colormap = 'jro'
27 CONFLATE = False
29 CONFLATE = False
28 __MAXNUMX = 80
30 __MAXNUMX = 80
29 __missing = 1E30
31 __missing = 1E30
30
32
31 def __init__(self, **kwargs):
33 def __init__(self, **kwargs):
32
34
33 Operation.__init__(self, plot=True, **kwargs)
35 Operation.__init__(self, plot=True, **kwargs)
34 Process.__init__(self)
36 Process.__init__(self)
35 self.kwargs['code'] = self.CODE
37 self.kwargs['code'] = self.CODE
36 self.mp = False
38 self.mp = False
37 self.dataOut = None
39 self.dataOut = None
38 self.isConfig = False
40 self.isConfig = False
39 self.figure = None
41 self.figure = None
40 self.axes = []
42 self.axes = []
41 self.localtime = kwargs.pop('localtime', True)
43 self.localtime = kwargs.pop('localtime', True)
42 self.show = kwargs.get('show', True)
44 self.show = kwargs.get('show', True)
43 self.save = kwargs.get('save', False)
45 self.save = kwargs.get('save', False)
44 self.colormap = kwargs.get('colormap', self.colormap)
46 self.colormap = kwargs.get('colormap', self.colormap)
45 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
47 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
46 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
48 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
47 self.showprofile = kwargs.get('showprofile', True)
49 self.showprofile = kwargs.get('showprofile', True)
48 self.title = kwargs.get('wintitle', '')
50 self.title = kwargs.get('wintitle', '')
49 self.xaxis = kwargs.get('xaxis', 'frequency')
51 self.xaxis = kwargs.get('xaxis', 'frequency')
50 self.zmin = kwargs.get('zmin', None)
52 self.zmin = kwargs.get('zmin', None)
51 self.zmax = kwargs.get('zmax', None)
53 self.zmax = kwargs.get('zmax', None)
52 self.xmin = kwargs.get('xmin', None)
54 self.xmin = kwargs.get('xmin', None)
53 self.xmax = kwargs.get('xmax', None)
55 self.xmax = kwargs.get('xmax', None)
54 self.xrange = kwargs.get('xrange', 24)
56 self.xrange = kwargs.get('xrange', 24)
55 self.ymin = kwargs.get('ymin', None)
57 self.ymin = kwargs.get('ymin', None)
56 self.ymax = kwargs.get('ymax', None)
58 self.ymax = kwargs.get('ymax', None)
57 self.__MAXNUMY = kwargs.get('decimation', 80)
59 self.__MAXNUMY = kwargs.get('decimation', 5000)
58 self.throttle_value = 5
60 self.throttle_value = 5
59 self.times = []
61 self.times = []
60 #self.interactive = self.kwargs['parent']
62 #self.interactive = self.kwargs['parent']
61
63
62
64
63 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
65 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
64
66
65 if x_buffer.shape[0] < 2:
67 if x_buffer.shape[0] < 2:
66 return x_buffer, y_buffer, z_buffer
68 return x_buffer, y_buffer, z_buffer
67
69
68 deltas = x_buffer[1:] - x_buffer[0:-1]
70 deltas = x_buffer[1:] - x_buffer[0:-1]
69 x_median = np.median(deltas)
71 x_median = np.median(deltas)
70
72
71 index = np.where(deltas > 5*x_median)
73 index = np.where(deltas > 5*x_median)
72
74
73 if len(index[0]) != 0:
75 if len(index[0]) != 0:
74 z_buffer[::, index[0], ::] = self.__missing
76 z_buffer[::, index[0], ::] = self.__missing
75 z_buffer = np.ma.masked_inside(z_buffer,
77 z_buffer = np.ma.masked_inside(z_buffer,
76 0.99*self.__missing,
78 0.99*self.__missing,
77 1.01*self.__missing)
79 1.01*self.__missing)
78
80
79 return x_buffer, y_buffer, z_buffer
81 return x_buffer, y_buffer, z_buffer
80
82
81 def decimate(self):
83 def decimate(self):
82
84
83 # dx = int(len(self.x)/self.__MAXNUMX) + 1
85 # dx = int(len(self.x)/self.__MAXNUMX) + 1
84 dy = int(len(self.y)/self.__MAXNUMY) + 1
86 dy = int(len(self.y)/self.__MAXNUMY) + 1
85
87
86 # x = self.x[::dx]
88 # x = self.x[::dx]
87 x = self.x
89 x = self.x
88 y = self.y[::dy]
90 y = self.y[::dy]
89 z = self.z[::, ::, ::dy]
91 z = self.z[::, ::, ::dy]
90
92
91 return x, y, z
93 return x, y, z
92
94
93 def __plot(self):
95 def __plot(self):
94
96
95 print 'plotting...{}'.format(self.CODE)
97 print 'plotting...{}'.format(self.CODE)
96
98
97 if self.show:
99 if self.show:
98 self.figure.show()
100 self.figure.show()
99
101
100 self.plot()
102 self.plot()
101 plt.tight_layout()
103 plt.tight_layout()
102 self.figure.canvas.manager.set_window_title('{} {} - {}'.format(self.title, self.CODE.upper(),
104 self.figure.canvas.manager.set_window_title('{} {} - {}'.format(self.title, self.CODE.upper(),
103 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
105 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
104
106
105 if self.save:
107 if self.save:
106 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
108 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
107 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
109 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
108 print 'Saving figure: {}'.format(figname)
110 print 'Saving figure: {}'.format(figname)
109 self.figure.savefig(figname)
111 self.figure.savefig(figname)
110
112
111 self.figure.canvas.draw()
113 self.figure.canvas.draw()
112
114
113 def plot(self):
115 def plot(self):
114
116
115 print 'plotting...{}'.format(self.CODE.upper())
117 print 'plotting...{}'.format(self.CODE.upper())
116 return
118 return
117
119
118 def run(self):
120 def run(self):
119
121
120 print '[Starting] {}'.format(self.name)
122 print '[Starting] {}'.format(self.name)
121
123
122 context = zmq.Context()
124 context = zmq.Context()
123 receiver = context.socket(zmq.SUB)
125 receiver = context.socket(zmq.SUB)
124 receiver.setsockopt(zmq.SUBSCRIBE, '')
126 receiver.setsockopt(zmq.SUBSCRIBE, '')
125 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
127 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
126
128
127 if 'server' in self.kwargs['parent']:
129 if 'server' in self.kwargs['parent']:
128 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
130 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
129 else:
131 else:
130 receiver.connect("ipc:///tmp/zmq.plots")
132 receiver.connect("ipc:///tmp/zmq.plots")
131
133
132 seconds_passed = 0
134 seconds_passed = 0
133
135
134 while True:
136 while True:
135 try:
137 try:
136 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK
138 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK
137 self.started = self.data['STARTED']
139 self.started = self.data['STARTED']
138 self.dataOut = self.data['dataOut']
140 self.dataOut = self.data['dataOut']
139
141
140 if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']):
142 if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']):
141 continue
143 continue
142
144
143 self.times = self.data['times']
145 self.times = self.data['times']
144 self.times.sort()
146 self.times.sort()
145 self.throttle_value = self.data['throttle']
147 self.throttle_value = self.data['throttle']
146 self.min_time = self.times[0]
148 self.min_time = self.times[0]
147 self.max_time = self.times[-1]
149 self.max_time = self.times[-1]
148
150
149 if self.isConfig is False:
151 if self.isConfig is False:
150 print 'setting up'
152 print 'setting up'
151 self.setup()
153 self.setup()
152 self.isConfig = True
154 self.isConfig = True
153 self.__plot()
155 self.__plot()
154
156
155 if self.data['ENDED'] is True:
157 if self.data['ENDED'] is True:
156 print '********GRAPHIC ENDED********'
158 print '********GRAPHIC ENDED********'
157 self.ended = True
159 self.ended = True
158 self.isConfig = False
160 self.isConfig = False
159 self.__plot()
161 self.__plot()
160 elif seconds_passed >= self.data['throttle']:
162 elif seconds_passed >= self.data['throttle']:
161 print 'passed', seconds_passed
163 print 'passed', seconds_passed
162 self.__plot()
164 self.__plot()
163 seconds_passed = 0
165 seconds_passed = 0
164
166
165 except zmq.Again as e:
167 except zmq.Again as e:
166 print 'Waiting for data...'
168 print 'Waiting for data...'
167 plt.pause(2)
169 plt.pause(2)
168 seconds_passed += 2
170 seconds_passed += 2
169
171
170 def close(self):
172 def close(self):
171 if self.dataOut:
173 if self.dataOut:
172 self.__plot()
174 self.__plot()
173
175
174
176
175 class PlotSpectraData(PlotData):
177 class PlotSpectraData(PlotData):
176
178
177 CODE = 'spc'
179 CODE = 'spc'
178 colormap = 'jro'
180 colormap = 'jro'
179 CONFLATE = False
181 CONFLATE = False
180
182
181 def setup(self):
183 def setup(self):
182
184
183 ncolspan = 1
185 ncolspan = 1
184 colspan = 1
186 colspan = 1
185 self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
187 self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
186 self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
188 self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
187 self.width = 3.6*self.ncols
189 self.width = 3.6*self.ncols
188 self.height = 3.2*self.nrows
190 self.height = 3.2*self.nrows
189 if self.showprofile:
191 if self.showprofile:
190 ncolspan = 3
192 ncolspan = 3
191 colspan = 2
193 colspan = 2
192 self.width += 1.2*self.ncols
194 self.width += 1.2*self.ncols
193
195
194 self.ylabel = 'Range [Km]'
196 self.ylabel = 'Range [Km]'
195 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
197 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
196
198
197 if self.figure is None:
199 if self.figure is None:
198 self.figure = plt.figure(figsize=(self.width, self.height),
200 self.figure = plt.figure(figsize=(self.width, self.height),
199 edgecolor='k',
201 edgecolor='k',
200 facecolor='w')
202 facecolor='w')
201 else:
203 else:
202 self.figure.clf()
204 self.figure.clf()
203
205
204 n = 0
206 n = 0
205 for y in range(self.nrows):
207 for y in range(self.nrows):
206 for x in range(self.ncols):
208 for x in range(self.ncols):
207 if n >= self.dataOut.nChannels:
209 if n >= self.dataOut.nChannels:
208 break
210 break
209 ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
211 ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
210 if self.showprofile:
212 if self.showprofile:
211 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
213 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
212
214
213 ax.firsttime = True
215 ax.firsttime = True
214 self.axes.append(ax)
216 self.axes.append(ax)
215 n += 1
217 n += 1
216
218
217 def plot(self):
219 def plot(self):
218
220
219 if self.xaxis == "frequency":
221 if self.xaxis == "frequency":
220 x = self.dataOut.getFreqRange(1)/1000.
222 x = self.dataOut.getFreqRange(1)/1000.
221 xlabel = "Frequency (kHz)"
223 xlabel = "Frequency (kHz)"
222 elif self.xaxis == "time":
224 elif self.xaxis == "time":
223 x = self.dataOut.getAcfRange(1)
225 x = self.dataOut.getAcfRange(1)
224 xlabel = "Time (ms)"
226 xlabel = "Time (ms)"
225 else:
227 else:
226 x = self.dataOut.getVelRange(1)
228 x = self.dataOut.getVelRange(1)
227 xlabel = "Velocity (m/s)"
229 xlabel = "Velocity (m/s)"
228
230
229 y = self.dataOut.getHeiRange()
231 y = self.dataOut.getHeiRange()
230 z = self.data[self.CODE]
232 z = self.data[self.CODE]
231
233
232 for n, ax in enumerate(self.axes):
234 for n, ax in enumerate(self.axes):
233
235
234 if ax.firsttime:
236 if ax.firsttime:
235 self.xmax = self.xmax if self.xmax else np.nanmax(x)
237 self.xmax = self.xmax if self.xmax else np.nanmax(x)
236 self.xmin = self.xmin if self.xmin else -self.xmax
238 self.xmin = self.xmin if self.xmin else -self.xmax
237 self.ymin = self.ymin if self.ymin else np.nanmin(y)
239 self.ymin = self.ymin if self.ymin else np.nanmin(y)
238 self.ymax = self.ymax if self.ymax else np.nanmax(y)
240 self.ymax = self.ymax if self.ymax else np.nanmax(y)
239 self.zmin = self.zmin if self.zmin else np.nanmin(z)
241 self.zmin = self.zmin if self.zmin else np.nanmin(z)
240 self.zmax = self.zmax if self.zmax else np.nanmax(z)
242 self.zmax = self.zmax if self.zmax else np.nanmax(z)
241 ax.plot = ax.pcolormesh(x, y, z[n].T,
243 ax.plot = ax.pcolormesh(x, y, z[n].T,
242 vmin=self.zmin,
244 vmin=self.zmin,
243 vmax=self.zmax,
245 vmax=self.zmax,
244 cmap=plt.get_cmap(self.colormap)
246 cmap=plt.get_cmap(self.colormap)
245 )
247 )
246 divider = make_axes_locatable(ax)
248 divider = make_axes_locatable(ax)
247 cax = divider.new_horizontal(size='3%', pad=0.05)
249 cax = divider.new_horizontal(size='3%', pad=0.05)
248 self.figure.add_axes(cax)
250 self.figure.add_axes(cax)
249 plt.colorbar(ax.plot, cax)
251 plt.colorbar(ax.plot, cax)
250
252
251 ax.set_xlim(self.xmin, self.xmax)
253 ax.set_xlim(self.xmin, self.xmax)
252 ax.set_ylim(self.ymin, self.ymax)
254 ax.set_ylim(self.ymin, self.ymax)
253
255
254 ax.set_ylabel(self.ylabel)
256 ax.set_ylabel(self.ylabel)
255 ax.set_xlabel(xlabel)
257 ax.set_xlabel(xlabel)
256
258
257 ax.firsttime = False
259 ax.firsttime = False
258
260
259 if self.showprofile:
261 if self.showprofile:
260 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
262 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
261 ax.ax_profile.set_xlim(self.zmin, self.zmax)
263 ax.ax_profile.set_xlim(self.zmin, self.zmax)
262 ax.ax_profile.set_ylim(self.ymin, self.ymax)
264 ax.ax_profile.set_ylim(self.ymin, self.ymax)
263 ax.ax_profile.set_xlabel('dB')
265 ax.ax_profile.set_xlabel('dB')
264 ax.ax_profile.grid(b=True, axis='x')
266 ax.ax_profile.grid(b=True, axis='x')
265 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
267 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
266 color="k", linestyle="dashed", lw=2)[0]
268 color="k", linestyle="dashed", lw=2)[0]
267 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
269 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
268 else:
270 else:
269 ax.plot.set_array(z[n].T.ravel())
271 ax.plot.set_array(z[n].T.ravel())
270 if self.showprofile:
272 if self.showprofile:
271 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
273 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
272 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
274 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
273
275
274 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
276 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
275 size=8)
277 size=8)
276 self.saveTime = self.max_time
278 self.saveTime = self.max_time
277
279
278
280
279 class PlotCrossSpectraData(PlotData):
281 class PlotCrossSpectraData(PlotData):
280
282
281 CODE = 'cspc'
283 CODE = 'cspc'
282 zmin_coh = None
284 zmin_coh = None
283 zmax_coh = None
285 zmax_coh = None
284 zmin_phase = None
286 zmin_phase = None
285 zmax_phase = None
287 zmax_phase = None
286 CONFLATE = False
288 CONFLATE = False
287
289
288 def setup(self):
290 def setup(self):
289
291
290 ncolspan = 1
292 ncolspan = 1
291 colspan = 1
293 colspan = 1
292 self.ncols = 2
294 self.ncols = 2
293 self.nrows = self.dataOut.nPairs
295 self.nrows = self.dataOut.nPairs
294 self.width = 3.6*self.ncols
296 self.width = 3.6*self.ncols
295 self.height = 3.2*self.nrows
297 self.height = 3.2*self.nrows
296
298
297 self.ylabel = 'Range [Km]'
299 self.ylabel = 'Range [Km]'
298 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
300 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
299
301
300 if self.figure is None:
302 if self.figure is None:
301 self.figure = plt.figure(figsize=(self.width, self.height),
303 self.figure = plt.figure(figsize=(self.width, self.height),
302 edgecolor='k',
304 edgecolor='k',
303 facecolor='w')
305 facecolor='w')
304 else:
306 else:
305 self.figure.clf()
307 self.figure.clf()
306
308
307 for y in range(self.nrows):
309 for y in range(self.nrows):
308 for x in range(self.ncols):
310 for x in range(self.ncols):
309 ax = plt.subplot2grid((self.nrows, self.ncols), (y, x), 1, 1)
311 ax = plt.subplot2grid((self.nrows, self.ncols), (y, x), 1, 1)
310 ax.firsttime = True
312 ax.firsttime = True
311 self.axes.append(ax)
313 self.axes.append(ax)
312
314
313 def plot(self):
315 def plot(self):
314
316
315 if self.xaxis == "frequency":
317 if self.xaxis == "frequency":
316 x = self.dataOut.getFreqRange(1)/1000.
318 x = self.dataOut.getFreqRange(1)/1000.
317 xlabel = "Frequency (kHz)"
319 xlabel = "Frequency (kHz)"
318 elif self.xaxis == "time":
320 elif self.xaxis == "time":
319 x = self.dataOut.getAcfRange(1)
321 x = self.dataOut.getAcfRange(1)
320 xlabel = "Time (ms)"
322 xlabel = "Time (ms)"
321 else:
323 else:
322 x = self.dataOut.getVelRange(1)
324 x = self.dataOut.getVelRange(1)
323 xlabel = "Velocity (m/s)"
325 xlabel = "Velocity (m/s)"
324
326
325 y = self.dataOut.getHeiRange()
327 y = self.dataOut.getHeiRange()
326 z_coh = self.data['cspc_coh']
328 z_coh = self.data['cspc_coh']
327 z_phase = self.data['cspc_phase']
329 z_phase = self.data['cspc_phase']
328
330
329 for n in range(self.nrows):
331 for n in range(self.nrows):
330 ax = self.axes[2*n]
332 ax = self.axes[2*n]
331 ax1 = self.axes[2*n+1]
333 ax1 = self.axes[2*n+1]
332 if ax.firsttime:
334 if ax.firsttime:
333 self.xmax = self.xmax if self.xmax else np.nanmax(x)
335 self.xmax = self.xmax if self.xmax else np.nanmax(x)
334 self.xmin = self.xmin if self.xmin else -self.xmax
336 self.xmin = self.xmin if self.xmin else -self.xmax
335 self.ymin = self.ymin if self.ymin else np.nanmin(y)
337 self.ymin = self.ymin if self.ymin else np.nanmin(y)
336 self.ymax = self.ymax if self.ymax else np.nanmax(y)
338 self.ymax = self.ymax if self.ymax else np.nanmax(y)
337 self.zmin_coh = self.zmin_coh if self.zmin_coh else 0.0
339 self.zmin_coh = self.zmin_coh if self.zmin_coh else 0.0
338 self.zmax_coh = self.zmax_coh if self.zmax_coh else 1.0
340 self.zmax_coh = self.zmax_coh if self.zmax_coh else 1.0
339 self.zmin_phase = self.zmin_phase if self.zmin_phase else -180
341 self.zmin_phase = self.zmin_phase if self.zmin_phase else -180
340 self.zmax_phase = self.zmax_phase if self.zmax_phase else 180
342 self.zmax_phase = self.zmax_phase if self.zmax_phase else 180
341
343
342 ax.plot = ax.pcolormesh(x, y, z_coh[n].T,
344 ax.plot = ax.pcolormesh(x, y, z_coh[n].T,
343 vmin=self.zmin_coh,
345 vmin=self.zmin_coh,
344 vmax=self.zmax_coh,
346 vmax=self.zmax_coh,
345 cmap=plt.get_cmap(self.colormap_coh)
347 cmap=plt.get_cmap(self.colormap_coh)
346 )
348 )
347 divider = make_axes_locatable(ax)
349 divider = make_axes_locatable(ax)
348 cax = divider.new_horizontal(size='3%', pad=0.05)
350 cax = divider.new_horizontal(size='3%', pad=0.05)
349 self.figure.add_axes(cax)
351 self.figure.add_axes(cax)
350 plt.colorbar(ax.plot, cax)
352 plt.colorbar(ax.plot, cax)
351
353
352 ax.set_xlim(self.xmin, self.xmax)
354 ax.set_xlim(self.xmin, self.xmax)
353 ax.set_ylim(self.ymin, self.ymax)
355 ax.set_ylim(self.ymin, self.ymax)
354
356
355 ax.set_ylabel(self.ylabel)
357 ax.set_ylabel(self.ylabel)
356 ax.set_xlabel(xlabel)
358 ax.set_xlabel(xlabel)
357 ax.firsttime = False
359 ax.firsttime = False
358
360
359 ax1.plot = ax1.pcolormesh(x, y, z_phase[n].T,
361 ax1.plot = ax1.pcolormesh(x, y, z_phase[n].T,
360 vmin=self.zmin_phase,
362 vmin=self.zmin_phase,
361 vmax=self.zmax_phase,
363 vmax=self.zmax_phase,
362 cmap=plt.get_cmap(self.colormap_phase)
364 cmap=plt.get_cmap(self.colormap_phase)
363 )
365 )
364 divider = make_axes_locatable(ax1)
366 divider = make_axes_locatable(ax1)
365 cax = divider.new_horizontal(size='3%', pad=0.05)
367 cax = divider.new_horizontal(size='3%', pad=0.05)
366 self.figure.add_axes(cax)
368 self.figure.add_axes(cax)
367 plt.colorbar(ax1.plot, cax)
369 plt.colorbar(ax1.plot, cax)
368
370
369 ax1.set_xlim(self.xmin, self.xmax)
371 ax1.set_xlim(self.xmin, self.xmax)
370 ax1.set_ylim(self.ymin, self.ymax)
372 ax1.set_ylim(self.ymin, self.ymax)
371
373
372 ax1.set_ylabel(self.ylabel)
374 ax1.set_ylabel(self.ylabel)
373 ax1.set_xlabel(xlabel)
375 ax1.set_xlabel(xlabel)
374 ax1.firsttime = False
376 ax1.firsttime = False
375 else:
377 else:
376 ax.plot.set_array(z_coh[n].T.ravel())
378 ax.plot.set_array(z_coh[n].T.ravel())
377 ax1.plot.set_array(z_phase[n].T.ravel())
379 ax1.plot.set_array(z_phase[n].T.ravel())
378
380
379 ax.set_title('Coherence Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
381 ax.set_title('Coherence Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
380 ax1.set_title('Phase Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
382 ax1.set_title('Phase Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
381 self.saveTime = self.max_time
383 self.saveTime = self.max_time
382
384
383
385
384 class PlotSpectraMeanData(PlotSpectraData):
386 class PlotSpectraMeanData(PlotSpectraData):
385
387
386 CODE = 'spc_mean'
388 CODE = 'spc_mean'
387 colormap = 'jet'
389 colormap = 'jet'
388
390
389 def plot(self):
391 def plot(self):
390
392
391 if self.xaxis == "frequency":
393 if self.xaxis == "frequency":
392 x = self.dataOut.getFreqRange(1)/1000.
394 x = self.dataOut.getFreqRange(1)/1000.
393 xlabel = "Frequency (kHz)"
395 xlabel = "Frequency (kHz)"
394 elif self.xaxis == "time":
396 elif self.xaxis == "time":
395 x = self.dataOut.getAcfRange(1)
397 x = self.dataOut.getAcfRange(1)
396 xlabel = "Time (ms)"
398 xlabel = "Time (ms)"
397 else:
399 else:
398 x = self.dataOut.getVelRange(1)
400 x = self.dataOut.getVelRange(1)
399 xlabel = "Velocity (m/s)"
401 xlabel = "Velocity (m/s)"
400
402
401 y = self.dataOut.getHeiRange()
403 y = self.dataOut.getHeiRange()
402 z = self.data['spc']
404 z = self.data['spc']
403 mean = self.data['mean'][self.max_time]
405 mean = self.data['mean'][self.max_time]
404
406
405 for n, ax in enumerate(self.axes):
407 for n, ax in enumerate(self.axes):
406
408
407 if ax.firsttime:
409 if ax.firsttime:
408 self.xmax = self.xmax if self.xmax else np.nanmax(x)
410 self.xmax = self.xmax if self.xmax else np.nanmax(x)
409 self.xmin = self.xmin if self.xmin else -self.xmax
411 self.xmin = self.xmin if self.xmin else -self.xmax
410 self.ymin = self.ymin if self.ymin else np.nanmin(y)
412 self.ymin = self.ymin if self.ymin else np.nanmin(y)
411 self.ymax = self.ymax if self.ymax else np.nanmax(y)
413 self.ymax = self.ymax if self.ymax else np.nanmax(y)
412 self.zmin = self.zmin if self.zmin else np.nanmin(z)
414 self.zmin = self.zmin if self.zmin else np.nanmin(z)
413 self.zmax = self.zmax if self.zmax else np.nanmax(z)
415 self.zmax = self.zmax if self.zmax else np.nanmax(z)
414 ax.plt = ax.pcolormesh(x, y, z[n].T,
416 ax.plt = ax.pcolormesh(x, y, z[n].T,
415 vmin=self.zmin,
417 vmin=self.zmin,
416 vmax=self.zmax,
418 vmax=self.zmax,
417 cmap=plt.get_cmap(self.colormap)
419 cmap=plt.get_cmap(self.colormap)
418 )
420 )
419 ax.plt_dop = ax.plot(mean[n], y,
421 ax.plt_dop = ax.plot(mean[n], y,
420 color='k')[0]
422 color='k')[0]
421
423
422 divider = make_axes_locatable(ax)
424 divider = make_axes_locatable(ax)
423 cax = divider.new_horizontal(size='3%', pad=0.05)
425 cax = divider.new_horizontal(size='3%', pad=0.05)
424 self.figure.add_axes(cax)
426 self.figure.add_axes(cax)
425 plt.colorbar(ax.plt, cax)
427 plt.colorbar(ax.plt, cax)
426
428
427 ax.set_xlim(self.xmin, self.xmax)
429 ax.set_xlim(self.xmin, self.xmax)
428 ax.set_ylim(self.ymin, self.ymax)
430 ax.set_ylim(self.ymin, self.ymax)
429
431
430 ax.set_ylabel(self.ylabel)
432 ax.set_ylabel(self.ylabel)
431 ax.set_xlabel(xlabel)
433 ax.set_xlabel(xlabel)
432
434
433 ax.firsttime = False
435 ax.firsttime = False
434
436
435 if self.showprofile:
437 if self.showprofile:
436 ax.plt_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
438 ax.plt_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
437 ax.ax_profile.set_xlim(self.zmin, self.zmax)
439 ax.ax_profile.set_xlim(self.zmin, self.zmax)
438 ax.ax_profile.set_ylim(self.ymin, self.ymax)
440 ax.ax_profile.set_ylim(self.ymin, self.ymax)
439 ax.ax_profile.set_xlabel('dB')
441 ax.ax_profile.set_xlabel('dB')
440 ax.ax_profile.grid(b=True, axis='x')
442 ax.ax_profile.grid(b=True, axis='x')
441 ax.plt_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
443 ax.plt_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
442 color="k", linestyle="dashed", lw=2)[0]
444 color="k", linestyle="dashed", lw=2)[0]
443 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
445 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
444 else:
446 else:
445 ax.plt.set_array(z[n].T.ravel())
447 ax.plt.set_array(z[n].T.ravel())
446 ax.plt_dop.set_data(mean[n], y)
448 ax.plt_dop.set_data(mean[n], y)
447 if self.showprofile:
449 if self.showprofile:
448 ax.plt_profile.set_data(self.data['rti'][self.max_time][n], y)
450 ax.plt_profile.set_data(self.data['rti'][self.max_time][n], y)
449 ax.plt_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
451 ax.plt_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
450
452
451 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
453 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
452 size=8)
454 size=8)
453 self.saveTime = self.max_time
455 self.saveTime = self.max_time
454
456
455
457
456 class PlotRTIData(PlotData):
458 class PlotRTIData(PlotData):
457
459
458 CODE = 'rti'
460 CODE = 'rti'
459 colormap = 'jro'
461 colormap = 'jro'
460
462
461 def setup(self):
463 def setup(self):
462 self.ncols = 1
464 self.ncols = 1
463 self.nrows = self.dataOut.nChannels
465 self.nrows = self.dataOut.nChannels
464 self.width = 10
466 self.width = 10
465 self.height = 2.2*self.nrows if self.nrows<6 else 12
467 self.height = 2.2*self.nrows if self.nrows<6 else 12
466 if self.nrows==1:
468 if self.nrows==1:
467 self.height += 1
469 self.height += 1
468 self.ylabel = 'Range [Km]'
470 self.ylabel = 'Range [Km]'
469 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
471 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
470
472
471 if self.figure is None:
473 if self.figure is None:
472 self.figure = plt.figure(figsize=(self.width, self.height),
474 self.figure = plt.figure(figsize=(self.width, self.height),
473 edgecolor='k',
475 edgecolor='k',
474 facecolor='w')
476 facecolor='w')
475 else:
477 else:
476 self.figure.clf()
478 self.figure.clf()
477 self.axes = []
479 self.axes = []
478
480
479 for n in range(self.nrows):
481 for n in range(self.nrows):
480 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
482 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
481 ax.firsttime = True
483 ax.firsttime = True
482 self.axes.append(ax)
484 self.axes.append(ax)
483
485
484 def plot(self):
486 def plot(self):
485
487
486 self.x = np.array(self.times)
488 self.x = np.array(self.times)
487 self.y = self.dataOut.getHeiRange()
489 self.y = self.dataOut.getHeiRange()
488 self.z = []
490 self.z = []
489
491
490 for ch in range(self.nrows):
492 for ch in range(self.nrows):
491 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
493 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
492
494
493 self.z = np.array(self.z)
495 self.z = np.array(self.z)
494 for n, ax in enumerate(self.axes):
496 for n, ax in enumerate(self.axes):
495 x, y, z = self.fill_gaps(*self.decimate())
497 x, y, z = self.fill_gaps(*self.decimate())
496 xmin = self.min_time
498 if self.xmin is None:
497 xmax = xmin+self.xrange*60*60
499 xmin = self.min_time
500 else:
501 xmin = fromtimestamp(int(self.xmin), self.min_time)
502 if self.xmax is None:
503 xmax = xmin + self.xrange*60*60
504 else:
505 xmax = xmin + (self.xmax - self.xmin) * 60 * 60
498 self.zmin = self.zmin if self.zmin else np.min(self.z)
506 self.zmin = self.zmin if self.zmin else np.min(self.z)
499 self.zmax = self.zmax if self.zmax else np.max(self.z)
507 self.zmax = self.zmax if self.zmax else np.max(self.z)
500 if ax.firsttime:
508 if ax.firsttime:
501 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
509 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
502 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
510 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
503 plot = ax.pcolormesh(x, y, z[n].T,
511 plot = ax.pcolormesh(x, y, z[n].T,
504 vmin=self.zmin,
512 vmin=self.zmin,
505 vmax=self.zmax,
513 vmax=self.zmax,
506 cmap=plt.get_cmap(self.colormap)
514 cmap=plt.get_cmap(self.colormap)
507 )
515 )
508 divider = make_axes_locatable(ax)
516 divider = make_axes_locatable(ax)
509 cax = divider.new_horizontal(size='2%', pad=0.05)
517 cax = divider.new_horizontal(size='2%', pad=0.05)
510 self.figure.add_axes(cax)
518 self.figure.add_axes(cax)
511 plt.colorbar(plot, cax)
519 plt.colorbar(plot, cax)
512 ax.set_ylim(self.ymin, self.ymax)
520 ax.set_ylim(self.ymin, self.ymax)
513
521
514 ax.xaxis.set_major_formatter(FuncFormatter(func))
522 ax.xaxis.set_major_formatter(FuncFormatter(func))
515 ax.xaxis.set_major_locator(LinearLocator(6))
523 ax.xaxis.set_major_locator(LinearLocator(6))
516
524
517 ax.set_ylabel(self.ylabel)
525 ax.set_ylabel(self.ylabel)
518
526
519 # if self.xmin is None:
527 # if self.xmin is None:
520 # xmin = self.min_time
528 # xmin = self.min_time
521 # else:
529 # else:
522 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
530 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
523 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
531 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
524
532
525 ax.set_xlim(xmin, xmax)
533 ax.set_xlim(xmin, xmax)
526 ax.firsttime = False
534 ax.firsttime = False
527 else:
535 else:
528 ax.collections.remove(ax.collections[0])
536 ax.collections.remove(ax.collections[0])
529 ax.set_xlim(xmin, xmax)
537 ax.set_xlim(xmin, xmax)
530 plot = ax.pcolormesh(x, y, z[n].T,
538 plot = ax.pcolormesh(x, y, z[n].T,
531 vmin=self.zmin,
539 vmin=self.zmin,
532 vmax=self.zmax,
540 vmax=self.zmax,
533 cmap=plt.get_cmap(self.colormap)
541 cmap=plt.get_cmap(self.colormap)
534 )
542 )
535 ax.set_title('{} {}'.format(self.titles[n],
543 ax.set_title('{} {}'.format(self.titles[n],
536 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
544 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
537 size=8)
545 size=8)
538
546
539 self.saveTime = self.min_time
547 self.saveTime = self.min_time
540
548
541
549
542 class PlotCOHData(PlotRTIData):
550 class PlotCOHData(PlotRTIData):
543
551
544 CODE = 'coh'
552 CODE = 'coh'
545
553
546 def setup(self):
554 def setup(self):
547
555
548 self.ncols = 1
556 self.ncols = 1
549 self.nrows = self.dataOut.nPairs
557 self.nrows = self.dataOut.nPairs
550 self.width = 10
558 self.width = 10
551 self.height = 2.2*self.nrows if self.nrows<6 else 12
559 self.height = 2.2*self.nrows if self.nrows<6 else 12
552 if self.nrows==1:
560 if self.nrows==1:
553 self.height += 1
561 self.height += 1
554 self.ylabel = 'Range [Km]'
562 self.ylabel = 'Range [Km]'
555 self.titles = ['{} Ch{} * Ch{}'.format(self.CODE.upper(), x[0], x[1]) for x in self.dataOut.pairsList]
563 self.titles = ['{} Ch{} * Ch{}'.format(self.CODE.upper(), x[0], x[1]) for x in self.dataOut.pairsList]
556
564
557 if self.figure is None:
565 if self.figure is None:
558 self.figure = plt.figure(figsize=(self.width, self.height),
566 self.figure = plt.figure(figsize=(self.width, self.height),
559 edgecolor='k',
567 edgecolor='k',
560 facecolor='w')
568 facecolor='w')
561 else:
569 else:
562 self.figure.clf()
570 self.figure.clf()
563 self.axes = []
571 self.axes = []
564
572
565 for n in range(self.nrows):
573 for n in range(self.nrows):
566 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
574 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
567 ax.firsttime = True
575 ax.firsttime = True
568 self.axes.append(ax)
576 self.axes.append(ax)
569
577
570
578
571 class PlotNoiseData(PlotData):
579 class PlotNoiseData(PlotData):
572 CODE = 'noise'
580 CODE = 'noise'
573
581
574 def setup(self):
582 def setup(self):
575
583
576 self.ncols = 1
584 self.ncols = 1
577 self.nrows = 1
585 self.nrows = 1
578 self.width = 10
586 self.width = 10
579 self.height = 3.2
587 self.height = 3.2
580 self.ylabel = 'Intensity [dB]'
588 self.ylabel = 'Intensity [dB]'
581 self.titles = ['Noise']
589 self.titles = ['Noise']
582
590
583 if self.figure is None:
591 if self.figure is None:
584 self.figure = plt.figure(figsize=(self.width, self.height),
592 self.figure = plt.figure(figsize=(self.width, self.height),
585 edgecolor='k',
593 edgecolor='k',
586 facecolor='w')
594 facecolor='w')
587 else:
595 else:
588 self.figure.clf()
596 self.figure.clf()
589 self.axes = []
597 self.axes = []
590
598
591 self.ax = self.figure.add_subplot(self.nrows, self.ncols, 1)
599 self.ax = self.figure.add_subplot(self.nrows, self.ncols, 1)
592 self.ax.firsttime = True
600 self.ax.firsttime = True
593
601
594 def plot(self):
602 def plot(self):
595
603
596 x = self.times
604 x = self.times
597 xmin = self.min_time
605 xmin = self.min_time
598 xmax = xmin+self.xrange*60*60
606 xmax = xmin+self.xrange*60*60
599 if self.ax.firsttime:
607 if self.ax.firsttime:
600 for ch in self.dataOut.channelList:
608 for ch in self.dataOut.channelList:
601 y = [self.data[self.CODE][t][ch] for t in self.times]
609 y = [self.data[self.CODE][t][ch] for t in self.times]
602 self.ax.plot(x, y, lw=1, label='Ch{}'.format(ch))
610 self.ax.plot(x, y, lw=1, label='Ch{}'.format(ch))
603 self.ax.firsttime = False
611 self.ax.firsttime = False
604 self.ax.xaxis.set_major_formatter(FuncFormatter(func))
612 self.ax.xaxis.set_major_formatter(FuncFormatter(func))
605 self.ax.xaxis.set_major_locator(LinearLocator(6))
613 self.ax.xaxis.set_major_locator(LinearLocator(6))
606 self.ax.set_ylabel(self.ylabel)
614 self.ax.set_ylabel(self.ylabel)
607 plt.legend()
615 plt.legend()
608 else:
616 else:
609 for ch in self.dataOut.channelList:
617 for ch in self.dataOut.channelList:
610 y = [self.data[self.CODE][t][ch] for t in self.times]
618 y = [self.data[self.CODE][t][ch] for t in self.times]
611 self.ax.lines[ch].set_data(x, y)
619 self.ax.lines[ch].set_data(x, y)
612
620
613 self.ax.set_xlim(xmin, xmax)
621 self.ax.set_xlim(xmin, xmax)
614 self.ax.set_ylim(min(y)-5, max(y)+5)
622 self.ax.set_ylim(min(y)-5, max(y)+5)
615 self.saveTime = self.min_time
623 self.saveTime = self.min_time
616
624
617
625
618 class PlotWindProfilerData(PlotRTIData):
626 class PlotWindProfilerData(PlotRTIData):
619
627
620 CODE = 'wind'
628 CODE = 'wind'
621 colormap = 'seismic'
629 colormap = 'seismic'
622
630
623 def setup(self):
631 def setup(self):
624 self.ncols = 1
632 self.ncols = 1
625 self.nrows = self.dataOut.data_output.shape[0]
633 self.nrows = self.dataOut.data_output.shape[0]
626 self.width = 10
634 self.width = 10
627 self.height = 2.2*self.nrows
635 self.height = 2.2*self.nrows
628 self.ylabel = 'Height [Km]'
636 self.ylabel = 'Height [Km]'
629 self.titles = ['Zonal Wind' ,'Meridional Wind', 'Vertical Wind']
637 self.titles = ['Zonal Wind' ,'Meridional Wind', 'Vertical Wind']
630 self.clabels = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
638 self.clabels = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
631 self.windFactor = [1, 1, 100]
639 self.windFactor = [1, 1, 100]
632
640
633 if self.figure is None:
641 if self.figure is None:
634 self.figure = plt.figure(figsize=(self.width, self.height),
642 self.figure = plt.figure(figsize=(self.width, self.height),
635 edgecolor='k',
643 edgecolor='k',
636 facecolor='w')
644 facecolor='w')
637 else:
645 else:
638 self.figure.clf()
646 self.figure.clf()
639 self.axes = []
647 self.axes = []
640
648
641 for n in range(self.nrows):
649 for n in range(self.nrows):
642 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
650 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
643 ax.firsttime = True
651 ax.firsttime = True
644 self.axes.append(ax)
652 self.axes.append(ax)
645
653
646 def plot(self):
654 def plot(self):
647
655
648 self.x = np.array(self.times)
656 self.x = np.array(self.times)
649 self.y = self.dataOut.heightList
657 self.y = self.dataOut.heightList
650 self.z = []
658 self.z = []
651
659
652 for ch in range(self.nrows):
660 for ch in range(self.nrows):
653 self.z.append([self.data['output'][t][ch] for t in self.times])
661 self.z.append([self.data['output'][t][ch] for t in self.times])
654
662
655 self.z = np.array(self.z)
663 self.z = np.array(self.z)
656 self.z = numpy.ma.masked_invalid(self.z)
664 self.z = numpy.ma.masked_invalid(self.z)
657
665
658 cmap=plt.get_cmap(self.colormap)
666 cmap=plt.get_cmap(self.colormap)
659 cmap.set_bad('black', 1.)
667 cmap.set_bad('black', 1.)
660
668
661 for n, ax in enumerate(self.axes):
669 for n, ax in enumerate(self.axes):
662 x, y, z = self.fill_gaps(*self.decimate())
670 x, y, z = self.fill_gaps(*self.decimate())
663 xmin = self.min_time
671 xmin = self.min_time
664 xmax = xmin+self.xrange*60*60
672 xmax = xmin+self.xrange*60*60
665 if ax.firsttime:
673 if ax.firsttime:
666 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
674 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
667 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
675 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
668 self.zmax = self.zmax if self.zmax else numpy.nanmax(abs(self.z[:-1, :]))
676 self.zmax = self.zmax if self.zmax else numpy.nanmax(abs(self.z[:-1, :]))
669 self.zmin = self.zmin if self.zmin else -self.zmax
677 self.zmin = self.zmin if self.zmin else -self.zmax
670
678
671 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
679 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
672 vmin=self.zmin,
680 vmin=self.zmin,
673 vmax=self.zmax,
681 vmax=self.zmax,
674 cmap=cmap
682 cmap=cmap
675 )
683 )
676 divider = make_axes_locatable(ax)
684 divider = make_axes_locatable(ax)
677 cax = divider.new_horizontal(size='2%', pad=0.05)
685 cax = divider.new_horizontal(size='2%', pad=0.05)
678 self.figure.add_axes(cax)
686 self.figure.add_axes(cax)
679 cb = plt.colorbar(plot, cax)
687 cb = plt.colorbar(plot, cax)
680 cb.set_label(self.clabels[n])
688 cb.set_label(self.clabels[n])
681 ax.set_ylim(self.ymin, self.ymax)
689 ax.set_ylim(self.ymin, self.ymax)
682
690
683 ax.xaxis.set_major_formatter(FuncFormatter(func))
691 ax.xaxis.set_major_formatter(FuncFormatter(func))
684 ax.xaxis.set_major_locator(LinearLocator(6))
692 ax.xaxis.set_major_locator(LinearLocator(6))
685
693
686 ax.set_ylabel(self.ylabel)
694 ax.set_ylabel(self.ylabel)
687
695
688 ax.set_xlim(xmin, xmax)
696 ax.set_xlim(xmin, xmax)
689 ax.firsttime = False
697 ax.firsttime = False
690 else:
698 else:
691 ax.collections.remove(ax.collections[0])
699 ax.collections.remove(ax.collections[0])
692 ax.set_xlim(xmin, xmax)
700 ax.set_xlim(xmin, xmax)
693 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
701 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
694 vmin=self.zmin,
702 vmin=self.zmin,
695 vmax=self.zmax,
703 vmax=self.zmax,
696 cmap=plt.get_cmap(self.colormap)
704 cmap=plt.get_cmap(self.colormap)
697 )
705 )
698 ax.set_title('{} {}'.format(self.titles[n],
706 ax.set_title('{} {}'.format(self.titles[n],
699 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
707 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
700 size=8)
708 size=8)
701
709
702 self.saveTime = self.min_time
710 self.saveTime = self.min_time
703
711
704
712
705 class PlotSNRData(PlotRTIData):
713 class PlotSNRData(PlotRTIData):
706 CODE = 'snr'
714 CODE = 'snr'
707 colormap = 'jet'
715 colormap = 'jet'
708
716
709 class PlotDOPData(PlotRTIData):
717 class PlotDOPData(PlotRTIData):
710 CODE = 'dop'
718 CODE = 'dop'
711 colormap = 'jet'
719 colormap = 'jet'
712
720
713
721
714 class PlotPHASEData(PlotCOHData):
722 class PlotPHASEData(PlotCOHData):
715 CODE = 'phase'
723 CODE = 'phase'
716 colormap = 'seismic'
724 colormap = 'seismic'
717
725
718
726
719 class PlotSkyMapData(PlotData):
727 class PlotSkyMapData(PlotData):
720
728
721 CODE = 'met'
729 CODE = 'met'
722
730
723 def setup(self):
731 def setup(self):
724
732
725 self.ncols = 1
733 self.ncols = 1
726 self.nrows = 1
734 self.nrows = 1
727 self.width = 7.2
735 self.width = 7.2
728 self.height = 7.2
736 self.height = 7.2
729
737
730 self.xlabel = 'Zonal Zenith Angle (deg)'
738 self.xlabel = 'Zonal Zenith Angle (deg)'
731 self.ylabel = 'Meridional Zenith Angle (deg)'
739 self.ylabel = 'Meridional Zenith Angle (deg)'
732
740
733 if self.figure is None:
741 if self.figure is None:
734 self.figure = plt.figure(figsize=(self.width, self.height),
742 self.figure = plt.figure(figsize=(self.width, self.height),
735 edgecolor='k',
743 edgecolor='k',
736 facecolor='w')
744 facecolor='w')
737 else:
745 else:
738 self.figure.clf()
746 self.figure.clf()
739
747
740 self.ax = plt.subplot2grid((self.nrows, self.ncols), (0, 0), 1, 1, polar=True)
748 self.ax = plt.subplot2grid((self.nrows, self.ncols), (0, 0), 1, 1, polar=True)
741 self.ax.firsttime = True
749 self.ax.firsttime = True
742
750
743
751
744 def plot(self):
752 def plot(self):
745
753
746 arrayParameters = np.concatenate([self.data['param'][t] for t in self.times])
754 arrayParameters = np.concatenate([self.data['param'][t] for t in self.times])
747 error = arrayParameters[:,-1]
755 error = arrayParameters[:,-1]
748 indValid = numpy.where(error == 0)[0]
756 indValid = numpy.where(error == 0)[0]
749 finalMeteor = arrayParameters[indValid,:]
757 finalMeteor = arrayParameters[indValid,:]
750 finalAzimuth = finalMeteor[:,3]
758 finalAzimuth = finalMeteor[:,3]
751 finalZenith = finalMeteor[:,4]
759 finalZenith = finalMeteor[:,4]
752
760
753 x = finalAzimuth*numpy.pi/180
761 x = finalAzimuth*numpy.pi/180
754 y = finalZenith
762 y = finalZenith
755
763
756 if self.ax.firsttime:
764 if self.ax.firsttime:
757 self.ax.plot = self.ax.plot(x, y, 'bo', markersize=5)[0]
765 self.ax.plot = self.ax.plot(x, y, 'bo', markersize=5)[0]
758 self.ax.set_ylim(0,90)
766 self.ax.set_ylim(0,90)
759 self.ax.set_yticks(numpy.arange(0,90,20))
767 self.ax.set_yticks(numpy.arange(0,90,20))
760 self.ax.set_xlabel(self.xlabel)
768 self.ax.set_xlabel(self.xlabel)
761 self.ax.set_ylabel(self.ylabel)
769 self.ax.set_ylabel(self.ylabel)
762 self.ax.yaxis.labelpad = 40
770 self.ax.yaxis.labelpad = 40
763 self.ax.firsttime = False
771 self.ax.firsttime = False
764 else:
772 else:
765 self.ax.plot.set_data(x, y)
773 self.ax.plot.set_data(x, y)
766
774
767
775
768 dt1 = datetime.datetime.fromtimestamp(self.min_time).strftime('%y/%m/%d %H:%M:%S')
776 dt1 = datetime.datetime.fromtimestamp(self.min_time).strftime('%y/%m/%d %H:%M:%S')
769 dt2 = datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')
777 dt2 = datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')
770 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
778 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
771 dt2,
779 dt2,
772 len(x))
780 len(x))
773 self.ax.set_title(title, size=8)
781 self.ax.set_title(title, size=8)
774
782
775 self.saveTime = self.max_time
783 self.saveTime = self.max_time
Show file before | Show file after | Hide comments
@@ -1,1816 +1,1813
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time, datetime
13 import time, datetime
14 import traceback
14 import traceback
15 import zmq
15 import zmq
16
16
17 try:
17 try:
18 from gevent import sleep
18 from gevent import sleep
19 except:
19 except:
20 from time import sleep
20 from time import sleep
21
21
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24
24
25 LOCALTIME = True
25 LOCALTIME = True
26
26
27 def isNumber(cad):
27 def isNumber(cad):
28 """
28 """
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30
30
31 Excepciones:
31 Excepciones:
32 Si un determinado string no puede ser convertido a numero
32 Si un determinado string no puede ser convertido a numero
33 Input:
33 Input:
34 str, string al cual se le analiza para determinar si convertible a un numero o no
34 str, string al cual se le analiza para determinar si convertible a un numero o no
35
35
36 Return:
36 Return:
37 True : si el string es uno numerico
37 True : si el string es uno numerico
38 False : no es un string numerico
38 False : no es un string numerico
39 """
39 """
40 try:
40 try:
41 float( cad )
41 float( cad )
42 return True
42 return True
43 except:
43 except:
44 return False
44 return False
45
45
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 """
47 """
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49
49
50 Inputs:
50 Inputs:
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52
52
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 segundos contados desde 01/01/1970.
54 segundos contados desde 01/01/1970.
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 segundos contados desde 01/01/1970.
56 segundos contados desde 01/01/1970.
57
57
58 Return:
58 Return:
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 fecha especificado, de lo contrario retorna False.
60 fecha especificado, de lo contrario retorna False.
61
61
62 Excepciones:
62 Excepciones:
63 Si el archivo no existe o no puede ser abierto
63 Si el archivo no existe o no puede ser abierto
64 Si la cabecera no puede ser leida.
64 Si la cabecera no puede ser leida.
65
65
66 """
66 """
67 basicHeaderObj = BasicHeader(LOCALTIME)
67 basicHeaderObj = BasicHeader(LOCALTIME)
68
68
69 try:
69 try:
70 fp = open(filename,'rb')
70 fp = open(filename,'rb')
71 except IOError:
71 except IOError:
72 print "The file %s can't be opened" %(filename)
72 print "The file %s can't be opened" %(filename)
73 return 0
73 return 0
74
74
75 sts = basicHeaderObj.read(fp)
75 sts = basicHeaderObj.read(fp)
76 fp.close()
76 fp.close()
77
77
78 if not(sts):
78 if not(sts):
79 print "Skipping the file %s because it has not a valid header" %(filename)
79 print "Skipping the file %s because it has not a valid header" %(filename)
80 return 0
80 return 0
81
81
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 return 0
83 return 0
84
84
85 return 1
85 return 1
86
86
87 def isTimeInRange(thisTime, startTime, endTime):
87 def isTimeInRange(thisTime, startTime, endTime):
88
88
89 if endTime >= startTime:
89 if endTime >= startTime:
90 if (thisTime < startTime) or (thisTime > endTime):
90 if (thisTime < startTime) or (thisTime > endTime):
91 return 0
91 return 0
92
92
93 return 1
93 return 1
94 else:
94 else:
95 if (thisTime < startTime) and (thisTime > endTime):
95 if (thisTime < startTime) and (thisTime > endTime):
96 return 0
96 return 0
97
97
98 return 1
98 return 1
99
99
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 """
101 """
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103
103
104 Inputs:
104 Inputs:
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106
106
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108
108
109 endDate : fecha final del rango seleccionado en formato datetime.date
109 endDate : fecha final del rango seleccionado en formato datetime.date
110
110
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112
112
113 endTime : tiempo final del rango seleccionado en formato datetime.time
113 endTime : tiempo final del rango seleccionado en formato datetime.time
114
114
115 Return:
115 Return:
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 fecha especificado, de lo contrario retorna False.
117 fecha especificado, de lo contrario retorna False.
118
118
119 Excepciones:
119 Excepciones:
120 Si el archivo no existe o no puede ser abierto
120 Si el archivo no existe o no puede ser abierto
121 Si la cabecera no puede ser leida.
121 Si la cabecera no puede ser leida.
122
122
123 """
123 """
124
124
125
125
126 try:
126 try:
127 fp = open(filename,'rb')
127 fp = open(filename,'rb')
128 except IOError:
128 except IOError:
129 print "The file %s can't be opened" %(filename)
129 print "The file %s can't be opened" %(filename)
130 return None
130 return None
131
131
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 systemHeaderObj = SystemHeader()
133 systemHeaderObj = SystemHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
135 processingHeaderObj = ProcessingHeader()
135 processingHeaderObj = ProcessingHeader()
136
136
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138
138
139 sts = firstBasicHeaderObj.read(fp)
139 sts = firstBasicHeaderObj.read(fp)
140
140
141 if not(sts):
141 if not(sts):
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 return None
143 return None
144
144
145 if not systemHeaderObj.read(fp):
145 if not systemHeaderObj.read(fp):
146 return None
146 return None
147
147
148 if not radarControllerHeaderObj.read(fp):
148 if not radarControllerHeaderObj.read(fp):
149 return None
149 return None
150
150
151 if not processingHeaderObj.read(fp):
151 if not processingHeaderObj.read(fp):
152 return None
152 return None
153
153
154 filesize = os.path.getsize(filename)
154 filesize = os.path.getsize(filename)
155
155
156 offset = processingHeaderObj.blockSize + 24 #header size
156 offset = processingHeaderObj.blockSize + 24 #header size
157
157
158 if filesize <= offset:
158 if filesize <= offset:
159 print "[Reading] %s: This file has not enough data" %filename
159 print "[Reading] %s: This file has not enough data" %filename
160 return None
160 return None
161
161
162 fp.seek(-offset, 2)
162 fp.seek(-offset, 2)
163
163
164 sts = lastBasicHeaderObj.read(fp)
164 sts = lastBasicHeaderObj.read(fp)
165
165
166 fp.close()
166 fp.close()
167
167
168 thisDatetime = lastBasicHeaderObj.datatime
168 thisDatetime = lastBasicHeaderObj.datatime
169 thisTime_last_block = thisDatetime.time()
169 thisTime_last_block = thisDatetime.time()
170
170
171 thisDatetime = firstBasicHeaderObj.datatime
171 thisDatetime = firstBasicHeaderObj.datatime
172 thisDate = thisDatetime.date()
172 thisDate = thisDatetime.date()
173 thisTime_first_block = thisDatetime.time()
173 thisTime_first_block = thisDatetime.time()
174
174
175 #General case
175 #General case
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 #-----------o----------------------------o-----------
177 #-----------o----------------------------o-----------
178 # startTime endTime
178 # startTime endTime
179
179
180 if endTime >= startTime:
180 if endTime >= startTime:
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 return None
182 return None
183
183
184 return thisDatetime
184 return thisDatetime
185
185
186 #If endTime < startTime then endTime belongs to the next day
186 #If endTime < startTime then endTime belongs to the next day
187
187
188
188
189 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 #-----------o----------------------------o-----------
190 #-----------o----------------------------o-----------
191 # endTime startTime
191 # endTime startTime
192
192
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 return None
194 return None
195
195
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 return None
197 return None
198
198
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 return None
200 return None
201
201
202 return thisDatetime
202 return thisDatetime
203
203
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 """
205 """
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207
207
208 Inputs:
208 Inputs:
209 folder : nombre completo del directorio.
209 folder : nombre completo del directorio.
210 Su formato deberia ser "/path_root/?YYYYDDD"
210 Su formato deberia ser "/path_root/?YYYYDDD"
211
211
212 siendo:
212 siendo:
213 YYYY : Anio (ejemplo 2015)
213 YYYY : Anio (ejemplo 2015)
214 DDD : Dia del anio (ejemplo 305)
214 DDD : Dia del anio (ejemplo 305)
215
215
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217
217
218 endDate : fecha final del rango seleccionado en formato datetime.date
218 endDate : fecha final del rango seleccionado en formato datetime.date
219
219
220 Return:
220 Return:
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 fecha especificado, de lo contrario retorna False.
222 fecha especificado, de lo contrario retorna False.
223 Excepciones:
223 Excepciones:
224 Si el directorio no tiene el formato adecuado
224 Si el directorio no tiene el formato adecuado
225 """
225 """
226
226
227 basename = os.path.basename(folder)
227 basename = os.path.basename(folder)
228
228
229 if not isRadarFolder(basename):
229 if not isRadarFolder(basename):
230 print "The folder %s has not the rigth format" %folder
230 print "The folder %s has not the rigth format" %folder
231 return 0
231 return 0
232
232
233 if startDate and endDate:
233 if startDate and endDate:
234 thisDate = getDateFromRadarFolder(basename)
234 thisDate = getDateFromRadarFolder(basename)
235
235
236 if thisDate < startDate:
236 if thisDate < startDate:
237 return 0
237 return 0
238
238
239 if thisDate > endDate:
239 if thisDate > endDate:
240 return 0
240 return 0
241
241
242 return 1
242 return 1
243
243
244 def isFileInDateRange(filename, startDate=None, endDate=None):
244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 """
245 """
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247
247
248 Inputs:
248 Inputs:
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250
250
251 Su formato deberia ser "?YYYYDDDsss"
251 Su formato deberia ser "?YYYYDDDsss"
252
252
253 siendo:
253 siendo:
254 YYYY : Anio (ejemplo 2015)
254 YYYY : Anio (ejemplo 2015)
255 DDD : Dia del anio (ejemplo 305)
255 DDD : Dia del anio (ejemplo 305)
256 sss : set
256 sss : set
257
257
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259
259
260 endDate : fecha final del rango seleccionado en formato datetime.date
260 endDate : fecha final del rango seleccionado en formato datetime.date
261
261
262 Return:
262 Return:
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 fecha especificado, de lo contrario retorna False.
264 fecha especificado, de lo contrario retorna False.
265 Excepciones:
265 Excepciones:
266 Si el archivo no tiene el formato adecuado
266 Si el archivo no tiene el formato adecuado
267 """
267 """
268
268
269 basename = os.path.basename(filename)
269 basename = os.path.basename(filename)
270
270
271 if not isRadarFile(basename):
271 if not isRadarFile(basename):
272 print "The filename %s has not the rigth format" %filename
272 print "The filename %s has not the rigth format" %filename
273 return 0
273 return 0
274
274
275 if startDate and endDate:
275 if startDate and endDate:
276 thisDate = getDateFromRadarFile(basename)
276 thisDate = getDateFromRadarFile(basename)
277
277
278 if thisDate < startDate:
278 if thisDate < startDate:
279 return 0
279 return 0
280
280
281 if thisDate > endDate:
281 if thisDate > endDate:
282 return 0
282 return 0
283
283
284 return 1
284 return 1
285
285
286 def getFileFromSet(path, ext, set):
286 def getFileFromSet(path, ext, set):
287 validFilelist = []
287 validFilelist = []
288 fileList = os.listdir(path)
288 fileList = os.listdir(path)
289
289
290 # 0 1234 567 89A BCDE
290 # 0 1234 567 89A BCDE
291 # H YYYY DDD SSS .ext
291 # H YYYY DDD SSS .ext
292
292
293 for thisFile in fileList:
293 for thisFile in fileList:
294 try:
294 try:
295 year = int(thisFile[1:5])
295 year = int(thisFile[1:5])
296 doy = int(thisFile[5:8])
296 doy = int(thisFile[5:8])
297 except:
297 except:
298 continue
298 continue
299
299
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 continue
301 continue
302
302
303 validFilelist.append(thisFile)
303 validFilelist.append(thisFile)
304
304
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306
306
307 if len(myfile)!= 0:
307 if len(myfile)!= 0:
308 return myfile[0]
308 return myfile[0]
309 else:
309 else:
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 print 'the filename %s does not exist'%filename
311 print 'the filename %s does not exist'%filename
312 print '...going to the last file: '
312 print '...going to the last file: '
313
313
314 if validFilelist:
314 if validFilelist:
315 validFilelist = sorted( validFilelist, key=str.lower )
315 validFilelist = sorted( validFilelist, key=str.lower )
316 return validFilelist[-1]
316 return validFilelist[-1]
317
317
318 return None
318 return None
319
319
320 def getlastFileFromPath(path, ext):
320 def getlastFileFromPath(path, ext):
321 """
321 """
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324
324
325 Input:
325 Input:
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 ext : extension de los files contenidos en una carpeta
327 ext : extension de los files contenidos en una carpeta
328
328
329 Return:
329 Return:
330 El ultimo file de una determinada carpeta, no se considera el path.
330 El ultimo file de una determinada carpeta, no se considera el path.
331 """
331 """
332 validFilelist = []
332 validFilelist = []
333 fileList = os.listdir(path)
333 fileList = os.listdir(path)
334
334
335 # 0 1234 567 89A BCDE
335 # 0 1234 567 89A BCDE
336 # H YYYY DDD SSS .ext
336 # H YYYY DDD SSS .ext
337
337
338 for thisFile in fileList:
338 for thisFile in fileList:
339
339
340 year = thisFile[1:5]
340 year = thisFile[1:5]
341 if not isNumber(year):
341 if not isNumber(year):
342 continue
342 continue
343
343
344 doy = thisFile[5:8]
344 doy = thisFile[5:8]
345 if not isNumber(doy):
345 if not isNumber(doy):
346 continue
346 continue
347
347
348 year = int(year)
348 year = int(year)
349 doy = int(doy)
349 doy = int(doy)
350
350
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 continue
352 continue
353
353
354 validFilelist.append(thisFile)
354 validFilelist.append(thisFile)
355
355
356 if validFilelist:
356 if validFilelist:
357 validFilelist = sorted( validFilelist, key=str.lower )
357 validFilelist = sorted( validFilelist, key=str.lower )
358 return validFilelist[-1]
358 return validFilelist[-1]
359
359
360 return None
360 return None
361
361
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 """
363 """
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 el path exacto de un determinado file.
366 el path exacto de un determinado file.
367
367
368 Example :
368 Example :
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370
370
371 Entonces la funcion prueba con las siguientes combinaciones
371 Entonces la funcion prueba con las siguientes combinaciones
372 .../.../y2009307367.ext
372 .../.../y2009307367.ext
373 .../.../Y2009307367.ext
373 .../.../Y2009307367.ext
374 .../.../x2009307/y2009307367.ext
374 .../.../x2009307/y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
376 .../.../X2009307/y2009307367.ext
376 .../.../X2009307/y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379
379
380 Return:
380 Return:
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 para el filename
383 para el filename
384 """
384 """
385 fullfilename = None
385 fullfilename = None
386 find_flag = False
386 find_flag = False
387 filename = None
387 filename = None
388
388
389 prefixDirList = [None,'d','D']
389 prefixDirList = [None,'d','D']
390 if ext.lower() == ".r": #voltage
390 if ext.lower() == ".r": #voltage
391 prefixFileList = ['d','D']
391 prefixFileList = ['d','D']
392 elif ext.lower() == ".pdata": #spectra
392 elif ext.lower() == ".pdata": #spectra
393 prefixFileList = ['p','P']
393 prefixFileList = ['p','P']
394 else:
394 else:
395 return None, filename
395 return None, filename
396
396
397 #barrido por las combinaciones posibles
397 #barrido por las combinaciones posibles
398 for prefixDir in prefixDirList:
398 for prefixDir in prefixDirList:
399 thispath = path
399 thispath = path
400 if prefixDir != None:
400 if prefixDir != None:
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 if foldercounter == 0:
402 if foldercounter == 0:
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 else:
404 else:
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409
409
410 if os.path.exists( fullfilename ): #verifico que exista
410 if os.path.exists( fullfilename ): #verifico que exista
411 find_flag = True
411 find_flag = True
412 break
412 break
413 if find_flag:
413 if find_flag:
414 break
414 break
415
415
416 if not(find_flag):
416 if not(find_flag):
417 return None, filename
417 return None, filename
418
418
419 return fullfilename, filename
419 return fullfilename, filename
420
420
421 def isRadarFolder(folder):
421 def isRadarFolder(folder):
422 try:
422 try:
423 year = int(folder[1:5])
423 year = int(folder[1:5])
424 doy = int(folder[5:8])
424 doy = int(folder[5:8])
425 except:
425 except:
426 return 0
426 return 0
427
427
428 return 1
428 return 1
429
429
430 def isRadarFile(file):
430 def isRadarFile(file):
431 try:
431 try:
432 year = int(file[1:5])
432 year = int(file[1:5])
433 doy = int(file[5:8])
433 doy = int(file[5:8])
434 set = int(file[8:11])
434 set = int(file[8:11])
435 except:
435 except:
436 return 0
436 return 0
437
437
438 return 1
438 return 1
439
439
440 def getDateFromRadarFile(file):
440 def getDateFromRadarFile(file):
441 try:
441 try:
442 year = int(file[1:5])
442 year = int(file[1:5])
443 doy = int(file[5:8])
443 doy = int(file[5:8])
444 set = int(file[8:11])
444 set = int(file[8:11])
445 except:
445 except:
446 return None
446 return None
447
447
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 return thisDate
449 return thisDate
450
450
451 def getDateFromRadarFolder(folder):
451 def getDateFromRadarFolder(folder):
452 try:
452 try:
453 year = int(folder[1:5])
453 year = int(folder[1:5])
454 doy = int(folder[5:8])
454 doy = int(folder[5:8])
455 except:
455 except:
456 return None
456 return None
457
457
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 return thisDate
459 return thisDate
460
460
461 class JRODataIO:
461 class JRODataIO:
462
462
463 c = 3E8
463 c = 3E8
464
464
465 isConfig = False
465 isConfig = False
466
466
467 basicHeaderObj = None
467 basicHeaderObj = None
468
468
469 systemHeaderObj = None
469 systemHeaderObj = None
470
470
471 radarControllerHeaderObj = None
471 radarControllerHeaderObj = None
472
472
473 processingHeaderObj = None
473 processingHeaderObj = None
474
474
475 dtype = None
475 dtype = None
476
476
477 pathList = []
477 pathList = []
478
478
479 filenameList = []
479 filenameList = []
480
480
481 filename = None
481 filename = None
482
482
483 ext = None
483 ext = None
484
484
485 flagIsNewFile = 1
485 flagIsNewFile = 1
486
486
487 flagDiscontinuousBlock = 0
487 flagDiscontinuousBlock = 0
488
488
489 flagIsNewBlock = 0
489 flagIsNewBlock = 0
490
490
491 fp = None
491 fp = None
492
492
493 firstHeaderSize = 0
493 firstHeaderSize = 0
494
494
495 basicHeaderSize = 24
495 basicHeaderSize = 24
496
496
497 versionFile = 1103
497 versionFile = 1103
498
498
499 fileSize = None
499 fileSize = None
500
500
501 # ippSeconds = None
501 # ippSeconds = None
502
502
503 fileSizeByHeader = None
503 fileSizeByHeader = None
504
504
505 fileIndex = None
505 fileIndex = None
506
506
507 profileIndex = None
507 profileIndex = None
508
508
509 blockIndex = None
509 blockIndex = None
510
510
511 nTotalBlocks = None
511 nTotalBlocks = None
512
512
513 maxTimeStep = 30
513 maxTimeStep = 30
514
514
515 lastUTTime = None
515 lastUTTime = None
516
516
517 datablock = None
517 datablock = None
518
518
519 dataOut = None
519 dataOut = None
520
520
521 blocksize = None
521 blocksize = None
522
522
523 getByBlock = False
523 getByBlock = False
524
524
525 def __init__(self):
525 def __init__(self):
526
526
527 raise NotImplementedError
527 raise NotImplementedError
528
528
529 def run(self):
529 def run(self):
530
530
531 raise NotImplementedError
531 raise NotImplementedError
532
532
533 def getDtypeWidth(self):
533 def getDtypeWidth(self):
534
534
535 dtype_index = get_dtype_index(self.dtype)
535 dtype_index = get_dtype_index(self.dtype)
536 dtype_width = get_dtype_width(dtype_index)
536 dtype_width = get_dtype_width(dtype_index)
537
537
538 return dtype_width
538 return dtype_width
539
539
540 def getAllowedArgs(self):
540 def getAllowedArgs(self):
541 return inspect.getargspec(self.run).args
541 return inspect.getargspec(self.run).args
542
542
543 class JRODataReader(JRODataIO):
543 class JRODataReader(JRODataIO):
544
544
545
545
546 online = 0
546 online = 0
547
547
548 realtime = 0
548 realtime = 0
549
549
550 nReadBlocks = 0
550 nReadBlocks = 0
551
551
552 delay = 10 #number of seconds waiting a new file
552 delay = 10 #number of seconds waiting a new file
553
553
554 nTries = 3 #quantity tries
554 nTries = 3 #quantity tries
555
555
556 nFiles = 3 #number of files for searching
556 nFiles = 3 #number of files for searching
557
557
558 path = None
558 path = None
559
559
560 foldercounter = 0
560 foldercounter = 0
561
561
562 flagNoMoreFiles = 0
562 flagNoMoreFiles = 0
563
563
564 datetimeList = []
564 datetimeList = []
565
565
566 __isFirstTimeOnline = 1
566 __isFirstTimeOnline = 1
567
567
568 __printInfo = True
568 __printInfo = True
569
569
570 profileIndex = None
570 profileIndex = None
571
571
572 nTxs = 1
572 nTxs = 1
573
573
574 txIndex = None
574 txIndex = None
575
575
576 #Added--------------------
576 #Added--------------------
577
577
578 selBlocksize = None
578 selBlocksize = None
579
579
580 selBlocktime = None
580 selBlocktime = None
581
581
582
582
583 def __init__(self):
583 def __init__(self):
584
584
585 """
585 """
586 This class is used to find data files
586 This class is used to find data files
587
587
588 Example:
588 Example:
589 reader = JRODataReader()
589 reader = JRODataReader()
590 fileList = reader.findDataFiles()
590 fileList = reader.findDataFiles()
591
591
592 """
592 """
593 pass
593 pass
594
594
595
595
596 def createObjByDefault(self):
596 def createObjByDefault(self):
597 """
597 """
598
598
599 """
599 """
600 raise NotImplementedError
600 raise NotImplementedError
601
601
602 def getBlockDimension(self):
602 def getBlockDimension(self):
603
603
604 raise NotImplementedError
604 raise NotImplementedError
605
605
606 def __searchFilesOffLine(self,
606 def __searchFilesOffLine(self,
607 path,
607 path,
608 startDate=None,
608 startDate=None,
609 endDate=None,
609 endDate=None,
610 startTime=datetime.time(0,0,0),
610 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
611 endTime=datetime.time(23,59,59),
612 set=None,
612 set=None,
613 expLabel='',
613 expLabel='',
614 ext='.r',
614 ext='.r',
615 queue=None,
615 queue=None,
616 cursor=None,
616 cursor=None,
617 skip=None,
617 skip=None,
618 walk=True):
618 walk=True):
619
619
620 self.filenameList = []
620 self.filenameList = []
621 self.datetimeList = []
621 self.datetimeList = []
622
622
623 pathList = []
623 pathList = []
624
624
625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
626
626
627 if dateList == []:
627 if dateList == []:
628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
629 return None, None
629 return None, None
630
630
631 if len(dateList) > 1:
631 if len(dateList) > 1:
632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
633 else:
633 else:
634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
635
635
636 filenameList = []
636 filenameList = []
637 datetimeList = []
637 datetimeList = []
638
638
639 for thisPath in pathList:
639 for thisPath in pathList:
640 # thisPath = pathList[pathDict[file]]
640 # thisPath = pathList[pathDict[file]]
641
641
642 fileList = glob.glob1(thisPath, "*%s" %ext)
642 fileList = glob.glob1(thisPath, "*%s" %ext)
643 fileList.sort()
643 fileList.sort()
644
644
645 skippedFileList = []
645 skippedFileList = []
646
646
647 if cursor is not None and skip is not None:
647 if cursor is not None and skip is not None:
648 # if cursor*skip > len(fileList):
648 # if cursor*skip > len(fileList):
649 if skip == 0:
649 if skip == 0:
650 if queue is not None:
650 if queue is not None:
651 queue.put(len(fileList))
651 queue.put(len(fileList))
652 skippedFileList = []
652 skippedFileList = []
653 else:
653 else:
654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
655
655
656 else:
656 else:
657 skippedFileList = fileList
657 skippedFileList = fileList
658
658
659 for file in skippedFileList:
659 for file in skippedFileList:
660
660
661 filename = os.path.join(thisPath,file)
661 filename = os.path.join(thisPath,file)
662
662
663 if not isFileInDateRange(filename, startDate, endDate):
663 if not isFileInDateRange(filename, startDate, endDate):
664 continue
664 continue
665
665
666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
667
667
668 if not(thisDatetime):
668 if not(thisDatetime):
669 continue
669 continue
670
670
671 filenameList.append(filename)
671 filenameList.append(filename)
672 datetimeList.append(thisDatetime)
672 datetimeList.append(thisDatetime)
673
673
674 if not(filenameList):
674 if not(filenameList):
675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
676 return None, None
676 return None, None
677
677
678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
679 print
679 print
680
680
681 for i in range(len(filenameList)):
681 for i in range(len(filenameList)):
682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
683
683
684 self.filenameList = filenameList
684 self.filenameList = filenameList
685 self.datetimeList = datetimeList
685 self.datetimeList = datetimeList
686
686
687 return pathList, filenameList
687 return pathList, filenameList
688
688
689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
690
690
691 """
691 """
692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
693 devuelve el archivo encontrado ademas de otros datos.
693 devuelve el archivo encontrado ademas de otros datos.
694
694
695 Input:
695 Input:
696 path : carpeta donde estan contenidos los files que contiene data
696 path : carpeta donde estan contenidos los files que contiene data
697
697
698 expLabel : Nombre del subexperimento (subfolder)
698 expLabel : Nombre del subexperimento (subfolder)
699
699
700 ext : extension de los files
700 ext : extension de los files
701
701
702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
703
703
704 Return:
704 Return:
705 directory : eL directorio donde esta el file encontrado
705 directory : eL directorio donde esta el file encontrado
706 filename : el ultimo file de una determinada carpeta
706 filename : el ultimo file de una determinada carpeta
707 year : el anho
707 year : el anho
708 doy : el numero de dia del anho
708 doy : el numero de dia del anho
709 set : el set del archivo
709 set : el set del archivo
710
710
711
711
712 """
712 """
713 if not os.path.isdir(path):
713 if not os.path.isdir(path):
714 return None, None, None, None, None, None
714 return None, None, None, None, None, None
715
715
716 dirList = []
716 dirList = []
717
717
718 if not walk:
718 if not walk:
719 fullpath = path
719 fullpath = path
720 foldercounter = 0
720 foldercounter = 0
721 else:
721 else:
722 #Filtra solo los directorios
722 #Filtra solo los directorios
723 for thisPath in os.listdir(path):
723 for thisPath in os.listdir(path):
724 if not os.path.isdir(os.path.join(path,thisPath)):
724 if not os.path.isdir(os.path.join(path,thisPath)):
725 continue
725 continue
726 if not isRadarFolder(thisPath):
726 if not isRadarFolder(thisPath):
727 continue
727 continue
728
728
729 dirList.append(thisPath)
729 dirList.append(thisPath)
730
730
731 if not(dirList):
731 if not(dirList):
732 return None, None, None, None, None, None
732 return None, None, None, None, None, None
733
733
734 dirList = sorted( dirList, key=str.lower )
734 dirList = sorted( dirList, key=str.lower )
735
735
736 doypath = dirList[-1]
736 doypath = dirList[-1]
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 fullpath = os.path.join(path, doypath, expLabel)
738 fullpath = os.path.join(path, doypath, expLabel)
739
739
740
740
741 print "[Reading] %s folder was found: " %(fullpath )
741 print "[Reading] %s folder was found: " %(fullpath )
742
742
743 if set == None:
743 if set == None:
744 filename = getlastFileFromPath(fullpath, ext)
744 filename = getlastFileFromPath(fullpath, ext)
745 else:
745 else:
746 filename = getFileFromSet(fullpath, ext, set)
746 filename = getFileFromSet(fullpath, ext, set)
747
747
748 if not(filename):
748 if not(filename):
749 return None, None, None, None, None, None
749 return None, None, None, None, None, None
750
750
751 print "[Reading] %s file was found" %(filename)
751 print "[Reading] %s file was found" %(filename)
752
752
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 return None, None, None, None, None, None
754 return None, None, None, None, None, None
755
755
756 year = int( filename[1:5] )
756 year = int( filename[1:5] )
757 doy = int( filename[5:8] )
757 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
758 set = int( filename[8:11] )
759
759
760 return fullpath, foldercounter, filename, year, doy, set
760 return fullpath, foldercounter, filename, year, doy, set
761
761
762 def __setNextFileOffline(self):
762 def __setNextFileOffline(self):
763
763
764 idFile = self.fileIndex
764 idFile = self.fileIndex
765
765
766 while (True):
766 while (True):
767 idFile += 1
767 idFile += 1
768 if not(idFile < len(self.filenameList)):
768 if not(idFile < len(self.filenameList)):
769 self.flagNoMoreFiles = 1
769 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
770 # print "[Reading] No more Files"
771 return 0
771 return 0
772
772
773 filename = self.filenameList[idFile]
773 filename = self.filenameList[idFile]
774
774
775 if not(self.__verifyFile(filename)):
775 if not(self.__verifyFile(filename)):
776 continue
776 continue
777
777
778 fileSize = os.path.getsize(filename)
778 fileSize = os.path.getsize(filename)
779 fp = open(filename,'rb')
779 fp = open(filename,'rb')
780 break
780 break
781
781
782 self.flagIsNewFile = 1
782 self.flagIsNewFile = 1
783 self.fileIndex = idFile
783 self.fileIndex = idFile
784 self.filename = filename
784 self.filename = filename
785 self.fileSize = fileSize
785 self.fileSize = fileSize
786 self.fp = fp
786 self.fp = fp
787
787
788 # print "[Reading] Setting the file: %s"%self.filename
788 # print "[Reading] Setting the file: %s"%self.filename
789
789
790 return 1
790 return 1
791
791
792 def __setNextFileOnline(self):
792 def __setNextFileOnline(self):
793 """
793 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
796 siguientes.
797
797
798 Affected:
798 Affected:
799 self.flagIsNewFile
799 self.flagIsNewFile
800 self.filename
800 self.filename
801 self.fileSize
801 self.fileSize
802 self.fp
802 self.fp
803 self.set
803 self.set
804 self.flagNoMoreFiles
804 self.flagNoMoreFiles
805
805
806 Return:
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
809
810 Excepciones:
810 Excepciones:
811 Si un determinado file no puede ser abierto
811 Si un determinado file no puede ser abierto
812 """
812 """
813 nFiles = 0
813 nFiles = 0
814 fileOk_flag = False
814 fileOk_flag = False
815 firstTime_flag = True
815 firstTime_flag = True
816
816
817 self.set += 1
817 self.set += 1
818
818
819 if self.set > 999:
819 if self.set > 999:
820 self.set = 0
820 self.set = 0
821 self.foldercounter += 1
821 self.foldercounter += 1
822
822
823 #busca el 1er file disponible
823 #busca el 1er file disponible
824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 if fullfilename:
825 if fullfilename:
826 if self.__verifyFile(fullfilename, False):
826 if self.__verifyFile(fullfilename, False):
827 fileOk_flag = True
827 fileOk_flag = True
828
828
829 #si no encuentra un file entonces espera y vuelve a buscar
829 #si no encuentra un file entonces espera y vuelve a buscar
830 if not(fileOk_flag):
830 if not(fileOk_flag):
831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832
832
833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 tries = self.nTries
834 tries = self.nTries
835 else:
835 else:
836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837
837
838 for nTries in range( tries ):
838 for nTries in range( tries ):
839 if firstTime_flag:
839 if firstTime_flag:
840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 sleep( self.delay )
841 sleep( self.delay )
842 else:
842 else:
843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844
844
845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 if fullfilename:
846 if fullfilename:
847 if self.__verifyFile(fullfilename):
847 if self.__verifyFile(fullfilename):
848 fileOk_flag = True
848 fileOk_flag = True
849 break
849 break
850
850
851 if fileOk_flag:
851 if fileOk_flag:
852 break
852 break
853
853
854 firstTime_flag = False
854 firstTime_flag = False
855
855
856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 self.set += 1
857 self.set += 1
858
858
859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 self.set = 0
860 self.set = 0
861 self.doy += 1
861 self.doy += 1
862 self.foldercounter = 0
862 self.foldercounter = 0
863
863
864 if fileOk_flag:
864 if fileOk_flag:
865 self.fileSize = os.path.getsize( fullfilename )
865 self.fileSize = os.path.getsize( fullfilename )
866 self.filename = fullfilename
866 self.filename = fullfilename
867 self.flagIsNewFile = 1
867 self.flagIsNewFile = 1
868 if self.fp != None: self.fp.close()
868 if self.fp != None: self.fp.close()
869 self.fp = open(fullfilename, 'rb')
869 self.fp = open(fullfilename, 'rb')
870 self.flagNoMoreFiles = 0
870 self.flagNoMoreFiles = 0
871 # print '[Reading] Setting the file: %s' % fullfilename
871 # print '[Reading] Setting the file: %s' % fullfilename
872 else:
872 else:
873 self.fileSize = 0
873 self.fileSize = 0
874 self.filename = None
874 self.filename = None
875 self.flagIsNewFile = 0
875 self.flagIsNewFile = 0
876 self.fp = None
876 self.fp = None
877 self.flagNoMoreFiles = 1
877 self.flagNoMoreFiles = 1
878 # print '[Reading] No more files to read'
878 # print '[Reading] No more files to read'
879
879
880 return fileOk_flag
880 return fileOk_flag
881
881
882 def setNextFile(self):
882 def setNextFile(self):
883 if self.fp != None:
883 if self.fp != None:
884 self.fp.close()
884 self.fp.close()
885
885
886 if self.online:
886 if self.online:
887 newFile = self.__setNextFileOnline()
887 newFile = self.__setNextFileOnline()
888 else:
888 else:
889 newFile = self.__setNextFileOffline()
889 newFile = self.__setNextFileOffline()
890
890
891 if not(newFile):
891 if not(newFile):
892 print '[Reading] No more files to read'
892 print '[Reading] No more files to read'
893 return 0
893 return 0
894
894
895 if self.verbose:
895 if self.verbose:
896 print '[Reading] Setting the file: %s' % self.filename
896 print '[Reading] Setting the file: %s' % self.filename
897
897
898 self.__readFirstHeader()
898 self.__readFirstHeader()
899 self.nReadBlocks = 0
899 self.nReadBlocks = 0
900 return 1
900 return 1
901
901
902 def __waitNewBlock(self):
902 def __waitNewBlock(self):
903 """
903 """
904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
905
905
906 Si el modo de lectura es OffLine siempre retorn 0
906 Si el modo de lectura es OffLine siempre retorn 0
907 """
907 """
908 if not self.online:
908 if not self.online:
909 return 0
909 return 0
910
910
911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
912 return 0
912 return 0
913
913
914 currentPointer = self.fp.tell()
914 currentPointer = self.fp.tell()
915
915
916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917
917
918 for nTries in range( self.nTries ):
918 for nTries in range( self.nTries ):
919
919
920 self.fp.close()
920 self.fp.close()
921 self.fp = open( self.filename, 'rb' )
921 self.fp = open( self.filename, 'rb' )
922 self.fp.seek( currentPointer )
922 self.fp.seek( currentPointer )
923
923
924 self.fileSize = os.path.getsize( self.filename )
924 self.fileSize = os.path.getsize( self.filename )
925 currentSize = self.fileSize - currentPointer
925 currentSize = self.fileSize - currentPointer
926
926
927 if ( currentSize >= neededSize ):
927 if ( currentSize >= neededSize ):
928 self.basicHeaderObj.read(self.fp)
928 self.basicHeaderObj.read(self.fp)
929 return 1
929 return 1
930
930
931 if self.fileSize == self.fileSizeByHeader:
931 if self.fileSize == self.fileSizeByHeader:
932 # self.flagEoF = True
932 # self.flagEoF = True
933 return 0
933 return 0
934
934
935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
936 sleep( self.delay )
936 sleep( self.delay )
937
937
938
938
939 return 0
939 return 0
940
940
941 def waitDataBlock(self,pointer_location):
941 def waitDataBlock(self,pointer_location):
942
942
943 currentPointer = pointer_location
943 currentPointer = pointer_location
944
944
945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
946
946
947 for nTries in range( self.nTries ):
947 for nTries in range( self.nTries ):
948 self.fp.close()
948 self.fp.close()
949 self.fp = open( self.filename, 'rb' )
949 self.fp = open( self.filename, 'rb' )
950 self.fp.seek( currentPointer )
950 self.fp.seek( currentPointer )
951
951
952 self.fileSize = os.path.getsize( self.filename )
952 self.fileSize = os.path.getsize( self.filename )
953 currentSize = self.fileSize - currentPointer
953 currentSize = self.fileSize - currentPointer
954
954
955 if ( currentSize >= neededSize ):
955 if ( currentSize >= neededSize ):
956 return 1
956 return 1
957
957
958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
959 sleep( self.delay )
959 sleep( self.delay )
960
960
961 return 0
961 return 0
962
962
963 def __jumpToLastBlock(self):
963 def __jumpToLastBlock(self):
964
964
965 if not(self.__isFirstTimeOnline):
965 if not(self.__isFirstTimeOnline):
966 return
966 return
967
967
968 csize = self.fileSize - self.fp.tell()
968 csize = self.fileSize - self.fp.tell()
969 blocksize = self.processingHeaderObj.blockSize
969 blocksize = self.processingHeaderObj.blockSize
970
970
971 #salta el primer bloque de datos
971 #salta el primer bloque de datos
972 if csize > self.processingHeaderObj.blockSize:
972 if csize > self.processingHeaderObj.blockSize:
973 self.fp.seek(self.fp.tell() + blocksize)
973 self.fp.seek(self.fp.tell() + blocksize)
974 else:
974 else:
975 return
975 return
976
976
977 csize = self.fileSize - self.fp.tell()
977 csize = self.fileSize - self.fp.tell()
978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
979 while True:
979 while True:
980
980
981 if self.fp.tell()<self.fileSize:
981 if self.fp.tell()<self.fileSize:
982 self.fp.seek(self.fp.tell() + neededsize)
982 self.fp.seek(self.fp.tell() + neededsize)
983 else:
983 else:
984 self.fp.seek(self.fp.tell() - neededsize)
984 self.fp.seek(self.fp.tell() - neededsize)
985 break
985 break
986
986
987 # csize = self.fileSize - self.fp.tell()
987 # csize = self.fileSize - self.fp.tell()
988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
989 # factor = int(csize/neededsize)
989 # factor = int(csize/neededsize)
990 # if factor > 0:
990 # if factor > 0:
991 # self.fp.seek(self.fp.tell() + factor*neededsize)
991 # self.fp.seek(self.fp.tell() + factor*neededsize)
992
992
993 self.flagIsNewFile = 0
993 self.flagIsNewFile = 0
994 self.__isFirstTimeOnline = 0
994 self.__isFirstTimeOnline = 0
995
995
996 def __setNewBlock(self):
996 def __setNewBlock(self):
997 #if self.server is None:
997 #if self.server is None:
998 if self.fp == None:
998 if self.fp == None:
999 return 0
999 return 0
1000
1000
1001 # if self.online:
1001 # if self.online:
1002 # self.__jumpToLastBlock()
1002 # self.__jumpToLastBlock()
1003 print 'xxxx'
1004
1003
1005 if self.flagIsNewFile:
1004 if self.flagIsNewFile:
1006 self.lastUTTime = self.basicHeaderObj.utc
1005 self.lastUTTime = self.basicHeaderObj.utc
1007 return 1
1006 return 1
1008
1007
1009 if self.realtime:
1008 if self.realtime:
1010 self.flagDiscontinuousBlock = 1
1009 self.flagDiscontinuousBlock = 1
1011 if not(self.setNextFile()):
1010 if not(self.setNextFile()):
1012 return 0
1011 return 0
1013 else:
1012 else:
1014 return 1
1013 return 1
1015 print 'xxxx'
1016 #if self.server is None:
1014 #if self.server is None:
1017 currentSize = self.fileSize - self.fp.tell()
1015 currentSize = self.fileSize - self.fp.tell()
1018 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1016 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1019 if (currentSize >= neededSize):
1017 if (currentSize >= neededSize):
1020 self.basicHeaderObj.read(self.fp)
1018 self.basicHeaderObj.read(self.fp)
1021 self.lastUTTime = self.basicHeaderObj.utc
1019 self.lastUTTime = self.basicHeaderObj.utc
1022 return 1
1020 return 1
1023 # else:
1021 # else:
1024 # self.basicHeaderObj.read(self.zHeader)
1022 # self.basicHeaderObj.read(self.zHeader)
1025 # self.lastUTTime = self.basicHeaderObj.utc
1023 # self.lastUTTime = self.basicHeaderObj.utc
1026 # return 1
1024 # return 1
1027 if self.__waitNewBlock():
1025 if self.__waitNewBlock():
1028 self.lastUTTime = self.basicHeaderObj.utc
1026 self.lastUTTime = self.basicHeaderObj.utc
1029 return 1
1027 return 1
1030 #if self.server is None:
1028 #if self.server is None:
1031 if not(self.setNextFile()):
1029 if not(self.setNextFile()):
1032 return 0
1030 return 0
1033
1031
1034 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1032 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1035 self.lastUTTime = self.basicHeaderObj.utc
1033 self.lastUTTime = self.basicHeaderObj.utc
1036
1034
1037 self.flagDiscontinuousBlock = 0
1035 self.flagDiscontinuousBlock = 0
1038
1036
1039 if deltaTime > self.maxTimeStep:
1037 if deltaTime > self.maxTimeStep:
1040 self.flagDiscontinuousBlock = 1
1038 self.flagDiscontinuousBlock = 1
1041
1039
1042 return 1
1040 return 1
1043
1041
1044 def readNextBlock(self):
1042 def readNextBlock(self):
1045
1043
1046 #Skip block out of startTime and endTime
1044 #Skip block out of startTime and endTime
1047 while True:
1045 while True:
1048 print 'cxxxx'
1049 if not(self.__setNewBlock()):
1046 if not(self.__setNewBlock()):
1050 print 'returning'
1047 print 'returning'
1051 return 0
1048 return 0
1052 print 'dxxx'
1049
1053 if not(self.readBlock()):
1050 if not(self.readBlock()):
1054 return 0
1051 return 0
1055
1052
1056 self.getBasicHeader()
1053 self.getBasicHeader()
1057
1054
1058 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1055 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1059
1056
1060 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1057 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1061 self.processingHeaderObj.dataBlocksPerFile,
1058 self.processingHeaderObj.dataBlocksPerFile,
1062 self.dataOut.datatime.ctime())
1059 self.dataOut.datatime.ctime())
1063 continue
1060 continue
1064
1061
1065 break
1062 break
1066
1063
1067 if self.verbose:
1064 if self.verbose:
1068 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1065 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1069 self.processingHeaderObj.dataBlocksPerFile,
1066 self.processingHeaderObj.dataBlocksPerFile,
1070 self.dataOut.datatime.ctime())
1067 self.dataOut.datatime.ctime())
1071 return 1
1068 return 1
1072
1069
1073 def __readFirstHeader(self):
1070 def __readFirstHeader(self):
1074
1071
1075 self.basicHeaderObj.read(self.fp)
1072 self.basicHeaderObj.read(self.fp)
1076 self.systemHeaderObj.read(self.fp)
1073 self.systemHeaderObj.read(self.fp)
1077 self.radarControllerHeaderObj.read(self.fp)
1074 self.radarControllerHeaderObj.read(self.fp)
1078 self.processingHeaderObj.read(self.fp)
1075 self.processingHeaderObj.read(self.fp)
1079
1076
1080 self.firstHeaderSize = self.basicHeaderObj.size
1077 self.firstHeaderSize = self.basicHeaderObj.size
1081
1078
1082 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1079 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1083 if datatype == 0:
1080 if datatype == 0:
1084 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1081 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1085 elif datatype == 1:
1082 elif datatype == 1:
1086 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1083 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1087 elif datatype == 2:
1084 elif datatype == 2:
1088 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1085 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1089 elif datatype == 3:
1086 elif datatype == 3:
1090 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1087 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1091 elif datatype == 4:
1088 elif datatype == 4:
1092 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1089 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1093 elif datatype == 5:
1090 elif datatype == 5:
1094 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1091 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1095 else:
1092 else:
1096 raise ValueError, 'Data type was not defined'
1093 raise ValueError, 'Data type was not defined'
1097
1094
1098 self.dtype = datatype_str
1095 self.dtype = datatype_str
1099 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1096 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1100 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1097 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1101 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1098 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1102 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1099 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1103 self.getBlockDimension()
1100 self.getBlockDimension()
1104
1101
1105 def __verifyFile(self, filename, msgFlag=True):
1102 def __verifyFile(self, filename, msgFlag=True):
1106
1103
1107 msg = None
1104 msg = None
1108
1105
1109 try:
1106 try:
1110 fp = open(filename, 'rb')
1107 fp = open(filename, 'rb')
1111 except IOError:
1108 except IOError:
1112
1109
1113 if msgFlag:
1110 if msgFlag:
1114 print "[Reading] File %s can't be opened" % (filename)
1111 print "[Reading] File %s can't be opened" % (filename)
1115
1112
1116 return False
1113 return False
1117
1114
1118 currentPosition = fp.tell()
1115 currentPosition = fp.tell()
1119 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1116 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1120
1117
1121 if neededSize == 0:
1118 if neededSize == 0:
1122 basicHeaderObj = BasicHeader(LOCALTIME)
1119 basicHeaderObj = BasicHeader(LOCALTIME)
1123 systemHeaderObj = SystemHeader()
1120 systemHeaderObj = SystemHeader()
1124 radarControllerHeaderObj = RadarControllerHeader()
1121 radarControllerHeaderObj = RadarControllerHeader()
1125 processingHeaderObj = ProcessingHeader()
1122 processingHeaderObj = ProcessingHeader()
1126
1123
1127 if not( basicHeaderObj.read(fp) ):
1124 if not( basicHeaderObj.read(fp) ):
1128 fp.close()
1125 fp.close()
1129 return False
1126 return False
1130
1127
1131 if not( systemHeaderObj.read(fp) ):
1128 if not( systemHeaderObj.read(fp) ):
1132 fp.close()
1129 fp.close()
1133 return False
1130 return False
1134
1131
1135 if not( radarControllerHeaderObj.read(fp) ):
1132 if not( radarControllerHeaderObj.read(fp) ):
1136 fp.close()
1133 fp.close()
1137 return False
1134 return False
1138
1135
1139 if not( processingHeaderObj.read(fp) ):
1136 if not( processingHeaderObj.read(fp) ):
1140 fp.close()
1137 fp.close()
1141 return False
1138 return False
1142
1139
1143 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1140 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1144 else:
1141 else:
1145 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1142 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1146
1143
1147 fp.close()
1144 fp.close()
1148
1145
1149 fileSize = os.path.getsize(filename)
1146 fileSize = os.path.getsize(filename)
1150 currentSize = fileSize - currentPosition
1147 currentSize = fileSize - currentPosition
1151
1148
1152 if currentSize < neededSize:
1149 if currentSize < neededSize:
1153 if msgFlag and (msg != None):
1150 if msgFlag and (msg != None):
1154 print msg
1151 print msg
1155 return False
1152 return False
1156
1153
1157 return True
1154 return True
1158
1155
1159 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1156 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1160
1157
1161 path_empty = True
1158 path_empty = True
1162
1159
1163 dateList = []
1160 dateList = []
1164 pathList = []
1161 pathList = []
1165
1162
1166 multi_path = path.split(',')
1163 multi_path = path.split(',')
1167
1164
1168 if not walk:
1165 if not walk:
1169
1166
1170 for single_path in multi_path:
1167 for single_path in multi_path:
1171
1168
1172 if not os.path.isdir(single_path):
1169 if not os.path.isdir(single_path):
1173 continue
1170 continue
1174
1171
1175 fileList = glob.glob1(single_path, "*"+ext)
1172 fileList = glob.glob1(single_path, "*"+ext)
1176
1173
1177 if not fileList:
1174 if not fileList:
1178 continue
1175 continue
1179
1176
1180 path_empty = False
1177 path_empty = False
1181
1178
1182 fileList.sort()
1179 fileList.sort()
1183
1180
1184 for thisFile in fileList:
1181 for thisFile in fileList:
1185
1182
1186 if not os.path.isfile(os.path.join(single_path, thisFile)):
1183 if not os.path.isfile(os.path.join(single_path, thisFile)):
1187 continue
1184 continue
1188
1185
1189 if not isRadarFile(thisFile):
1186 if not isRadarFile(thisFile):
1190 continue
1187 continue
1191
1188
1192 if not isFileInDateRange(thisFile, startDate, endDate):
1189 if not isFileInDateRange(thisFile, startDate, endDate):
1193 continue
1190 continue
1194
1191
1195 thisDate = getDateFromRadarFile(thisFile)
1192 thisDate = getDateFromRadarFile(thisFile)
1196
1193
1197 if thisDate in dateList:
1194 if thisDate in dateList:
1198 continue
1195 continue
1199
1196
1200 dateList.append(thisDate)
1197 dateList.append(thisDate)
1201 pathList.append(single_path)
1198 pathList.append(single_path)
1202
1199
1203 else:
1200 else:
1204 for single_path in multi_path:
1201 for single_path in multi_path:
1205
1202
1206 if not os.path.isdir(single_path):
1203 if not os.path.isdir(single_path):
1207 continue
1204 continue
1208
1205
1209 dirList = []
1206 dirList = []
1210
1207
1211 for thisPath in os.listdir(single_path):
1208 for thisPath in os.listdir(single_path):
1212
1209
1213 if not os.path.isdir(os.path.join(single_path,thisPath)):
1210 if not os.path.isdir(os.path.join(single_path,thisPath)):
1214 continue
1211 continue
1215
1212
1216 if not isRadarFolder(thisPath):
1213 if not isRadarFolder(thisPath):
1217 continue
1214 continue
1218
1215
1219 if not isFolderInDateRange(thisPath, startDate, endDate):
1216 if not isFolderInDateRange(thisPath, startDate, endDate):
1220 continue
1217 continue
1221
1218
1222 dirList.append(thisPath)
1219 dirList.append(thisPath)
1223
1220
1224 if not dirList:
1221 if not dirList:
1225 continue
1222 continue
1226
1223
1227 dirList.sort()
1224 dirList.sort()
1228
1225
1229 for thisDir in dirList:
1226 for thisDir in dirList:
1230
1227
1231 datapath = os.path.join(single_path, thisDir, expLabel)
1228 datapath = os.path.join(single_path, thisDir, expLabel)
1232 fileList = glob.glob1(datapath, "*"+ext)
1229 fileList = glob.glob1(datapath, "*"+ext)
1233
1230
1234 if not fileList:
1231 if not fileList:
1235 continue
1232 continue
1236
1233
1237 path_empty = False
1234 path_empty = False
1238
1235
1239 thisDate = getDateFromRadarFolder(thisDir)
1236 thisDate = getDateFromRadarFolder(thisDir)
1240
1237
1241 pathList.append(datapath)
1238 pathList.append(datapath)
1242 dateList.append(thisDate)
1239 dateList.append(thisDate)
1243
1240
1244 dateList.sort()
1241 dateList.sort()
1245
1242
1246 if walk:
1243 if walk:
1247 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1244 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1248 else:
1245 else:
1249 pattern_path = multi_path[0]
1246 pattern_path = multi_path[0]
1250
1247
1251 if path_empty:
1248 if path_empty:
1252 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1249 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1253 else:
1250 else:
1254 if not dateList:
1251 if not dateList:
1255 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1252 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1256
1253
1257 if include_path:
1254 if include_path:
1258 return dateList, pathList
1255 return dateList, pathList
1259
1256
1260 return dateList
1257 return dateList
1261
1258
1262 def setup(self,
1259 def setup(self,
1263 path=None,
1260 path=None,
1264 startDate=None,
1261 startDate=None,
1265 endDate=None,
1262 endDate=None,
1266 startTime=datetime.time(0,0,0),
1263 startTime=datetime.time(0,0,0),
1267 endTime=datetime.time(23,59,59),
1264 endTime=datetime.time(23,59,59),
1268 set=None,
1265 set=None,
1269 expLabel = "",
1266 expLabel = "",
1270 ext = None,
1267 ext = None,
1271 online = False,
1268 online = False,
1272 delay = 60,
1269 delay = 60,
1273 walk = True,
1270 walk = True,
1274 getblock = False,
1271 getblock = False,
1275 nTxs = 1,
1272 nTxs = 1,
1276 realtime=False,
1273 realtime=False,
1277 blocksize=None,
1274 blocksize=None,
1278 blocktime=None,
1275 blocktime=None,
1279 queue=None,
1276 queue=None,
1280 skip=None,
1277 skip=None,
1281 cursor=None,
1278 cursor=None,
1282 warnings=True,
1279 warnings=True,
1283 verbose=True,
1280 verbose=True,
1284 server=None):
1281 server=None):
1285 if server is not None:
1282 if server is not None:
1286 if 'tcp://' in server:
1283 if 'tcp://' in server:
1287 address = server
1284 address = server
1288 else:
1285 else:
1289 address = 'ipc:///tmp/%s' % server
1286 address = 'ipc:///tmp/%s' % server
1290 self.server = address
1287 self.server = address
1291 self.context = zmq.Context()
1288 self.context = zmq.Context()
1292 self.receiver = self.context.socket(zmq.PULL)
1289 self.receiver = self.context.socket(zmq.PULL)
1293 self.receiver.connect(self.server)
1290 self.receiver.connect(self.server)
1294 time.sleep(0.5)
1291 time.sleep(0.5)
1295 print '[Starting] ReceiverData from {}'.format(self.server)
1292 print '[Starting] ReceiverData from {}'.format(self.server)
1296 else:
1293 else:
1297 self.server = None
1294 self.server = None
1298 if path == None:
1295 if path == None:
1299 raise ValueError, "[Reading] The path is not valid"
1296 raise ValueError, "[Reading] The path is not valid"
1300
1297
1301 if ext == None:
1298 if ext == None:
1302 ext = self.ext
1299 ext = self.ext
1303
1300
1304 if online:
1301 if online:
1305 print "[Reading] Searching files in online mode..."
1302 print "[Reading] Searching files in online mode..."
1306
1303
1307 for nTries in range( self.nTries ):
1304 for nTries in range( self.nTries ):
1308 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1305 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1309
1306
1310 if fullpath:
1307 if fullpath:
1311 break
1308 break
1312
1309
1313 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1310 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1314 sleep( self.delay )
1311 sleep( self.delay )
1315
1312
1316 if not(fullpath):
1313 if not(fullpath):
1317 print "[Reading] There 'isn't any valid file in %s" % path
1314 print "[Reading] There 'isn't any valid file in %s" % path
1318 return
1315 return
1319
1316
1320 self.year = year
1317 self.year = year
1321 self.doy = doy
1318 self.doy = doy
1322 self.set = set - 1
1319 self.set = set - 1
1323 self.path = path
1320 self.path = path
1324 self.foldercounter = foldercounter
1321 self.foldercounter = foldercounter
1325 last_set = None
1322 last_set = None
1326 else:
1323 else:
1327 print "[Reading] Searching files in offline mode ..."
1324 print "[Reading] Searching files in offline mode ..."
1328 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1329 startTime=startTime, endTime=endTime,
1326 startTime=startTime, endTime=endTime,
1330 set=set, expLabel=expLabel, ext=ext,
1327 set=set, expLabel=expLabel, ext=ext,
1331 walk=walk, cursor=cursor,
1328 walk=walk, cursor=cursor,
1332 skip=skip, queue=queue)
1329 skip=skip, queue=queue)
1333
1330
1334 if not(pathList):
1331 if not(pathList):
1335 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1332 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1336 # datetime.datetime.combine(startDate,startTime).ctime(),
1333 # datetime.datetime.combine(startDate,startTime).ctime(),
1337 # datetime.datetime.combine(endDate,endTime).ctime())
1334 # datetime.datetime.combine(endDate,endTime).ctime())
1338
1335
1339 # sys.exit(-1)
1336 # sys.exit(-1)
1340
1337
1341 self.fileIndex = -1
1338 self.fileIndex = -1
1342 self.pathList = []
1339 self.pathList = []
1343 self.filenameList = []
1340 self.filenameList = []
1344 return
1341 return
1345
1342
1346 self.fileIndex = -1
1343 self.fileIndex = -1
1347 self.pathList = pathList
1344 self.pathList = pathList
1348 self.filenameList = filenameList
1345 self.filenameList = filenameList
1349 file_name = os.path.basename(filenameList[-1])
1346 file_name = os.path.basename(filenameList[-1])
1350 basename, ext = os.path.splitext(file_name)
1347 basename, ext = os.path.splitext(file_name)
1351 last_set = int(basename[-3:])
1348 last_set = int(basename[-3:])
1352
1349
1353 self.online = online
1350 self.online = online
1354 self.realtime = realtime
1351 self.realtime = realtime
1355 self.delay = delay
1352 self.delay = delay
1356 ext = ext.lower()
1353 ext = ext.lower()
1357 self.ext = ext
1354 self.ext = ext
1358 self.getByBlock = getblock
1355 self.getByBlock = getblock
1359 self.nTxs = nTxs
1356 self.nTxs = nTxs
1360 self.startTime = startTime
1357 self.startTime = startTime
1361 self.endTime = endTime
1358 self.endTime = endTime
1362
1359
1363 #Added-----------------
1360 #Added-----------------
1364 self.selBlocksize = blocksize
1361 self.selBlocksize = blocksize
1365 self.selBlocktime = blocktime
1362 self.selBlocktime = blocktime
1366
1363
1367 # Verbose-----------
1364 # Verbose-----------
1368 self.verbose = verbose
1365 self.verbose = verbose
1369 self.warnings = warnings
1366 self.warnings = warnings
1370
1367
1371 if not(self.setNextFile()):
1368 if not(self.setNextFile()):
1372 if (startDate!=None) and (endDate!=None):
1369 if (startDate!=None) and (endDate!=None):
1373 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1370 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1374 elif startDate != None:
1371 elif startDate != None:
1375 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1372 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1376 else:
1373 else:
1377 print "[Reading] No files"
1374 print "[Reading] No files"
1378
1375
1379 self.fileIndex = -1
1376 self.fileIndex = -1
1380 self.pathList = []
1377 self.pathList = []
1381 self.filenameList = []
1378 self.filenameList = []
1382 return
1379 return
1383
1380
1384 # self.getBasicHeader()
1381 # self.getBasicHeader()
1385
1382
1386 if last_set != None:
1383 if last_set != None:
1387 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1384 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1388 return
1385 return
1389
1386
1390 def getBasicHeader(self):
1387 def getBasicHeader(self):
1391
1388
1392 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1389 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1393
1390
1394 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1391 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1395
1392
1396 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1393 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1397
1394
1398 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1395 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1399
1396
1400 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1397 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1401
1398
1402 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1399 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1403
1400
1404 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1401 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1405
1402
1406 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1403 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1407
1404
1408
1405
1409 def getFirstHeader(self):
1406 def getFirstHeader(self):
1410
1407
1411 raise NotImplementedError
1408 raise NotImplementedError
1412
1409
1413 def getData(self):
1410 def getData(self):
1414
1411
1415 raise NotImplementedError
1412 raise NotImplementedError
1416
1413
1417 def hasNotDataInBuffer(self):
1414 def hasNotDataInBuffer(self):
1418
1415
1419 raise NotImplementedError
1416 raise NotImplementedError
1420
1417
1421 def readBlock(self):
1418 def readBlock(self):
1422
1419
1423 raise NotImplementedError
1420 raise NotImplementedError
1424
1421
1425 def isEndProcess(self):
1422 def isEndProcess(self):
1426
1423
1427 return self.flagNoMoreFiles
1424 return self.flagNoMoreFiles
1428
1425
1429 def printReadBlocks(self):
1426 def printReadBlocks(self):
1430
1427
1431 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1428 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1432
1429
1433 def printTotalBlocks(self):
1430 def printTotalBlocks(self):
1434
1431
1435 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1432 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1436
1433
1437 def printNumberOfBlock(self):
1434 def printNumberOfBlock(self):
1438
1435
1439 if self.flagIsNewBlock:
1436 if self.flagIsNewBlock:
1440 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1437 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1441 self.processingHeaderObj.dataBlocksPerFile,
1438 self.processingHeaderObj.dataBlocksPerFile,
1442 self.dataOut.datatime.ctime())
1439 self.dataOut.datatime.ctime())
1443
1440
1444 def printInfo(self):
1441 def printInfo(self):
1445
1442
1446 if self.__printInfo == False:
1443 if self.__printInfo == False:
1447 return
1444 return
1448
1445
1449 self.basicHeaderObj.printInfo()
1446 self.basicHeaderObj.printInfo()
1450 self.systemHeaderObj.printInfo()
1447 self.systemHeaderObj.printInfo()
1451 self.radarControllerHeaderObj.printInfo()
1448 self.radarControllerHeaderObj.printInfo()
1452 self.processingHeaderObj.printInfo()
1449 self.processingHeaderObj.printInfo()
1453
1450
1454 self.__printInfo = False
1451 self.__printInfo = False
1455
1452
1456
1453
1457 def run(self,
1454 def run(self,
1458 path=None,
1455 path=None,
1459 startDate=None,
1456 startDate=None,
1460 endDate=None,
1457 endDate=None,
1461 startTime=datetime.time(0,0,0),
1458 startTime=datetime.time(0,0,0),
1462 endTime=datetime.time(23,59,59),
1459 endTime=datetime.time(23,59,59),
1463 set=None,
1460 set=None,
1464 expLabel = "",
1461 expLabel = "",
1465 ext = None,
1462 ext = None,
1466 online = False,
1463 online = False,
1467 delay = 60,
1464 delay = 60,
1468 walk = True,
1465 walk = True,
1469 getblock = False,
1466 getblock = False,
1470 nTxs = 1,
1467 nTxs = 1,
1471 realtime=False,
1468 realtime=False,
1472 blocksize=None,
1469 blocksize=None,
1473 blocktime=None,
1470 blocktime=None,
1474 queue=None,
1471 queue=None,
1475 skip=None,
1472 skip=None,
1476 cursor=None,
1473 cursor=None,
1477 warnings=True,
1474 warnings=True,
1478 server=None,
1475 server=None,
1479 verbose=True, **kwargs):
1476 verbose=True, **kwargs):
1480
1477
1481 if not(self.isConfig):
1478 if not(self.isConfig):
1482 # self.dataOut = dataOut
1479 # self.dataOut = dataOut
1483 self.setup( path=path,
1480 self.setup( path=path,
1484 startDate=startDate,
1481 startDate=startDate,
1485 endDate=endDate,
1482 endDate=endDate,
1486 startTime=startTime,
1483 startTime=startTime,
1487 endTime=endTime,
1484 endTime=endTime,
1488 set=set,
1485 set=set,
1489 expLabel=expLabel,
1486 expLabel=expLabel,
1490 ext=ext,
1487 ext=ext,
1491 online=online,
1488 online=online,
1492 delay=delay,
1489 delay=delay,
1493 walk=walk,
1490 walk=walk,
1494 getblock=getblock,
1491 getblock=getblock,
1495 nTxs=nTxs,
1492 nTxs=nTxs,
1496 realtime=realtime,
1493 realtime=realtime,
1497 blocksize=blocksize,
1494 blocksize=blocksize,
1498 blocktime=blocktime,
1495 blocktime=blocktime,
1499 queue=queue,
1496 queue=queue,
1500 skip=skip,
1497 skip=skip,
1501 cursor=cursor,
1498 cursor=cursor,
1502 warnings=warnings,
1499 warnings=warnings,
1503 server=server,
1500 server=server,
1504 verbose=verbose)
1501 verbose=verbose)
1505 self.isConfig = True
1502 self.isConfig = True
1506 if server is None:
1503 if server is None:
1507 self.getData()
1504 self.getData()
1508 else:
1505 else:
1509 self.getFromServer()
1506 self.getFromServer()
1510
1507
1511 class JRODataWriter(JRODataIO):
1508 class JRODataWriter(JRODataIO):
1512
1509
1513 """
1510 """
1514 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1511 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1515 de los datos siempre se realiza por bloques.
1512 de los datos siempre se realiza por bloques.
1516 """
1513 """
1517
1514
1518 blockIndex = 0
1515 blockIndex = 0
1519
1516
1520 path = None
1517 path = None
1521
1518
1522 setFile = None
1519 setFile = None
1523
1520
1524 profilesPerBlock = None
1521 profilesPerBlock = None
1525
1522
1526 blocksPerFile = None
1523 blocksPerFile = None
1527
1524
1528 nWriteBlocks = 0
1525 nWriteBlocks = 0
1529
1526
1530 fileDate = None
1527 fileDate = None
1531
1528
1532 def __init__(self, dataOut=None):
1529 def __init__(self, dataOut=None):
1533 raise NotImplementedError
1530 raise NotImplementedError
1534
1531
1535
1532
1536 def hasAllDataInBuffer(self):
1533 def hasAllDataInBuffer(self):
1537 raise NotImplementedError
1534 raise NotImplementedError
1538
1535
1539
1536
1540 def setBlockDimension(self):
1537 def setBlockDimension(self):
1541 raise NotImplementedError
1538 raise NotImplementedError
1542
1539
1543
1540
1544 def writeBlock(self):
1541 def writeBlock(self):
1545 raise NotImplementedError
1542 raise NotImplementedError
1546
1543
1547
1544
1548 def putData(self):
1545 def putData(self):
1549 raise NotImplementedError
1546 raise NotImplementedError
1550
1547
1551
1548
1552 def getProcessFlags(self):
1549 def getProcessFlags(self):
1553
1550
1554 processFlags = 0
1551 processFlags = 0
1555
1552
1556 dtype_index = get_dtype_index(self.dtype)
1553 dtype_index = get_dtype_index(self.dtype)
1557 procflag_dtype = get_procflag_dtype(dtype_index)
1554 procflag_dtype = get_procflag_dtype(dtype_index)
1558
1555
1559 processFlags += procflag_dtype
1556 processFlags += procflag_dtype
1560
1557
1561 if self.dataOut.flagDecodeData:
1558 if self.dataOut.flagDecodeData:
1562 processFlags += PROCFLAG.DECODE_DATA
1559 processFlags += PROCFLAG.DECODE_DATA
1563
1560
1564 if self.dataOut.flagDeflipData:
1561 if self.dataOut.flagDeflipData:
1565 processFlags += PROCFLAG.DEFLIP_DATA
1562 processFlags += PROCFLAG.DEFLIP_DATA
1566
1563
1567 if self.dataOut.code is not None:
1564 if self.dataOut.code is not None:
1568 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1565 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1569
1566
1570 if self.dataOut.nCohInt > 1:
1567 if self.dataOut.nCohInt > 1:
1571 processFlags += PROCFLAG.COHERENT_INTEGRATION
1568 processFlags += PROCFLAG.COHERENT_INTEGRATION
1572
1569
1573 if self.dataOut.type == "Spectra":
1570 if self.dataOut.type == "Spectra":
1574 if self.dataOut.nIncohInt > 1:
1571 if self.dataOut.nIncohInt > 1:
1575 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1572 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1576
1573
1577 if self.dataOut.data_dc is not None:
1574 if self.dataOut.data_dc is not None:
1578 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1575 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1579
1576
1580 if self.dataOut.flagShiftFFT:
1577 if self.dataOut.flagShiftFFT:
1581 processFlags += PROCFLAG.SHIFT_FFT_DATA
1578 processFlags += PROCFLAG.SHIFT_FFT_DATA
1582
1579
1583 return processFlags
1580 return processFlags
1584
1581
1585 def setBasicHeader(self):
1582 def setBasicHeader(self):
1586
1583
1587 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1584 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1588 self.basicHeaderObj.version = self.versionFile
1585 self.basicHeaderObj.version = self.versionFile
1589 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1586 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1590
1587
1591 utc = numpy.floor(self.dataOut.utctime)
1588 utc = numpy.floor(self.dataOut.utctime)
1592 milisecond = (self.dataOut.utctime - utc)* 1000.0
1589 milisecond = (self.dataOut.utctime - utc)* 1000.0
1593
1590
1594 self.basicHeaderObj.utc = utc
1591 self.basicHeaderObj.utc = utc
1595 self.basicHeaderObj.miliSecond = milisecond
1592 self.basicHeaderObj.miliSecond = milisecond
1596 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1593 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1597 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1594 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1598 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1595 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1599
1596
1600 def setFirstHeader(self):
1597 def setFirstHeader(self):
1601 """
1598 """
1602 Obtiene una copia del First Header
1599 Obtiene una copia del First Header
1603
1600
1604 Affected:
1601 Affected:
1605
1602
1606 self.basicHeaderObj
1603 self.basicHeaderObj
1607 self.systemHeaderObj
1604 self.systemHeaderObj
1608 self.radarControllerHeaderObj
1605 self.radarControllerHeaderObj
1609 self.processingHeaderObj self.
1606 self.processingHeaderObj self.
1610
1607
1611 Return:
1608 Return:
1612 None
1609 None
1613 """
1610 """
1614
1611
1615 raise NotImplementedError
1612 raise NotImplementedError
1616
1613
1617 def __writeFirstHeader(self):
1614 def __writeFirstHeader(self):
1618 """
1615 """
1619 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1616 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1620
1617
1621 Affected:
1618 Affected:
1622 __dataType
1619 __dataType
1623
1620
1624 Return:
1621 Return:
1625 None
1622 None
1626 """
1623 """
1627
1624
1628 # CALCULAR PARAMETROS
1625 # CALCULAR PARAMETROS
1629
1626
1630 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1627 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1631 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1628 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1632
1629
1633 self.basicHeaderObj.write(self.fp)
1630 self.basicHeaderObj.write(self.fp)
1634 self.systemHeaderObj.write(self.fp)
1631 self.systemHeaderObj.write(self.fp)
1635 self.radarControllerHeaderObj.write(self.fp)
1632 self.radarControllerHeaderObj.write(self.fp)
1636 self.processingHeaderObj.write(self.fp)
1633 self.processingHeaderObj.write(self.fp)
1637
1634
1638 def __setNewBlock(self):
1635 def __setNewBlock(self):
1639 """
1636 """
1640 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1637 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1641
1638
1642 Return:
1639 Return:
1643 0 : si no pudo escribir nada
1640 0 : si no pudo escribir nada
1644 1 : Si escribio el Basic el First Header
1641 1 : Si escribio el Basic el First Header
1645 """
1642 """
1646 if self.fp == None:
1643 if self.fp == None:
1647 self.setNextFile()
1644 self.setNextFile()
1648
1645
1649 if self.flagIsNewFile:
1646 if self.flagIsNewFile:
1650 return 1
1647 return 1
1651
1648
1652 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1649 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1653 self.basicHeaderObj.write(self.fp)
1650 self.basicHeaderObj.write(self.fp)
1654 return 1
1651 return 1
1655
1652
1656 if not( self.setNextFile() ):
1653 if not( self.setNextFile() ):
1657 return 0
1654 return 0
1658
1655
1659 return 1
1656 return 1
1660
1657
1661
1658
1662 def writeNextBlock(self):
1659 def writeNextBlock(self):
1663 """
1660 """
1664 Selecciona el bloque siguiente de datos y los escribe en un file
1661 Selecciona el bloque siguiente de datos y los escribe en un file
1665
1662
1666 Return:
1663 Return:
1667 0 : Si no hizo pudo escribir el bloque de datos
1664 0 : Si no hizo pudo escribir el bloque de datos
1668 1 : Si no pudo escribir el bloque de datos
1665 1 : Si no pudo escribir el bloque de datos
1669 """
1666 """
1670 if not( self.__setNewBlock() ):
1667 if not( self.__setNewBlock() ):
1671 return 0
1668 return 0
1672
1669
1673 self.writeBlock()
1670 self.writeBlock()
1674
1671
1675 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1672 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1676 self.processingHeaderObj.dataBlocksPerFile)
1673 self.processingHeaderObj.dataBlocksPerFile)
1677
1674
1678 return 1
1675 return 1
1679
1676
1680 def setNextFile(self):
1677 def setNextFile(self):
1681 """
1678 """
1682 Determina el siguiente file que sera escrito
1679 Determina el siguiente file que sera escrito
1683
1680
1684 Affected:
1681 Affected:
1685 self.filename
1682 self.filename
1686 self.subfolder
1683 self.subfolder
1687 self.fp
1684 self.fp
1688 self.setFile
1685 self.setFile
1689 self.flagIsNewFile
1686 self.flagIsNewFile
1690
1687
1691 Return:
1688 Return:
1692 0 : Si el archivo no puede ser escrito
1689 0 : Si el archivo no puede ser escrito
1693 1 : Si el archivo esta listo para ser escrito
1690 1 : Si el archivo esta listo para ser escrito
1694 """
1691 """
1695 ext = self.ext
1692 ext = self.ext
1696 path = self.path
1693 path = self.path
1697
1694
1698 if self.fp != None:
1695 if self.fp != None:
1699 self.fp.close()
1696 self.fp.close()
1700
1697
1701 timeTuple = time.localtime( self.dataOut.utctime)
1698 timeTuple = time.localtime( self.dataOut.utctime)
1702 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1699 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1703
1700
1704 fullpath = os.path.join( path, subfolder )
1701 fullpath = os.path.join( path, subfolder )
1705 setFile = self.setFile
1702 setFile = self.setFile
1706
1703
1707 if not( os.path.exists(fullpath) ):
1704 if not( os.path.exists(fullpath) ):
1708 os.mkdir(fullpath)
1705 os.mkdir(fullpath)
1709 setFile = -1 #inicializo mi contador de seteo
1706 setFile = -1 #inicializo mi contador de seteo
1710 else:
1707 else:
1711 filesList = os.listdir( fullpath )
1708 filesList = os.listdir( fullpath )
1712 if len( filesList ) > 0:
1709 if len( filesList ) > 0:
1713 filesList = sorted( filesList, key=str.lower )
1710 filesList = sorted( filesList, key=str.lower )
1714 filen = filesList[-1]
1711 filen = filesList[-1]
1715 # el filename debera tener el siguiente formato
1712 # el filename debera tener el siguiente formato
1716 # 0 1234 567 89A BCDE (hex)
1713 # 0 1234 567 89A BCDE (hex)
1717 # x YYYY DDD SSS .ext
1714 # x YYYY DDD SSS .ext
1718 if isNumber( filen[8:11] ):
1715 if isNumber( filen[8:11] ):
1719 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1716 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1720 else:
1717 else:
1721 setFile = -1
1718 setFile = -1
1722 else:
1719 else:
1723 setFile = -1 #inicializo mi contador de seteo
1720 setFile = -1 #inicializo mi contador de seteo
1724
1721
1725 setFile += 1
1722 setFile += 1
1726
1723
1727 #If this is a new day it resets some values
1724 #If this is a new day it resets some values
1728 if self.dataOut.datatime.date() > self.fileDate:
1725 if self.dataOut.datatime.date() > self.fileDate:
1729 setFile = 0
1726 setFile = 0
1730 self.nTotalBlocks = 0
1727 self.nTotalBlocks = 0
1731
1728
1732 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1729 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1733
1730
1734 filename = os.path.join( path, subfolder, filen )
1731 filename = os.path.join( path, subfolder, filen )
1735
1732
1736 fp = open( filename,'wb' )
1733 fp = open( filename,'wb' )
1737
1734
1738 self.blockIndex = 0
1735 self.blockIndex = 0
1739
1736
1740 #guardando atributos
1737 #guardando atributos
1741 self.filename = filename
1738 self.filename = filename
1742 self.subfolder = subfolder
1739 self.subfolder = subfolder
1743 self.fp = fp
1740 self.fp = fp
1744 self.setFile = setFile
1741 self.setFile = setFile
1745 self.flagIsNewFile = 1
1742 self.flagIsNewFile = 1
1746 self.fileDate = self.dataOut.datatime.date()
1743 self.fileDate = self.dataOut.datatime.date()
1747
1744
1748 self.setFirstHeader()
1745 self.setFirstHeader()
1749
1746
1750 print '[Writing] Opening file: %s'%self.filename
1747 print '[Writing] Opening file: %s'%self.filename
1751
1748
1752 self.__writeFirstHeader()
1749 self.__writeFirstHeader()
1753
1750
1754 return 1
1751 return 1
1755
1752
1756 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1753 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1757 """
1754 """
1758 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1755 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1759
1756
1760 Inputs:
1757 Inputs:
1761 path : directory where data will be saved
1758 path : directory where data will be saved
1762 profilesPerBlock : number of profiles per block
1759 profilesPerBlock : number of profiles per block
1763 set : initial file set
1760 set : initial file set
1764 datatype : An integer number that defines data type:
1761 datatype : An integer number that defines data type:
1765 0 : int8 (1 byte)
1762 0 : int8 (1 byte)
1766 1 : int16 (2 bytes)
1763 1 : int16 (2 bytes)
1767 2 : int32 (4 bytes)
1764 2 : int32 (4 bytes)
1768 3 : int64 (8 bytes)
1765 3 : int64 (8 bytes)
1769 4 : float32 (4 bytes)
1766 4 : float32 (4 bytes)
1770 5 : double64 (8 bytes)
1767 5 : double64 (8 bytes)
1771
1768
1772 Return:
1769 Return:
1773 0 : Si no realizo un buen seteo
1770 0 : Si no realizo un buen seteo
1774 1 : Si realizo un buen seteo
1771 1 : Si realizo un buen seteo
1775 """
1772 """
1776
1773
1777 if ext == None:
1774 if ext == None:
1778 ext = self.ext
1775 ext = self.ext
1779
1776
1780 self.ext = ext.lower()
1777 self.ext = ext.lower()
1781
1778
1782 self.path = path
1779 self.path = path
1783
1780
1784 if set is None:
1781 if set is None:
1785 self.setFile = -1
1782 self.setFile = -1
1786 else:
1783 else:
1787 self.setFile = set - 1
1784 self.setFile = set - 1
1788
1785
1789 self.blocksPerFile = blocksPerFile
1786 self.blocksPerFile = blocksPerFile
1790
1787
1791 self.profilesPerBlock = profilesPerBlock
1788 self.profilesPerBlock = profilesPerBlock
1792
1789
1793 self.dataOut = dataOut
1790 self.dataOut = dataOut
1794 self.fileDate = self.dataOut.datatime.date()
1791 self.fileDate = self.dataOut.datatime.date()
1795 #By default
1792 #By default
1796 self.dtype = self.dataOut.dtype
1793 self.dtype = self.dataOut.dtype
1797
1794
1798 if datatype is not None:
1795 if datatype is not None:
1799 self.dtype = get_numpy_dtype(datatype)
1796 self.dtype = get_numpy_dtype(datatype)
1800
1797
1801 if not(self.setNextFile()):
1798 if not(self.setNextFile()):
1802 print "[Writing] There isn't a next file"
1799 print "[Writing] There isn't a next file"
1803 return 0
1800 return 0
1804
1801
1805 self.setBlockDimension()
1802 self.setBlockDimension()
1806
1803
1807 return 1
1804 return 1
1808
1805
1809 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1806 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1810
1807
1811 if not(self.isConfig):
1808 if not(self.isConfig):
1812
1809
1813 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1810 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1814 self.isConfig = True
1811 self.isConfig = True
1815
1812
1816 self.putData()
1813 self.putData()
Show file before | Show file after | Hide comments
@@ -1,739 +1,737
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import numpy
7 import numpy
8
8
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13 import zmq
13 import zmq
14 import tempfile
14 import tempfile
15 from StringIO import StringIO
15 from StringIO import StringIO
16 # from _sha import blocksize
16 # from _sha import blocksize
17
17
18 class VoltageReader(JRODataReader, ProcessingUnit):
18 class VoltageReader(JRODataReader, ProcessingUnit):
19 """
19 """
20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
21 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
22 perfiles*alturas*canales) son almacenados en la variable "buffer".
22 perfiles*alturas*canales) son almacenados en la variable "buffer".
23
23
24 perfiles * alturas * canales
24 perfiles * alturas * canales
25
25
26 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
26 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
27 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
27 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
28 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
28 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
29 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
29 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
30
30
31 Example:
31 Example:
32
32
33 dpath = "/home/myuser/data"
33 dpath = "/home/myuser/data"
34
34
35 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
35 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
36
36
37 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
37 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
38
38
39 readerObj = VoltageReader()
39 readerObj = VoltageReader()
40
40
41 readerObj.setup(dpath, startTime, endTime)
41 readerObj.setup(dpath, startTime, endTime)
42
42
43 while(True):
43 while(True):
44
44
45 #to get one profile
45 #to get one profile
46 profile = readerObj.getData()
46 profile = readerObj.getData()
47
47
48 #print the profile
48 #print the profile
49 print profile
49 print profile
50
50
51 #If you want to see all datablock
51 #If you want to see all datablock
52 print readerObj.datablock
52 print readerObj.datablock
53
53
54 if readerObj.flagNoMoreFiles:
54 if readerObj.flagNoMoreFiles:
55 break
55 break
56
56
57 """
57 """
58
58
59 ext = ".r"
59 ext = ".r"
60
60
61 optchar = "D"
61 optchar = "D"
62 dataOut = None
62 dataOut = None
63
63
64 def __init__(self, **kwargs):
64 def __init__(self, **kwargs):
65 """
65 """
66 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
66 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
67
67
68 Input:
68 Input:
69 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
69 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
70 almacenar un perfil de datos cada vez que se haga un requerimiento
70 almacenar un perfil de datos cada vez que se haga un requerimiento
71 (getData). El perfil sera obtenido a partir del buffer de datos,
71 (getData). El perfil sera obtenido a partir del buffer de datos,
72 si el buffer esta vacio se hara un nuevo proceso de lectura de un
72 si el buffer esta vacio se hara un nuevo proceso de lectura de un
73 bloque de datos.
73 bloque de datos.
74 Si este parametro no es pasado se creara uno internamente.
74 Si este parametro no es pasado se creara uno internamente.
75
75
76 Variables afectadas:
76 Variables afectadas:
77 self.dataOut
77 self.dataOut
78
78
79 Return:
79 Return:
80 None
80 None
81 """
81 """
82
82
83 ProcessingUnit.__init__(self, **kwargs)
83 ProcessingUnit.__init__(self, **kwargs)
84
84
85 self.isConfig = False
85 self.isConfig = False
86
86
87 self.datablock = None
87 self.datablock = None
88
88
89 self.utc = 0
89 self.utc = 0
90
90
91 self.ext = ".r"
91 self.ext = ".r"
92
92
93 self.optchar = "D"
93 self.optchar = "D"
94
94
95 self.basicHeaderObj = BasicHeader(LOCALTIME)
95 self.basicHeaderObj = BasicHeader(LOCALTIME)
96
96
97 self.systemHeaderObj = SystemHeader()
97 self.systemHeaderObj = SystemHeader()
98
98
99 self.radarControllerHeaderObj = RadarControllerHeader()
99 self.radarControllerHeaderObj = RadarControllerHeader()
100
100
101 self.processingHeaderObj = ProcessingHeader()
101 self.processingHeaderObj = ProcessingHeader()
102
102
103 self.online = 0
103 self.online = 0
104
104
105 self.fp = None
105 self.fp = None
106
106
107 self.idFile = None
107 self.idFile = None
108
108
109 self.dtype = None
109 self.dtype = None
110
110
111 self.fileSizeByHeader = None
111 self.fileSizeByHeader = None
112
112
113 self.filenameList = []
113 self.filenameList = []
114
114
115 self.filename = None
115 self.filename = None
116
116
117 self.fileSize = None
117 self.fileSize = None
118
118
119 self.firstHeaderSize = 0
119 self.firstHeaderSize = 0
120
120
121 self.basicHeaderSize = 24
121 self.basicHeaderSize = 24
122
122
123 self.pathList = []
123 self.pathList = []
124
124
125 self.filenameList = []
125 self.filenameList = []
126
126
127 self.lastUTTime = 0
127 self.lastUTTime = 0
128
128
129 self.maxTimeStep = 30
129 self.maxTimeStep = 30
130
130
131 self.flagNoMoreFiles = 0
131 self.flagNoMoreFiles = 0
132
132
133 self.set = 0
133 self.set = 0
134
134
135 self.path = None
135 self.path = None
136
136
137 self.profileIndex = 2**32-1
137 self.profileIndex = 2**32-1
138
138
139 self.delay = 3 #seconds
139 self.delay = 3 #seconds
140
140
141 self.nTries = 3 #quantity tries
141 self.nTries = 3 #quantity tries
142
142
143 self.nFiles = 3 #number of files for searching
143 self.nFiles = 3 #number of files for searching
144
144
145 self.nReadBlocks = 0
145 self.nReadBlocks = 0
146
146
147 self.flagIsNewFile = 1
147 self.flagIsNewFile = 1
148
148
149 self.__isFirstTimeOnline = 1
149 self.__isFirstTimeOnline = 1
150
150
151 # self.ippSeconds = 0
151 # self.ippSeconds = 0
152
152
153 self.flagDiscontinuousBlock = 0
153 self.flagDiscontinuousBlock = 0
154
154
155 self.flagIsNewBlock = 0
155 self.flagIsNewBlock = 0
156
156
157 self.nTotalBlocks = 0
157 self.nTotalBlocks = 0
158
158
159 self.blocksize = 0
159 self.blocksize = 0
160
160
161 self.dataOut = self.createObjByDefault()
161 self.dataOut = self.createObjByDefault()
162
162
163 self.nTxs = 1
163 self.nTxs = 1
164
164
165 self.txIndex = 0
165 self.txIndex = 0
166
166
167 def createObjByDefault(self):
167 def createObjByDefault(self):
168
168
169 dataObj = Voltage()
169 dataObj = Voltage()
170
170
171 return dataObj
171 return dataObj
172
172
173 def __hasNotDataInBuffer(self):
173 def __hasNotDataInBuffer(self):
174
174
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
176 return 1
176 return 1
177
177
178 return 0
178 return 0
179
179
180
180
181 def getBlockDimension(self):
181 def getBlockDimension(self):
182 """
182 """
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184
184
185 Affected:
185 Affected:
186 self.blocksize
186 self.blocksize
187
187
188 Return:
188 Return:
189 None
189 None
190 """
190 """
191 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
191 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
192 self.blocksize = pts2read
192 self.blocksize = pts2read
193
193
194
194
195
195
196 def readBlock(self):
196 def readBlock(self):
197 """
197 """
198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
199 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
199 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
200 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
200 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
201 es seteado a 0
201 es seteado a 0
202
202
203 Inputs:
203 Inputs:
204 None
204 None
205
205
206 Return:
206 Return:
207 None
207 None
208
208
209 Affected:
209 Affected:
210 self.profileIndex
210 self.profileIndex
211 self.datablock
211 self.datablock
212 self.flagIsNewFile
212 self.flagIsNewFile
213 self.flagIsNewBlock
213 self.flagIsNewBlock
214 self.nTotalBlocks
214 self.nTotalBlocks
215
215
216 Exceptions:
216 Exceptions:
217 Si un bloque leido no es un bloque valido
217 Si un bloque leido no es un bloque valido
218 """
218 """
219
219
220 print 'READ BLOCK'
221 # if self.server is not None:
220 # if self.server is not None:
222 # self.zBlock = self.receiver.recv()
221 # self.zBlock = self.receiver.recv()
223 # self.zHeader = self.zBlock[:24]
222 # self.zHeader = self.zBlock[:24]
224 # self.zDataBlock = self.zBlock[24:]
223 # self.zDataBlock = self.zBlock[24:]
225 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
224 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
226 # self.processingHeaderObj.profilesPerBlock = 240
225 # self.processingHeaderObj.profilesPerBlock = 240
227 # self.processingHeaderObj.nHeights = 248
226 # self.processingHeaderObj.nHeights = 248
228 # self.systemHeaderObj.nChannels
227 # self.systemHeaderObj.nChannels
229 # else:
228 # else:
230 current_pointer_location = self.fp.tell()
229 current_pointer_location = self.fp.tell()
231 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
230 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
232
231
233 try:
232 try:
234 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
233 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
235 print'junked'
236 except:
234 except:
237 #print "The read block (%3d) has not enough data" %self.nReadBlocks
235 #print "The read block (%3d) has not enough data" %self.nReadBlocks
238
236
239 if self.waitDataBlock(pointer_location=current_pointer_location):
237 if self.waitDataBlock(pointer_location=current_pointer_location):
240 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
238 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
241 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
239 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
242 # return 0
240 # return 0
243
241
244 #Dimensions : nChannels, nProfiles, nSamples
242 #Dimensions : nChannels, nProfiles, nSamples
245
243
246 junk = numpy.transpose(junk, (2,0,1))
244 junk = numpy.transpose(junk, (2,0,1))
247 self.datablock = junk['real'] + junk['imag']*1j
245 self.datablock = junk['real'] + junk['imag']*1j
248
246
249 self.profileIndex = 0
247 self.profileIndex = 0
250
248
251 self.flagIsNewFile = 0
249 self.flagIsNewFile = 0
252 self.flagIsNewBlock = 1
250 self.flagIsNewBlock = 1
253
251
254 self.nTotalBlocks += 1
252 self.nTotalBlocks += 1
255 self.nReadBlocks += 1
253 self.nReadBlocks += 1
256
254
257 return 1
255 return 1
258
256
259 def getFirstHeader(self):
257 def getFirstHeader(self):
260
258
261 self.getBasicHeader()
259 self.getBasicHeader()
262
260
263 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
261 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
264
262
265 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
263 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
266
264
267 if self.nTxs > 1:
265 if self.nTxs > 1:
268 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
266 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
269
267
270 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
268 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
271
269
272 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
270 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
273 #
271 #
274 # if self.radarControllerHeaderObj.code is not None:
272 # if self.radarControllerHeaderObj.code is not None:
275 #
273 #
276 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
274 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
277 #
275 #
278 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
276 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
279 #
277 #
280 # self.dataOut.code = self.radarControllerHeaderObj.code
278 # self.dataOut.code = self.radarControllerHeaderObj.code
281
279
282 self.dataOut.dtype = self.dtype
280 self.dataOut.dtype = self.dtype
283
281
284 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
282 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
285
283
286 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
284 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
287
285
288 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
286 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
289
287
290 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
288 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
291
289
292 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
290 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
293
291
294 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
292 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
295
293
296 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
294 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
297
295
298 def reshapeData(self):
296 def reshapeData(self):
299
297
300 if self.nTxs < 0:
298 if self.nTxs < 0:
301 return
299 return
302
300
303 if self.nTxs == 1:
301 if self.nTxs == 1:
304 return
302 return
305
303
306 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
304 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
307 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
305 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
308
306
309 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
307 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
310 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
308 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
311
309
312 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
310 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
313
311
314 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
312 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
315 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
313 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
316 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
314 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
317
315
318 return
316 return
319
317
320 def readFirstHeaderFromServer(self):
318 def readFirstHeaderFromServer(self):
321
319
322 self.getFirstHeader()
320 self.getFirstHeader()
323
321
324 self.firstHeaderSize = self.basicHeaderObj.size
322 self.firstHeaderSize = self.basicHeaderObj.size
325
323
326 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
324 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
327 if datatype == 0:
325 if datatype == 0:
328 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
326 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
329 elif datatype == 1:
327 elif datatype == 1:
330 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
328 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
331 elif datatype == 2:
329 elif datatype == 2:
332 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
330 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
333 elif datatype == 3:
331 elif datatype == 3:
334 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
332 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
335 elif datatype == 4:
333 elif datatype == 4:
336 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
334 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
337 elif datatype == 5:
335 elif datatype == 5:
338 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
336 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
339 else:
337 else:
340 raise ValueError, 'Data type was not defined'
338 raise ValueError, 'Data type was not defined'
341
339
342 self.dtype = datatype_str
340 self.dtype = datatype_str
343 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
341 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
344 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
342 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
345 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
343 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
346 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
344 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
347 self.getBlockDimension()
345 self.getBlockDimension()
348
346
349
347
350 def getFromServer(self):
348 def getFromServer(self):
351 self.flagDiscontinuousBlock = 0
349 self.flagDiscontinuousBlock = 0
352 self.profileIndex = 0
350 self.profileIndex = 0
353 self.flagIsNewBlock = 1
351 self.flagIsNewBlock = 1
354 self.dataOut.flagNoData = False
352 self.dataOut.flagNoData = False
355 self.nTotalBlocks += 1
353 self.nTotalBlocks += 1
356 self.nReadBlocks += 1
354 self.nReadBlocks += 1
357 self.blockPointer = 0
355 self.blockPointer = 0
358
356
359 block = self.receiver.recv()
357 block = self.receiver.recv()
360
358
361 self.basicHeaderObj.read(block[self.blockPointer:])
359 self.basicHeaderObj.read(block[self.blockPointer:])
362 self.blockPointer += self.basicHeaderObj.length
360 self.blockPointer += self.basicHeaderObj.length
363 self.systemHeaderObj.read(block[self.blockPointer:])
361 self.systemHeaderObj.read(block[self.blockPointer:])
364 self.blockPointer += self.systemHeaderObj.length
362 self.blockPointer += self.systemHeaderObj.length
365 self.radarControllerHeaderObj.read(block[self.blockPointer:])
363 self.radarControllerHeaderObj.read(block[self.blockPointer:])
366 self.blockPointer += self.radarControllerHeaderObj.length
364 self.blockPointer += self.radarControllerHeaderObj.length
367 self.processingHeaderObj.read(block[self.blockPointer:])
365 self.processingHeaderObj.read(block[self.blockPointer:])
368 self.blockPointer += self.processingHeaderObj.length
366 self.blockPointer += self.processingHeaderObj.length
369 self.readFirstHeaderFromServer()
367 self.readFirstHeaderFromServer()
370
368
371 timestamp = self.basicHeaderObj.get_datatime()
369 timestamp = self.basicHeaderObj.get_datatime()
372 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
370 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
373 current_pointer_location = self.blockPointer
371 current_pointer_location = self.blockPointer
374 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
372 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
375
373
376 try:
374 try:
377 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
375 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
378 except:
376 except:
379 #print "The read block (%3d) has not enough data" %self.nReadBlocks
377 #print "The read block (%3d) has not enough data" %self.nReadBlocks
380 if self.waitDataBlock(pointer_location=current_pointer_location):
378 if self.waitDataBlock(pointer_location=current_pointer_location):
381 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
379 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
382 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
380 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
383 # return 0
381 # return 0
384
382
385 #Dimensions : nChannels, nProfiles, nSamples
383 #Dimensions : nChannels, nProfiles, nSamples
386
384
387 junk = numpy.transpose(junk, (2,0,1))
385 junk = numpy.transpose(junk, (2,0,1))
388 self.datablock = junk['real'] + junk['imag'] * 1j
386 self.datablock = junk['real'] + junk['imag'] * 1j
389 self.profileIndex = 0
387 self.profileIndex = 0
390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
388 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 if self.selBlocktime != None:
389 if self.selBlocktime != None:
392 if self.dataOut.nCohInt is not None:
390 if self.dataOut.nCohInt is not None:
393 nCohInt = self.dataOut.nCohInt
391 nCohInt = self.dataOut.nCohInt
394 else:
392 else:
395 nCohInt = 1
393 nCohInt = 1
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
394 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
395 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
398 datasize = self.dataOut.data.shape[1]
396 datasize = self.dataOut.data.shape[1]
399 if datasize < self.selBlocksize:
397 if datasize < self.selBlocksize:
400 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
398 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
401 buffer[:,:datasize,:] = self.dataOut.data
399 buffer[:,:datasize,:] = self.dataOut.data
402 self.dataOut.data = buffer
400 self.dataOut.data = buffer
403 self.profileIndex = blockIndex
401 self.profileIndex = blockIndex
404
402
405 self.dataOut.flagDataAsBlock = True
403 self.dataOut.flagDataAsBlock = True
406 self.flagIsNewBlock = 1
404 self.flagIsNewBlock = 1
407 self.dataOut.realtime = self.online
405 self.dataOut.realtime = self.online
408
406
409 return self.dataOut.data
407 return self.dataOut.data
410
408
411 def getData(self):
409 def getData(self):
412 """
410 """
413 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
411 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
414 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
412 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
415 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
413 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
416 "readNextBlock"
414 "readNextBlock"
417
415
418 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
416 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
419
417
420 Return:
418 Return:
421
419
422 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
420 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
423 es igual al total de perfiles leidos desde el archivo.
421 es igual al total de perfiles leidos desde el archivo.
424
422
425 Si self.getByBlock == False:
423 Si self.getByBlock == False:
426
424
427 self.dataOut.data = buffer[:, thisProfile, :]
425 self.dataOut.data = buffer[:, thisProfile, :]
428
426
429 shape = [nChannels, nHeis]
427 shape = [nChannels, nHeis]
430
428
431 Si self.getByBlock == True:
429 Si self.getByBlock == True:
432
430
433 self.dataOut.data = buffer[:, :, :]
431 self.dataOut.data = buffer[:, :, :]
434
432
435 shape = [nChannels, nProfiles, nHeis]
433 shape = [nChannels, nProfiles, nHeis]
436
434
437 Variables afectadas:
435 Variables afectadas:
438 self.dataOut
436 self.dataOut
439 self.profileIndex
437 self.profileIndex
440
438
441 Affected:
439 Affected:
442 self.dataOut
440 self.dataOut
443 self.profileIndex
441 self.profileIndex
444 self.flagDiscontinuousBlock
442 self.flagDiscontinuousBlock
445 self.flagIsNewBlock
443 self.flagIsNewBlock
446 """
444 """
447 if self.flagNoMoreFiles:
445 if self.flagNoMoreFiles:
448 self.dataOut.flagNoData = True
446 self.dataOut.flagNoData = True
449 print 'Process finished'
447 print 'Process finished'
450 return 0
448 return 0
451 self.flagDiscontinuousBlock = 0
449 self.flagDiscontinuousBlock = 0
452 self.flagIsNewBlock = 0
450 self.flagIsNewBlock = 0
453 if self.__hasNotDataInBuffer():
451 if self.__hasNotDataInBuffer():
454 if not( self.readNextBlock() ):
452 if not( self.readNextBlock() ):
455 return 0
453 return 0
456
454
457 self.getFirstHeader()
455 self.getFirstHeader()
458
456
459 self.reshapeData()
457 self.reshapeData()
460 if self.datablock is None:
458 if self.datablock is None:
461 self.dataOut.flagNoData = True
459 self.dataOut.flagNoData = True
462 return 0
460 return 0
463
461
464 if not self.getByBlock:
462 if not self.getByBlock:
465
463
466 """
464 """
467 Return profile by profile
465 Return profile by profile
468
466
469 If nTxs > 1 then one profile is divided by nTxs and number of total
467 If nTxs > 1 then one profile is divided by nTxs and number of total
470 blocks is increased by nTxs (nProfiles *= nTxs)
468 blocks is increased by nTxs (nProfiles *= nTxs)
471 """
469 """
472 self.dataOut.flagDataAsBlock = False
470 self.dataOut.flagDataAsBlock = False
473 self.dataOut.data = self.datablock[:,self.profileIndex,:]
471 self.dataOut.data = self.datablock[:,self.profileIndex,:]
474 self.dataOut.profileIndex = self.profileIndex
472 self.dataOut.profileIndex = self.profileIndex
475
473
476 self.profileIndex += 1
474 self.profileIndex += 1
477
475
478 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
476 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
479 # """
477 # """
480 # Return all block
478 # Return all block
481 # """
479 # """
482 # self.dataOut.flagDataAsBlock = True
480 # self.dataOut.flagDataAsBlock = True
483 # self.dataOut.data = self.datablock
481 # self.dataOut.data = self.datablock
484 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
482 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
485 #
483 #
486 # self.profileIndex = self.dataOut.nProfiles
484 # self.profileIndex = self.dataOut.nProfiles
487
485
488 else:
486 else:
489 """
487 """
490 Return a block
488 Return a block
491 """
489 """
492 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
490 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
493 if self.selBlocktime != None:
491 if self.selBlocktime != None:
494 if self.dataOut.nCohInt is not None:
492 if self.dataOut.nCohInt is not None:
495 nCohInt = self.dataOut.nCohInt
493 nCohInt = self.dataOut.nCohInt
496 else:
494 else:
497 nCohInt = 1
495 nCohInt = 1
498 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
496 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
499
497
500 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
498 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
501 self.profileIndex += self.selBlocksize
499 self.profileIndex += self.selBlocksize
502 datasize = self.dataOut.data.shape[1]
500 datasize = self.dataOut.data.shape[1]
503
501
504 if datasize < self.selBlocksize:
502 if datasize < self.selBlocksize:
505 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
503 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
506 buffer[:,:datasize,:] = self.dataOut.data
504 buffer[:,:datasize,:] = self.dataOut.data
507
505
508 while datasize < self.selBlocksize: #Not enough profiles to fill the block
506 while datasize < self.selBlocksize: #Not enough profiles to fill the block
509 if not( self.readNextBlock() ):
507 if not( self.readNextBlock() ):
510 return 0
508 return 0
511 self.getFirstHeader()
509 self.getFirstHeader()
512 self.reshapeData()
510 self.reshapeData()
513 if self.datablock is None:
511 if self.datablock is None:
514 self.dataOut.flagNoData = True
512 self.dataOut.flagNoData = True
515 return 0
513 return 0
516 #stack data
514 #stack data
517 blockIndex = self.selBlocksize - datasize
515 blockIndex = self.selBlocksize - datasize
518 datablock1 = self.datablock[:,:blockIndex,:]
516 datablock1 = self.datablock[:,:blockIndex,:]
519
517
520 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
518 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
521 datasize += datablock1.shape[1]
519 datasize += datablock1.shape[1]
522
520
523 self.dataOut.data = buffer
521 self.dataOut.data = buffer
524 self.profileIndex = blockIndex
522 self.profileIndex = blockIndex
525
523
526 self.dataOut.flagDataAsBlock = True
524 self.dataOut.flagDataAsBlock = True
527 self.dataOut.nProfiles = self.dataOut.data.shape[1]
525 self.dataOut.nProfiles = self.dataOut.data.shape[1]
528
526
529 self.dataOut.flagNoData = False
527 self.dataOut.flagNoData = False
530
528
531 self.getBasicHeader()
529 self.getBasicHeader()
532
530
533 self.dataOut.realtime = self.online
531 self.dataOut.realtime = self.online
534
532
535 return self.dataOut.data
533 return self.dataOut.data
536
534
537 class VoltageWriter(JRODataWriter, Operation):
535 class VoltageWriter(JRODataWriter, Operation):
538 """
536 """
539 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
537 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
540 de los datos siempre se realiza por bloques.
538 de los datos siempre se realiza por bloques.
541 """
539 """
542
540
543 ext = ".r"
541 ext = ".r"
544
542
545 optchar = "D"
543 optchar = "D"
546
544
547 shapeBuffer = None
545 shapeBuffer = None
548
546
549
547
550 def __init__(self, **kwargs):
548 def __init__(self, **kwargs):
551 """
549 """
552 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
550 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
553
551
554 Affected:
552 Affected:
555 self.dataOut
553 self.dataOut
556
554
557 Return: None
555 Return: None
558 """
556 """
559 Operation.__init__(self, **kwargs)
557 Operation.__init__(self, **kwargs)
560
558
561 self.nTotalBlocks = 0
559 self.nTotalBlocks = 0
562
560
563 self.profileIndex = 0
561 self.profileIndex = 0
564
562
565 self.isConfig = False
563 self.isConfig = False
566
564
567 self.fp = None
565 self.fp = None
568
566
569 self.flagIsNewFile = 1
567 self.flagIsNewFile = 1
570
568
571 self.blockIndex = 0
569 self.blockIndex = 0
572
570
573 self.flagIsNewBlock = 0
571 self.flagIsNewBlock = 0
574
572
575 self.setFile = None
573 self.setFile = None
576
574
577 self.dtype = None
575 self.dtype = None
578
576
579 self.path = None
577 self.path = None
580
578
581 self.filename = None
579 self.filename = None
582
580
583 self.basicHeaderObj = BasicHeader(LOCALTIME)
581 self.basicHeaderObj = BasicHeader(LOCALTIME)
584
582
585 self.systemHeaderObj = SystemHeader()
583 self.systemHeaderObj = SystemHeader()
586
584
587 self.radarControllerHeaderObj = RadarControllerHeader()
585 self.radarControllerHeaderObj = RadarControllerHeader()
588
586
589 self.processingHeaderObj = ProcessingHeader()
587 self.processingHeaderObj = ProcessingHeader()
590
588
591 def hasAllDataInBuffer(self):
589 def hasAllDataInBuffer(self):
592 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
590 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
593 return 1
591 return 1
594 return 0
592 return 0
595
593
596
594
597 def setBlockDimension(self):
595 def setBlockDimension(self):
598 """
596 """
599 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
597 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
600
598
601 Affected:
599 Affected:
602 self.shape_spc_Buffer
600 self.shape_spc_Buffer
603 self.shape_cspc_Buffer
601 self.shape_cspc_Buffer
604 self.shape_dc_Buffer
602 self.shape_dc_Buffer
605
603
606 Return: None
604 Return: None
607 """
605 """
608 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
606 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
609 self.processingHeaderObj.nHeights,
607 self.processingHeaderObj.nHeights,
610 self.systemHeaderObj.nChannels)
608 self.systemHeaderObj.nChannels)
611
609
612 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
610 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
613 self.processingHeaderObj.profilesPerBlock,
611 self.processingHeaderObj.profilesPerBlock,
614 self.processingHeaderObj.nHeights),
612 self.processingHeaderObj.nHeights),
615 dtype=numpy.dtype('complex64'))
613 dtype=numpy.dtype('complex64'))
616
614
617 def writeBlock(self):
615 def writeBlock(self):
618 """
616 """
619 Escribe el buffer en el file designado
617 Escribe el buffer en el file designado
620
618
621 Affected:
619 Affected:
622 self.profileIndex
620 self.profileIndex
623 self.flagIsNewFile
621 self.flagIsNewFile
624 self.flagIsNewBlock
622 self.flagIsNewBlock
625 self.nTotalBlocks
623 self.nTotalBlocks
626 self.blockIndex
624 self.blockIndex
627
625
628 Return: None
626 Return: None
629 """
627 """
630 data = numpy.zeros( self.shapeBuffer, self.dtype )
628 data = numpy.zeros( self.shapeBuffer, self.dtype )
631
629
632 junk = numpy.transpose(self.datablock, (1,2,0))
630 junk = numpy.transpose(self.datablock, (1,2,0))
633
631
634 data['real'] = junk.real
632 data['real'] = junk.real
635 data['imag'] = junk.imag
633 data['imag'] = junk.imag
636
634
637 data = data.reshape( (-1) )
635 data = data.reshape( (-1) )
638
636
639 data.tofile( self.fp )
637 data.tofile( self.fp )
640
638
641 self.datablock.fill(0)
639 self.datablock.fill(0)
642
640
643 self.profileIndex = 0
641 self.profileIndex = 0
644 self.flagIsNewFile = 0
642 self.flagIsNewFile = 0
645 self.flagIsNewBlock = 1
643 self.flagIsNewBlock = 1
646
644
647 self.blockIndex += 1
645 self.blockIndex += 1
648 self.nTotalBlocks += 1
646 self.nTotalBlocks += 1
649
647
650 # print "[Writing] Block = %04d" %self.blockIndex
648 # print "[Writing] Block = %04d" %self.blockIndex
651
649
652 def putData(self):
650 def putData(self):
653 """
651 """
654 Setea un bloque de datos y luego los escribe en un file
652 Setea un bloque de datos y luego los escribe en un file
655
653
656 Affected:
654 Affected:
657 self.flagIsNewBlock
655 self.flagIsNewBlock
658 self.profileIndex
656 self.profileIndex
659
657
660 Return:
658 Return:
661 0 : Si no hay data o no hay mas files que puedan escribirse
659 0 : Si no hay data o no hay mas files que puedan escribirse
662 1 : Si se escribio la data de un bloque en un file
660 1 : Si se escribio la data de un bloque en un file
663 """
661 """
664 if self.dataOut.flagNoData:
662 if self.dataOut.flagNoData:
665 return 0
663 return 0
666
664
667 self.flagIsNewBlock = 0
665 self.flagIsNewBlock = 0
668
666
669 if self.dataOut.flagDiscontinuousBlock:
667 if self.dataOut.flagDiscontinuousBlock:
670 self.datablock.fill(0)
668 self.datablock.fill(0)
671 self.profileIndex = 0
669 self.profileIndex = 0
672 self.setNextFile()
670 self.setNextFile()
673
671
674 if self.profileIndex == 0:
672 if self.profileIndex == 0:
675 self.setBasicHeader()
673 self.setBasicHeader()
676
674
677 self.datablock[:,self.profileIndex,:] = self.dataOut.data
675 self.datablock[:,self.profileIndex,:] = self.dataOut.data
678
676
679 self.profileIndex += 1
677 self.profileIndex += 1
680
678
681 if self.hasAllDataInBuffer():
679 if self.hasAllDataInBuffer():
682 #if self.flagIsNewFile:
680 #if self.flagIsNewFile:
683 self.writeNextBlock()
681 self.writeNextBlock()
684 # self.setFirstHeader()
682 # self.setFirstHeader()
685
683
686 return 1
684 return 1
687
685
688 def __getBlockSize(self):
686 def __getBlockSize(self):
689 '''
687 '''
690 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
688 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
691 '''
689 '''
692
690
693 dtype_width = self.getDtypeWidth()
691 dtype_width = self.getDtypeWidth()
694
692
695 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
693 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
696
694
697 return blocksize
695 return blocksize
698
696
699 def setFirstHeader(self):
697 def setFirstHeader(self):
700
698
701 """
699 """
702 Obtiene una copia del First Header
700 Obtiene una copia del First Header
703
701
704 Affected:
702 Affected:
705 self.systemHeaderObj
703 self.systemHeaderObj
706 self.radarControllerHeaderObj
704 self.radarControllerHeaderObj
707 self.dtype
705 self.dtype
708
706
709 Return:
707 Return:
710 None
708 None
711 """
709 """
712
710
713 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
711 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
714 self.systemHeaderObj.nChannels = self.dataOut.nChannels
712 self.systemHeaderObj.nChannels = self.dataOut.nChannels
715 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
713 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
716
714
717 self.processingHeaderObj.dtype = 0 # Voltage
715 self.processingHeaderObj.dtype = 0 # Voltage
718 self.processingHeaderObj.blockSize = self.__getBlockSize()
716 self.processingHeaderObj.blockSize = self.__getBlockSize()
719 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
717 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
720 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
718 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
721 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
719 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
722 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
720 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
723 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
721 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
724 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
722 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
725
723
726 if self.dataOut.code is not None:
724 if self.dataOut.code is not None:
727 self.processingHeaderObj.code = self.dataOut.code
725 self.processingHeaderObj.code = self.dataOut.code
728 self.processingHeaderObj.nCode = self.dataOut.nCode
726 self.processingHeaderObj.nCode = self.dataOut.nCode
729 self.processingHeaderObj.nBaud = self.dataOut.nBaud
727 self.processingHeaderObj.nBaud = self.dataOut.nBaud
730
728
731 if self.processingHeaderObj.nWindows != 0:
729 if self.processingHeaderObj.nWindows != 0:
732 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
730 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
733 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
731 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
734 self.processingHeaderObj.nHeights = self.dataOut.nHeights
732 self.processingHeaderObj.nHeights = self.dataOut.nHeights
735 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
733 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
736
734
737 self.processingHeaderObj.processFlags = self.getProcessFlags()
735 self.processingHeaderObj.processFlags = self.getProcessFlags()
738
736
739 self.setBasicHeader()
737 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -1,58 +1,139
1 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
2 #define NUM_CPY_THREADS 8
1 #include <Python.h>
3 #include <Python.h>
2 #include <numpy/arrayobject.h>
4 #include <numpy/arrayobject.h>
3 #include <math.h>
5 #include <math.h>
6 #include <complex.h>
7 #include <time.h>
4
8
9 // void printArr(int *array);
5 static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args);
10 static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args);
11 static PyObject *correlateByBlock(PyObject *self, PyObject *args);
12 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
13 #define PyMODINIT_FUNC void
14 #endif
6
15
7 static PyMethodDef extensionsMethods[] = {
16 static PyMethodDef extensionsMethods[] = {
8 { "hildebrand_sekhon", (PyCFunction)hildebrand_sekhon, METH_VARARGS, "get noise with" },
17 { "correlateByBlock", (PyCFunction)correlateByBlock, METH_VARARGS, "get correlation by block" },
9 { NULL, NULL, 0, NULL }
18 { "hildebrand_sekhon", (PyCFunction)hildebrand_sekhon, METH_VARARGS, "get noise with hildebrand_sekhon" },
19 { NULL, NULL, 0, NULL }
10 };
20 };
11
21
12 PyMODINIT_FUNC initcSchain() {
22 PyMODINIT_FUNC initcSchain() {
13 Py_InitModule("cSchain", extensionsMethods);
23 Py_InitModule("cSchain", extensionsMethods);
14 import_array();
24 import_array();
15 }
25 }
16
26
27 static PyObject *correlateByBlock(PyObject *self, PyObject *args) {
28
29 // int *x = (int*) malloc(4000000 * 216 * sizeof(int));;
30 // int a = 5;
31 // x = &a;
32 // int b = 6;
33 // x = &b;
34 // printf("Antes de imprimir x \n");
35 // printf("%d \n", x[0]);
36
37 PyObject *data_obj1, *data_obj2;
38 PyArrayObject *data_array1, *data_array2, *correlateRow, *out, *dataRow, *codeRow; //, ,
39 int mode;
40
41 if (!PyArg_ParseTuple(args, "OOi", &data_obj1, &data_obj2, &mode)) return NULL;
42
43 data_array1 = (PyArrayObject *) PyArray_FROM_OTF(data_obj1, NPY_COMPLEX128, NPY_ARRAY_DEFAULT);
44 data_array2 = (PyArrayObject *) PyArray_FROM_OTF(data_obj2, NPY_FLOAT64, NPY_ARRAY_DEFAULT);
45
46 npy_intp dims[1];
47 dims[0] = 200;
48 npy_intp dims_code[1];
49 dims_code[0] = 16;
50
51 double complex * dataRaw;
52 double * codeRaw;
53 dataRaw = (double complex*) PyArray_DATA(data_array1);
54 codeRaw = (double *) PyArray_DATA(data_array2);
55 double complex ** outC = malloc(40000*200*sizeof(double complex));
56 int i;
57
58 clock_t start = clock();
59 for(i=0; i<40000; i++){
60 // codeRow = PyArray_SimpleNewFromData(1, dims_code, NPY_FLOAT64, codeRaw + 16 * i);
61 // dataRow = PyArray_SimpleNewFromData(1, dims, NPY_COMPLEX128, dataRaw + 200 * i);
62 // Py_INCREF(codeRow);
63 // Py_INCREF(dataRow);
64 // PyArray_ENABLEFLAGS(codeRow, NPY_ARRAY_OWNDATA);
65 // PyArray_ENABLEFLAGS(dataRow, NPY_ARRAY_OWNDATA);
66 correlateRow = (PyArrayObject *) PyArray_Correlate2(PyArray_SimpleNewFromData(1, dims_code, NPY_FLOAT64, codeRaw + 16 * i), PyArray_SimpleNewFromData(1, dims, NPY_COMPLEX128, dataRaw + 200 * i), (npy_intp) 2);
67 //Py_INCREF(correlateRow);
68 // PyArray_ENABLEFLAGS(correlateRow, NPY_ARRAY_OWNDATA);
69 memcpy(outC + 200*i, (double complex*) PyArray_DATA(correlateRow), 200 * sizeof(double complex));
70
71 Py_DECREF(correlateRow);
72 // Py_DECREF(codeRow);
73 // Py_DECREF(dataRow);
74 }
75 clock_t end = clock();
76 float seconds = (float)(end - start) / CLOCKS_PER_SEC;
77 printf("%f", seconds);
78 //
79 npy_intp dimsret[2];
80 dimsret[0] = 40000;
81 dimsret[1] = 200;
82 out = PyArray_SimpleNewFromData(2, dimsret, NPY_COMPLEX128, outC);
83 PyArray_ENABLEFLAGS(out, NPY_ARRAY_OWNDATA);
84 //Py_INCREF(out);
85 Py_DECREF(data_array1);
86 Py_DECREF(data_array2);
87 // PyArray_DebugPrint(out);
88 // Py_DECREF(data_obj2);
89 // Py_DECREF(data_obj1);
90 // Py_DECREF(codeRow);
91 // Py_DECREF(dataRow);
92 // free(dataRaw);
93 // free(codeRaw);
94
95 return PyArray_Return(out);
96 }
97
17 static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) {
98 static PyObject *hildebrand_sekhon(PyObject *self, PyObject *args) {
18 /* Do your stuff here. */
19 double navg;
99 double navg;
20 PyObject *data_obj, *data_array;
100 PyObject *data_obj, *data_array;
21
101
22 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) return NULL;
102 if (!PyArg_ParseTuple(args, "Od", &data_obj, &navg)) return NULL;
23 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_IN_ARRAY);
103 data_array = PyArray_FROM_OTF(data_obj, NPY_FLOAT64, NPY_ARRAY_DEFAULT);
24 if (data_array == NULL) {
104 if (data_array == NULL) {
25 Py_XDECREF(data_array);
105 Py_XDECREF(data_array);
26 Py_XDECREF(data_obj);
106 Py_XDECREF(data_obj);
27 return NULL;
107 return NULL;
28 }
108 }
29 double *sortdata = (double*)PyArray_DATA(data_array);
109 double *sortdata = (double*)PyArray_DATA(data_array);
30 int lenOfData = (int)PyArray_SIZE(data_array) ;
110 int lenOfData = (int)PyArray_SIZE(data_array) ;
31 double nums_min = lenOfData*0.2;
111 double nums_min = lenOfData*0.2;
32 if (nums_min <= 5) nums_min = 5;
112 if (nums_min <= 5) nums_min = 5;
33 double sump = 0;
113 double sump = 0;
34 double sumq = 0;
114 double sumq = 0;
35 int j = 0;
115 int j = 0;
36 int cont = 1;
116 int cont = 1;
37 double rtest = 0;
117 double rtest = 0;
38 while ((cont == 1) && (j < lenOfData)) {
118 while ((cont == 1) && (j < lenOfData)) {
39 sump = sump + sortdata[j];
119 sump = sump + sortdata[j];
40 sumq = sumq + pow(sortdata[j], 2);
120 sumq = sumq + pow(sortdata[j], 2);
41 if (j > nums_min) {
121 if (j > nums_min) {
42 rtest = (double)j/(j-1) + 1/navg;
122 rtest = (double)j/(j-1) + 1/navg;
43 if ((sumq*j) > (rtest*pow(sump, 2))) {
123 if ((sumq*j) > (rtest*pow(sump, 2))) {
44 j = j - 1;
124 j = j - 1;
45 sump = sump - sortdata[j];
125 sump = sump - sortdata[j];
46 sumq = sumq - pow(sortdata[j],2);
126 sumq = sumq - pow(sortdata[j],2);
47 cont = 0;
127 cont = 0;
48 }
128 }
49 }
129 }
50 j = j + 1;
130 j = j + 1;
51 }
131 }
52
132
53 double lnoise = sump / j;
133 double lnoise = sump / j;
54
134
55 Py_DECREF(data_array);
135 Py_DECREF(data_array);
56
136
57 return Py_BuildValue("d", lnoise);
137 return Py_BuildValue("d", lnoise);
58 }
138 }
139
Show file before | Show file after | Hide comments
@@ -1,1285 +1,1312
1 import sys
1 import sys
2 import numpy
2 import numpy
3 from profilehooks import profile
3 from scipy import interpolate
4 from scipy import interpolate
4
5 from schainpy import cSchain
5 from jroproc_base import ProcessingUnit, Operation
6 from jroproc_base import ProcessingUnit, Operation
6 from schainpy.model.data.jrodata import Voltage
7 from schainpy.model.data.jrodata import Voltage
8 from time import time
7
9
8 class VoltageProc(ProcessingUnit):
10 class VoltageProc(ProcessingUnit):
9
11
10
12
11 def __init__(self, **kwargs):
13 def __init__(self, **kwargs):
12
14
13 ProcessingUnit.__init__(self, **kwargs)
15 ProcessingUnit.__init__(self, **kwargs)
14
16
15 # self.objectDict = {}
17 # self.objectDict = {}
16 self.dataOut = Voltage()
18 self.dataOut = Voltage()
17 self.flip = 1
19 self.flip = 1
18
20
19 def run(self):
21 def run(self):
20 if self.dataIn.type == 'AMISR':
22 if self.dataIn.type == 'AMISR':
21 self.__updateObjFromAmisrInput()
23 self.__updateObjFromAmisrInput()
22
24
23 if self.dataIn.type == 'Voltage':
25 if self.dataIn.type == 'Voltage':
24 self.dataOut.copy(self.dataIn)
26 self.dataOut.copy(self.dataIn)
25
27
26 # self.dataOut.copy(self.dataIn)
28 # self.dataOut.copy(self.dataIn)
27
29
28 def __updateObjFromAmisrInput(self):
30 def __updateObjFromAmisrInput(self):
29
31
30 self.dataOut.timeZone = self.dataIn.timeZone
32 self.dataOut.timeZone = self.dataIn.timeZone
31 self.dataOut.dstFlag = self.dataIn.dstFlag
33 self.dataOut.dstFlag = self.dataIn.dstFlag
32 self.dataOut.errorCount = self.dataIn.errorCount
34 self.dataOut.errorCount = self.dataIn.errorCount
33 self.dataOut.useLocalTime = self.dataIn.useLocalTime
35 self.dataOut.useLocalTime = self.dataIn.useLocalTime
34
36
35 self.dataOut.flagNoData = self.dataIn.flagNoData
37 self.dataOut.flagNoData = self.dataIn.flagNoData
36 self.dataOut.data = self.dataIn.data
38 self.dataOut.data = self.dataIn.data
37 self.dataOut.utctime = self.dataIn.utctime
39 self.dataOut.utctime = self.dataIn.utctime
38 self.dataOut.channelList = self.dataIn.channelList
40 self.dataOut.channelList = self.dataIn.channelList
39 # self.dataOut.timeInterval = self.dataIn.timeInterval
41 # self.dataOut.timeInterval = self.dataIn.timeInterval
40 self.dataOut.heightList = self.dataIn.heightList
42 self.dataOut.heightList = self.dataIn.heightList
41 self.dataOut.nProfiles = self.dataIn.nProfiles
43 self.dataOut.nProfiles = self.dataIn.nProfiles
42
44
43 self.dataOut.nCohInt = self.dataIn.nCohInt
45 self.dataOut.nCohInt = self.dataIn.nCohInt
44 self.dataOut.ippSeconds = self.dataIn.ippSeconds
46 self.dataOut.ippSeconds = self.dataIn.ippSeconds
45 self.dataOut.frequency = self.dataIn.frequency
47 self.dataOut.frequency = self.dataIn.frequency
46
48
47 self.dataOut.azimuth = self.dataIn.azimuth
49 self.dataOut.azimuth = self.dataIn.azimuth
48 self.dataOut.zenith = self.dataIn.zenith
50 self.dataOut.zenith = self.dataIn.zenith
49
51
50 self.dataOut.beam.codeList = self.dataIn.beam.codeList
52 self.dataOut.beam.codeList = self.dataIn.beam.codeList
51 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
53 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
52 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
54 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
53 #
55 #
54 # pass#
56 # pass#
55 #
57 #
56 # def init(self):
58 # def init(self):
57 #
59 #
58 #
60 #
59 # if self.dataIn.type == 'AMISR':
61 # if self.dataIn.type == 'AMISR':
60 # self.__updateObjFromAmisrInput()
62 # self.__updateObjFromAmisrInput()
61 #
63 #
62 # if self.dataIn.type == 'Voltage':
64 # if self.dataIn.type == 'Voltage':
63 # self.dataOut.copy(self.dataIn)
65 # self.dataOut.copy(self.dataIn)
64 # # No necesita copiar en cada init() los atributos de dataIn
66 # # No necesita copiar en cada init() los atributos de dataIn
65 # # la copia deberia hacerse por cada nuevo bloque de datos
67 # # la copia deberia hacerse por cada nuevo bloque de datos
66
68
67 def selectChannels(self, channelList):
69 def selectChannels(self, channelList):
68
70
69 channelIndexList = []
71 channelIndexList = []
70
72
71 for channel in channelList:
73 for channel in channelList:
72 if channel not in self.dataOut.channelList:
74 if channel not in self.dataOut.channelList:
73 raise ValueError, "Channel %d is not in %s" %(channel, str(self.dataOut.channelList))
75 raise ValueError, "Channel %d is not in %s" %(channel, str(self.dataOut.channelList))
74
76
75 index = self.dataOut.channelList.index(channel)
77 index = self.dataOut.channelList.index(channel)
76 channelIndexList.append(index)
78 channelIndexList.append(index)
77
79
78 self.selectChannelsByIndex(channelIndexList)
80 self.selectChannelsByIndex(channelIndexList)
79
81
80 def selectChannelsByIndex(self, channelIndexList):
82 def selectChannelsByIndex(self, channelIndexList):
81 """
83 """
82 Selecciona un bloque de datos en base a canales segun el channelIndexList
84 Selecciona un bloque de datos en base a canales segun el channelIndexList
83
85
84 Input:
86 Input:
85 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
87 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
86
88
87 Affected:
89 Affected:
88 self.dataOut.data
90 self.dataOut.data
89 self.dataOut.channelIndexList
91 self.dataOut.channelIndexList
90 self.dataOut.nChannels
92 self.dataOut.nChannels
91 self.dataOut.m_ProcessingHeader.totalSpectra
93 self.dataOut.m_ProcessingHeader.totalSpectra
92 self.dataOut.systemHeaderObj.numChannels
94 self.dataOut.systemHeaderObj.numChannels
93 self.dataOut.m_ProcessingHeader.blockSize
95 self.dataOut.m_ProcessingHeader.blockSize
94
96
95 Return:
97 Return:
96 None
98 None
97 """
99 """
98
100
99 for channelIndex in channelIndexList:
101 for channelIndex in channelIndexList:
100 if channelIndex not in self.dataOut.channelIndexList:
102 if channelIndex not in self.dataOut.channelIndexList:
101 print channelIndexList
103 print channelIndexList
102 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
104 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
103
105
104 if self.dataOut.flagDataAsBlock:
106 if self.dataOut.flagDataAsBlock:
105 """
107 """
106 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
108 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
107 """
109 """
108 data = self.dataOut.data[channelIndexList,:,:]
110 data = self.dataOut.data[channelIndexList,:,:]
109 else:
111 else:
110 data = self.dataOut.data[channelIndexList,:]
112 data = self.dataOut.data[channelIndexList,:]
111
113
112 self.dataOut.data = data
114 self.dataOut.data = data
113 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
115 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
114 # self.dataOut.nChannels = nChannels
116 # self.dataOut.nChannels = nChannels
115
117
116 return 1
118 return 1
117
119
118 def selectHeights(self, minHei=None, maxHei=None):
120 def selectHeights(self, minHei=None, maxHei=None):
119 """
121 """
120 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
122 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
121 minHei <= height <= maxHei
123 minHei <= height <= maxHei
122
124
123 Input:
125 Input:
124 minHei : valor minimo de altura a considerar
126 minHei : valor minimo de altura a considerar
125 maxHei : valor maximo de altura a considerar
127 maxHei : valor maximo de altura a considerar
126
128
127 Affected:
129 Affected:
128 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
130 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
129
131
130 Return:
132 Return:
131 1 si el metodo se ejecuto con exito caso contrario devuelve 0
133 1 si el metodo se ejecuto con exito caso contrario devuelve 0
132 """
134 """
133
135
134 if minHei == None:
136 if minHei == None:
135 minHei = self.dataOut.heightList[0]
137 minHei = self.dataOut.heightList[0]
136
138
137 if maxHei == None:
139 if maxHei == None:
138 maxHei = self.dataOut.heightList[-1]
140 maxHei = self.dataOut.heightList[-1]
139
141
140 if (minHei < self.dataOut.heightList[0]):
142 if (minHei < self.dataOut.heightList[0]):
141 minHei = self.dataOut.heightList[0]
143 minHei = self.dataOut.heightList[0]
142
144
143 if (maxHei > self.dataOut.heightList[-1]):
145 if (maxHei > self.dataOut.heightList[-1]):
144 maxHei = self.dataOut.heightList[-1]
146 maxHei = self.dataOut.heightList[-1]
145
147
146 minIndex = 0
148 minIndex = 0
147 maxIndex = 0
149 maxIndex = 0
148 heights = self.dataOut.heightList
150 heights = self.dataOut.heightList
149
151
150 inda = numpy.where(heights >= minHei)
152 inda = numpy.where(heights >= minHei)
151 indb = numpy.where(heights <= maxHei)
153 indb = numpy.where(heights <= maxHei)
152
154
153 try:
155 try:
154 minIndex = inda[0][0]
156 minIndex = inda[0][0]
155 except:
157 except:
156 minIndex = 0
158 minIndex = 0
157
159
158 try:
160 try:
159 maxIndex = indb[0][-1]
161 maxIndex = indb[0][-1]
160 except:
162 except:
161 maxIndex = len(heights)
163 maxIndex = len(heights)
162
164
163 self.selectHeightsByIndex(minIndex, maxIndex)
165 self.selectHeightsByIndex(minIndex, maxIndex)
164
166
165 return 1
167 return 1
166
168
167
169
168 def selectHeightsByIndex(self, minIndex, maxIndex):
170 def selectHeightsByIndex(self, minIndex, maxIndex):
169 """
171 """
170 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
172 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
171 minIndex <= index <= maxIndex
173 minIndex <= index <= maxIndex
172
174
173 Input:
175 Input:
174 minIndex : valor de indice minimo de altura a considerar
176 minIndex : valor de indice minimo de altura a considerar
175 maxIndex : valor de indice maximo de altura a considerar
177 maxIndex : valor de indice maximo de altura a considerar
176
178
177 Affected:
179 Affected:
178 self.dataOut.data
180 self.dataOut.data
179 self.dataOut.heightList
181 self.dataOut.heightList
180
182
181 Return:
183 Return:
182 1 si el metodo se ejecuto con exito caso contrario devuelve 0
184 1 si el metodo se ejecuto con exito caso contrario devuelve 0
183 """
185 """
184
186
185 if (minIndex < 0) or (minIndex > maxIndex):
187 if (minIndex < 0) or (minIndex > maxIndex):
186 raise ValueError, "Height index range (%d,%d) is not valid" % (minIndex, maxIndex)
188 raise ValueError, "Height index range (%d,%d) is not valid" % (minIndex, maxIndex)
187
189
188 if (maxIndex >= self.dataOut.nHeights):
190 if (maxIndex >= self.dataOut.nHeights):
189 maxIndex = self.dataOut.nHeights
191 maxIndex = self.dataOut.nHeights
190
192
191 #voltage
193 #voltage
192 if self.dataOut.flagDataAsBlock:
194 if self.dataOut.flagDataAsBlock:
193 """
195 """
194 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
196 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
195 """
197 """
196 data = self.dataOut.data[:,:, minIndex:maxIndex]
198 data = self.dataOut.data[:,:, minIndex:maxIndex]
197 else:
199 else:
198 data = self.dataOut.data[:, minIndex:maxIndex]
200 data = self.dataOut.data[:, minIndex:maxIndex]
199
201
200 # firstHeight = self.dataOut.heightList[minIndex]
202 # firstHeight = self.dataOut.heightList[minIndex]
201
203
202 self.dataOut.data = data
204 self.dataOut.data = data
203 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
205 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
204
206
205 if self.dataOut.nHeights <= 1:
207 if self.dataOut.nHeights <= 1:
206 raise ValueError, "selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights)
208 raise ValueError, "selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights)
207
209
208 return 1
210 return 1
209
211
210
212
211 def filterByHeights(self, window):
213 def filterByHeights(self, window):
212
214
213 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
215 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
214
216
215 if window == None:
217 if window == None:
216 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
218 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
217
219
218 newdelta = deltaHeight * window
220 newdelta = deltaHeight * window
219 r = self.dataOut.nHeights % window
221 r = self.dataOut.nHeights % window
220 newheights = (self.dataOut.nHeights-r)/window
222 newheights = (self.dataOut.nHeights-r)/window
221
223
222 if newheights <= 1:
224 if newheights <= 1:
223 raise ValueError, "filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window)
225 raise ValueError, "filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window)
224
226
225 if self.dataOut.flagDataAsBlock:
227 if self.dataOut.flagDataAsBlock:
226 """
228 """
227 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
229 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
228 """
230 """
229 buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
231 buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
230 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
232 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
231 buffer = numpy.sum(buffer,3)
233 buffer = numpy.sum(buffer,3)
232
234
233 else:
235 else:
234 buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
236 buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
235 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
237 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
236 buffer = numpy.sum(buffer,2)
238 buffer = numpy.sum(buffer,2)
237
239
238 self.dataOut.data = buffer
240 self.dataOut.data = buffer
239 self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
241 self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
240 self.dataOut.windowOfFilter = window
242 self.dataOut.windowOfFilter = window
241
243
242 def setH0(self, h0, deltaHeight = None):
244 def setH0(self, h0, deltaHeight = None):
243
245
244 if not deltaHeight:
246 if not deltaHeight:
245 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
247 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
246
248
247 nHeights = self.dataOut.nHeights
249 nHeights = self.dataOut.nHeights
248
250
249 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
251 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
250
252
251 self.dataOut.heightList = newHeiRange
253 self.dataOut.heightList = newHeiRange
252
254
253 def deFlip(self, channelList = []):
255 def deFlip(self, channelList = []):
254
256
255 data = self.dataOut.data.copy()
257 data = self.dataOut.data.copy()
256
258
257 if self.dataOut.flagDataAsBlock:
259 if self.dataOut.flagDataAsBlock:
258 flip = self.flip
260 flip = self.flip
259 profileList = range(self.dataOut.nProfiles)
261 profileList = range(self.dataOut.nProfiles)
260
262
261 if not channelList:
263 if not channelList:
262 for thisProfile in profileList:
264 for thisProfile in profileList:
263 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
265 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
264 flip *= -1.0
266 flip *= -1.0
265 else:
267 else:
266 for thisChannel in channelList:
268 for thisChannel in channelList:
267 if thisChannel not in self.dataOut.channelList:
269 if thisChannel not in self.dataOut.channelList:
268 continue
270 continue
269
271
270 for thisProfile in profileList:
272 for thisProfile in profileList:
271 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
273 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
272 flip *= -1.0
274 flip *= -1.0
273
275
274 self.flip = flip
276 self.flip = flip
275
277
276 else:
278 else:
277 if not channelList:
279 if not channelList:
278 data[:,:] = data[:,:]*self.flip
280 data[:,:] = data[:,:]*self.flip
279 else:
281 else:
280 for thisChannel in channelList:
282 for thisChannel in channelList:
281 if thisChannel not in self.dataOut.channelList:
283 if thisChannel not in self.dataOut.channelList:
282 continue
284 continue
283
285
284 data[thisChannel,:] = data[thisChannel,:]*self.flip
286 data[thisChannel,:] = data[thisChannel,:]*self.flip
285
287
286 self.flip *= -1.
288 self.flip *= -1.
287
289
288 self.dataOut.data = data
290 self.dataOut.data = data
289
291
290 def setRadarFrequency(self, frequency=None):
292 def setRadarFrequency(self, frequency=None):
291
293
292 if frequency != None:
294 if frequency != None:
293 self.dataOut.frequency = frequency
295 self.dataOut.frequency = frequency
294
296
295 return 1
297 return 1
296
298
297 def interpolateHeights(self, topLim, botLim):
299 def interpolateHeights(self, topLim, botLim):
298 #69 al 72 para julia
300 #69 al 72 para julia
299 #82-84 para meteoros
301 #82-84 para meteoros
300 if len(numpy.shape(self.dataOut.data))==2:
302 if len(numpy.shape(self.dataOut.data))==2:
301 sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
303 sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
302 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
304 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
303 #self.dataOut.data[:,botLim:limSup+1] = sampInterp
305 #self.dataOut.data[:,botLim:limSup+1] = sampInterp
304 self.dataOut.data[:,botLim:topLim+1] = sampInterp
306 self.dataOut.data[:,botLim:topLim+1] = sampInterp
305 else:
307 else:
306 nHeights = self.dataOut.data.shape[2]
308 nHeights = self.dataOut.data.shape[2]
307 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
309 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
308 y = self.dataOut.data[:,:,range(botLim)+range(topLim+1,nHeights)]
310 y = self.dataOut.data[:,:,range(botLim)+range(topLim+1,nHeights)]
309 f = interpolate.interp1d(x, y, axis = 2)
311 f = interpolate.interp1d(x, y, axis = 2)
310 xnew = numpy.arange(botLim,topLim+1)
312 xnew = numpy.arange(botLim,topLim+1)
311 ynew = f(xnew)
313 ynew = f(xnew)
312
314
313 self.dataOut.data[:,:,botLim:topLim+1] = ynew
315 self.dataOut.data[:,:,botLim:topLim+1] = ynew
314
316
315 # import collections
317 # import collections
316
318
317 class CohInt(Operation):
319 class CohInt(Operation):
318
320
319 isConfig = False
321 isConfig = False
320
322
321 __profIndex = 0
323 __profIndex = 0
322 __withOverapping = False
324 __withOverapping = False
323
325
324 __byTime = False
326 __byTime = False
325 __initime = None
327 __initime = None
326 __lastdatatime = None
328 __lastdatatime = None
327 __integrationtime = None
329 __integrationtime = None
328
330
329 __buffer = None
331 __buffer = None
330
332
331 __dataReady = False
333 __dataReady = False
332
334
333 n = None
335 n = None
334
336
335
337
336 def __init__(self, **kwargs):
338 def __init__(self, **kwargs):
337
339
338 Operation.__init__(self, **kwargs)
340 Operation.__init__(self, **kwargs)
339
341
340 # self.isConfig = False
342 # self.isConfig = False
341
343
342 def setup(self, n=None, timeInterval=None, overlapping=False, byblock=False):
344 def setup(self, n=None, timeInterval=None, overlapping=False, byblock=False):
343 """
345 """
344 Set the parameters of the integration class.
346 Set the parameters of the integration class.
345
347
346 Inputs:
348 Inputs:
347
349
348 n : Number of coherent integrations
350 n : Number of coherent integrations
349 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
351 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
350 overlapping :
352 overlapping :
351
353
352 """
354 """
353
355
354 self.__initime = None
356 self.__initime = None
355 self.__lastdatatime = 0
357 self.__lastdatatime = 0
356 self.__buffer = None
358 self.__buffer = None
357 self.__dataReady = False
359 self.__dataReady = False
358 self.byblock = byblock
360 self.byblock = byblock
359
361
360 if n == None and timeInterval == None:
362 if n == None and timeInterval == None:
361 raise ValueError, "n or timeInterval should be specified ..."
363 raise ValueError, "n or timeInterval should be specified ..."
362
364
363 if n != None:
365 if n != None:
364 self.n = n
366 self.n = n
365 self.__byTime = False
367 self.__byTime = False
366 else:
368 else:
367 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
369 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
368 self.n = 9999
370 self.n = 9999
369 self.__byTime = True
371 self.__byTime = True
370
372
371 if overlapping:
373 if overlapping:
372 self.__withOverapping = True
374 self.__withOverapping = True
373 self.__buffer = None
375 self.__buffer = None
374 else:
376 else:
375 self.__withOverapping = False
377 self.__withOverapping = False
376 self.__buffer = 0
378 self.__buffer = 0
377
379
378 self.__profIndex = 0
380 self.__profIndex = 0
379
381
380 def putData(self, data):
382 def putData(self, data):
381
383
382 """
384 """
383 Add a profile to the __buffer and increase in one the __profileIndex
385 Add a profile to the __buffer and increase in one the __profileIndex
384
386
385 """
387 """
386
388
387 if not self.__withOverapping:
389 if not self.__withOverapping:
388 self.__buffer += data.copy()
390 self.__buffer += data.copy()
389 self.__profIndex += 1
391 self.__profIndex += 1
390 return
392 return
391
393
392 #Overlapping data
394 #Overlapping data
393 nChannels, nHeis = data.shape
395 nChannels, nHeis = data.shape
394 data = numpy.reshape(data, (1, nChannels, nHeis))
396 data = numpy.reshape(data, (1, nChannels, nHeis))
395
397
396 #If the buffer is empty then it takes the data value
398 #If the buffer is empty then it takes the data value
397 if self.__buffer is None:
399 if self.__buffer is None:
398 self.__buffer = data
400 self.__buffer = data
399 self.__profIndex += 1
401 self.__profIndex += 1
400 return
402 return
401
403
402 #If the buffer length is lower than n then stakcing the data value
404 #If the buffer length is lower than n then stakcing the data value
403 if self.__profIndex < self.n:
405 if self.__profIndex < self.n:
404 self.__buffer = numpy.vstack((self.__buffer, data))
406 self.__buffer = numpy.vstack((self.__buffer, data))
405 self.__profIndex += 1
407 self.__profIndex += 1
406 return
408 return
407
409
408 #If the buffer length is equal to n then replacing the last buffer value with the data value
410 #If the buffer length is equal to n then replacing the last buffer value with the data value
409 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
411 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
410 self.__buffer[self.n-1] = data
412 self.__buffer[self.n-1] = data
411 self.__profIndex = self.n
413 self.__profIndex = self.n
412 return
414 return
413
415
414
416
415 def pushData(self):
417 def pushData(self):
416 """
418 """
417 Return the sum of the last profiles and the profiles used in the sum.
419 Return the sum of the last profiles and the profiles used in the sum.
418
420
419 Affected:
421 Affected:
420
422
421 self.__profileIndex
423 self.__profileIndex
422
424
423 """
425 """
424
426
425 if not self.__withOverapping:
427 if not self.__withOverapping:
426 data = self.__buffer
428 data = self.__buffer
427 n = self.__profIndex
429 n = self.__profIndex
428
430
429 self.__buffer = 0
431 self.__buffer = 0
430 self.__profIndex = 0
432 self.__profIndex = 0
431
433
432 return data, n
434 return data, n
433
435
434 #Integration with Overlapping
436 #Integration with Overlapping
435 data = numpy.sum(self.__buffer, axis=0)
437 data = numpy.sum(self.__buffer, axis=0)
436 n = self.__profIndex
438 n = self.__profIndex
437
439
438 return data, n
440 return data, n
439
441
440 def byProfiles(self, data):
442 def byProfiles(self, data):
441
443
442 self.__dataReady = False
444 self.__dataReady = False
443 avgdata = None
445 avgdata = None
444 # n = None
446 # n = None
445
447
446 self.putData(data)
448 self.putData(data)
447
449
448 if self.__profIndex == self.n:
450 if self.__profIndex == self.n:
449
451
450 avgdata, n = self.pushData()
452 avgdata, n = self.pushData()
451 self.__dataReady = True
453 self.__dataReady = True
452
454
453 return avgdata
455 return avgdata
454
456
455 def byTime(self, data, datatime):
457 def byTime(self, data, datatime):
456
458
457 self.__dataReady = False
459 self.__dataReady = False
458 avgdata = None
460 avgdata = None
459 n = None
461 n = None
460
462
461 self.putData(data)
463 self.putData(data)
462
464
463 if (datatime - self.__initime) >= self.__integrationtime:
465 if (datatime - self.__initime) >= self.__integrationtime:
464 avgdata, n = self.pushData()
466 avgdata, n = self.pushData()
465 self.n = n
467 self.n = n
466 self.__dataReady = True
468 self.__dataReady = True
467
469
468 return avgdata
470 return avgdata
469
471
470 def integrate(self, data, datatime=None):
472 def integrate(self, data, datatime=None):
471
473
472 if self.__initime == None:
474 if self.__initime == None:
473 self.__initime = datatime
475 self.__initime = datatime
474
476
475 if self.__byTime:
477 if self.__byTime:
476 avgdata = self.byTime(data, datatime)
478 avgdata = self.byTime(data, datatime)
477 else:
479 else:
478 avgdata = self.byProfiles(data)
480 avgdata = self.byProfiles(data)
479
481
480
482
481 self.__lastdatatime = datatime
483 self.__lastdatatime = datatime
482
484
483 if avgdata is None:
485 if avgdata is None:
484 return None, None
486 return None, None
485
487
486 avgdatatime = self.__initime
488 avgdatatime = self.__initime
487
489
488 deltatime = datatime -self.__lastdatatime
490 deltatime = datatime -self.__lastdatatime
489
491
490 if not self.__withOverapping:
492 if not self.__withOverapping:
491 self.__initime = datatime
493 self.__initime = datatime
492 else:
494 else:
493 self.__initime += deltatime
495 self.__initime += deltatime
494
496
495 return avgdata, avgdatatime
497 return avgdata, avgdatatime
496
498
497 def integrateByBlock(self, dataOut):
499 def integrateByBlock(self, dataOut):
498
500
499 times = int(dataOut.data.shape[1]/self.n)
501 times = int(dataOut.data.shape[1]/self.n)
500 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
502 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
501
503
502 id_min = 0
504 id_min = 0
503 id_max = self.n
505 id_max = self.n
504
506
505 for i in range(times):
507 for i in range(times):
506 junk = dataOut.data[:,id_min:id_max,:]
508 junk = dataOut.data[:,id_min:id_max,:]
507 avgdata[:,i,:] = junk.sum(axis=1)
509 avgdata[:,i,:] = junk.sum(axis=1)
508 id_min += self.n
510 id_min += self.n
509 id_max += self.n
511 id_max += self.n
510
512
511 timeInterval = dataOut.ippSeconds*self.n
513 timeInterval = dataOut.ippSeconds*self.n
512 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
514 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
513 self.__dataReady = True
515 self.__dataReady = True
514 return avgdata, avgdatatime
516 return avgdata, avgdatatime
515
517
516
518
517 def run(self, dataOut, n=None, timeInterval=None, overlapping=False, byblock=False, **kwargs):
519 def run(self, dataOut, n=None, timeInterval=None, overlapping=False, byblock=False, **kwargs):
518 if not self.isConfig:
520 if not self.isConfig:
519 self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
521 self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
520 self.isConfig = True
522 self.isConfig = True
521
523
522 if dataOut.flagDataAsBlock:
524 if dataOut.flagDataAsBlock:
523 """
525 """
524 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
526 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
525 """
527 """
526 avgdata, avgdatatime = self.integrateByBlock(dataOut)
528 avgdata, avgdatatime = self.integrateByBlock(dataOut)
527 dataOut.nProfiles /= self.n
529 dataOut.nProfiles /= self.n
528 else:
530 else:
529 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
531 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
530
532
531 # dataOut.timeInterval *= n
533 # dataOut.timeInterval *= n
532 dataOut.flagNoData = True
534 dataOut.flagNoData = True
533
535
534 if self.__dataReady:
536 if self.__dataReady:
535 dataOut.data = avgdata
537 dataOut.data = avgdata
536 dataOut.nCohInt *= self.n
538 dataOut.nCohInt *= self.n
537 dataOut.utctime = avgdatatime
539 dataOut.utctime = avgdatatime
538 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
540 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
539 dataOut.flagNoData = False
541 dataOut.flagNoData = False
540
542
541 class Decoder(Operation):
543 class Decoder(Operation):
542
544
543 isConfig = False
545 isConfig = False
544 __profIndex = 0
546 __profIndex = 0
545
547
546 code = None
548 code = None
547
549
548 nCode = None
550 nCode = None
549 nBaud = None
551 nBaud = None
550
552
551
553
552 def __init__(self, **kwargs):
554 def __init__(self, **kwargs):
553
555
554 Operation.__init__(self, **kwargs)
556 Operation.__init__(self, **kwargs)
555
557
556 self.times = None
558 self.times = None
557 self.osamp = None
559 self.osamp = None
558 # self.__setValues = False
560 # self.__setValues = False
559 self.isConfig = False
561 self.isConfig = False
560
562
561 def setup(self, code, osamp, dataOut):
563 def setup(self, code, osamp, dataOut):
562
564
563 self.__profIndex = 0
565 self.__profIndex = 0
564
566
565 self.code = code
567 self.code = code
566
568
567 self.nCode = len(code)
569 self.nCode = len(code)
568 self.nBaud = len(code[0])
570 self.nBaud = len(code[0])
569
571
570 if (osamp != None) and (osamp >1):
572 if (osamp != None) and (osamp >1):
571 self.osamp = osamp
573 self.osamp = osamp
572 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
574 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
573 self.nBaud = self.nBaud*self.osamp
575 self.nBaud = self.nBaud*self.osamp
574
576
575 self.__nChannels = dataOut.nChannels
577 self.__nChannels = dataOut.nChannels
576 self.__nProfiles = dataOut.nProfiles
578 self.__nProfiles = dataOut.nProfiles
577 self.__nHeis = dataOut.nHeights
579 self.__nHeis = dataOut.nHeights
578
580
579 if self.__nHeis < self.nBaud:
581 if self.__nHeis < self.nBaud:
580 raise ValueError, 'Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
582 raise ValueError, 'Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
581
583
582 #Frequency
584 #Frequency
583 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
585 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
584
586
585 __codeBuffer[:,0:self.nBaud] = self.code
587 __codeBuffer[:,0:self.nBaud] = self.code
586
588
587 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
589 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
588
590
589 if dataOut.flagDataAsBlock:
591 if dataOut.flagDataAsBlock:
590
592
591 self.ndatadec = self.__nHeis #- self.nBaud + 1
593 self.ndatadec = self.__nHeis #- self.nBaud + 1
592
594
593 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
595 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
594
596
595 else:
597 else:
596
598
597 #Time
599 #Time
598 self.ndatadec = self.__nHeis #- self.nBaud + 1
600 self.ndatadec = self.__nHeis #- self.nBaud + 1
599
601
600 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
602 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
601
603
602 def __convolutionInFreq(self, data):
604 def __convolutionInFreq(self, data):
603
605
604 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
606 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
605
607
606 fft_data = numpy.fft.fft(data, axis=1)
608 fft_data = numpy.fft.fft(data, axis=1)
607
609
608 conv = fft_data*fft_code
610 conv = fft_data*fft_code
609
611
610 data = numpy.fft.ifft(conv,axis=1)
612 data = numpy.fft.ifft(conv,axis=1)
611
613
612 return data
614 return data
613
615
614 def __convolutionInFreqOpt(self, data):
616 def __convolutionInFreqOpt(self, data):
615
617
616 raise NotImplementedError
618 raise NotImplementedError
617
619
618 def __convolutionInTime(self, data):
620 def __convolutionInTime(self, data):
619
621
620 code = self.code[self.__profIndex]
622 code = self.code[self.__profIndex]
621
623
622 for i in range(self.__nChannels):
624 for i in range(self.__nChannels):
623 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
625 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
624
626
625 return self.datadecTime
627 return self.datadecTime
626
628
629 #@profile
630 def oldCorrelate(self, i, data, code_block):
631 profilesList = xrange(self.__nProfiles)
632 for j in profilesList:
633 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
634
635 @profile
627 def __convolutionByBlockInTime(self, data):
636 def __convolutionByBlockInTime(self, data):
628
637
629 repetitions = self.__nProfiles / self.nCode
638 repetitions = self.__nProfiles / self.nCode
630
639
631 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
640 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
632 junk = junk.flatten()
641 junk = junk.flatten()
633 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
642 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
634
643
635 for i in range(self.__nChannels):
644
636 for j in range(self.__nProfiles):
645 # def toVectorize(a,b):
637 print self.datadecTime[i,j,:].shape
646 # return numpy.correlate(a,b, mode='full')
638 print numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:].shape
647 # vectorized = numpy.vectorize(toVectorize, signature='(n),(m)->(k)')
639 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
648 a = time()
640
649 for i in range(self.__nChannels):
650 # self.datadecTime[i,:,:] = numpy.array([numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:] for j in profilesList ])
651 # def func(i, j):
652 # self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
653 # map(lambda j: func(i, j), range(self.__nProfiles))
654 #print data[i,:,:].shape
655 # self.datadecTime[i,:,:] = vectorized(data[i,:,:], code_block[:,:])[:,self.nBaud-1:]
656 self.oldCorrelate(i, data, code_block)
657 print self.datadecTime[i,:,:]
658 # print data[i,:,:]
659 # print cSchain.correlateByBlock(data[i,:,:], code_block, 2)
660 self.datadecTime[i,:,:] = cSchain.correlateByBlock(data[i,:,:], code_block, 2)
661 print self.datadecTime[i,:,:]
662 #print self.datadecTime[i,:,:].shape
663 print time() - a
641 return self.datadecTime
664 return self.datadecTime
665
642
666
643 def __convolutionByBlockInFreq(self, data):
667 def __convolutionByBlockInFreq(self, data):
644
668
645 raise NotImplementedError, "Decoder by frequency fro Blocks not implemented"
669 raise NotImplementedError, "Decoder by frequency fro Blocks not implemented"
646
670
647
671
648 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
672 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
649
673
650 fft_data = numpy.fft.fft(data, axis=2)
674 fft_data = numpy.fft.fft(data, axis=2)
651
675
652 conv = fft_data*fft_code
676 conv = fft_data*fft_code
653
677
654 data = numpy.fft.ifft(conv,axis=2)
678 data = numpy.fft.ifft(conv,axis=2)
655
679
656 return data
680 return data
657
681
682
658 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
683 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
659
684
660 if dataOut.flagDecodeData:
685 if dataOut.flagDecodeData:
661 print "This data is already decoded, recoding again ..."
686 print "This data is already decoded, recoding again ..."
662
687
663 if not self.isConfig:
688 if not self.isConfig:
664
689
665 if code is None:
690 if code is None:
666 if dataOut.code is None:
691 if dataOut.code is None:
667 raise ValueError, "Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type
692 raise ValueError, "Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type
668
693
669 code = dataOut.code
694 code = dataOut.code
670 else:
695 else:
671 code = numpy.array(code).reshape(nCode,nBaud)
696 code = numpy.array(code).reshape(nCode,nBaud)
672
697
673 self.setup(code, osamp, dataOut)
698 self.setup(code, osamp, dataOut)
674
699
675 self.isConfig = True
700 self.isConfig = True
676
701
677 if mode == 3:
702 if mode == 3:
678 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
703 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
679
704
680 if times != None:
705 if times != None:
681 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
706 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
682
707
683 if self.code is None:
708 if self.code is None:
684 print "Fail decoding: Code is not defined."
709 print "Fail decoding: Code is not defined."
685 return
710 return
686
711
712 self.__nProfiles = dataOut.nProfiles
687 datadec = None
713 datadec = None
714
688 if mode == 3:
715 if mode == 3:
689 mode = 0
716 mode = 0
690
717
691 if dataOut.flagDataAsBlock:
718 if dataOut.flagDataAsBlock:
692 """
719 """
693 Decoding when data have been read as block,
720 Decoding when data have been read as block,
694 """
721 """
695
722
696 if mode == 0:
723 if mode == 0:
697 datadec = self.__convolutionByBlockInTime(dataOut.data)
724 datadec = self.__convolutionByBlockInTime(dataOut.data)
698 if mode == 1:
725 if mode == 1:
699 datadec = self.__convolutionByBlockInFreq(dataOut.data)
726 datadec = self.__convolutionByBlockInFreq(dataOut.data)
700 else:
727 else:
701 """
728 """
702 Decoding when data have been read profile by profile
729 Decoding when data have been read profile by profile
703 """
730 """
704 if mode == 0:
731 if mode == 0:
705 datadec = self.__convolutionInTime(dataOut.data)
732 datadec = self.__convolutionInTime(dataOut.data)
706
733
707 if mode == 1:
734 if mode == 1:
708 datadec = self.__convolutionInFreq(dataOut.data)
735 datadec = self.__convolutionInFreq(dataOut.data)
709
736
710 if mode == 2:
737 if mode == 2:
711 datadec = self.__convolutionInFreqOpt(dataOut.data)
738 datadec = self.__convolutionInFreqOpt(dataOut.data)
712
739
713 if datadec is None:
740 if datadec is None:
714 raise ValueError, "Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode
741 raise ValueError, "Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode
715
742
716 dataOut.code = self.code
743 dataOut.code = self.code
717 dataOut.nCode = self.nCode
744 dataOut.nCode = self.nCode
718 dataOut.nBaud = self.nBaud
745 dataOut.nBaud = self.nBaud
719
746
720 dataOut.data = datadec
747 dataOut.data = datadec
721
748
722 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
749 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
723
750
724 dataOut.flagDecodeData = True #asumo q la data esta decodificada
751 dataOut.flagDecodeData = True #asumo q la data esta decodificada
725
752
726 if self.__profIndex == self.nCode-1:
753 if self.__profIndex == self.nCode-1:
727 self.__profIndex = 0
754 self.__profIndex = 0
728 return 1
755 return 1
729
756
730 self.__profIndex += 1
757 self.__profIndex += 1
731
758
732 return 1
759 return 1
733 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
760 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
734
761
735
762
736 class ProfileConcat(Operation):
763 class ProfileConcat(Operation):
737
764
738 isConfig = False
765 isConfig = False
739 buffer = None
766 buffer = None
740
767
741 def __init__(self, **kwargs):
768 def __init__(self, **kwargs):
742
769
743 Operation.__init__(self, **kwargs)
770 Operation.__init__(self, **kwargs)
744 self.profileIndex = 0
771 self.profileIndex = 0
745
772
746 def reset(self):
773 def reset(self):
747 self.buffer = numpy.zeros_like(self.buffer)
774 self.buffer = numpy.zeros_like(self.buffer)
748 self.start_index = 0
775 self.start_index = 0
749 self.times = 1
776 self.times = 1
750
777
751 def setup(self, data, m, n=1):
778 def setup(self, data, m, n=1):
752 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
779 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
753 self.nHeights = data.shape[1]#.nHeights
780 self.nHeights = data.shape[1]#.nHeights
754 self.start_index = 0
781 self.start_index = 0
755 self.times = 1
782 self.times = 1
756
783
757 def concat(self, data):
784 def concat(self, data):
758
785
759 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
786 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
760 self.start_index = self.start_index + self.nHeights
787 self.start_index = self.start_index + self.nHeights
761
788
762 def run(self, dataOut, m):
789 def run(self, dataOut, m):
763
790
764 dataOut.flagNoData = True
791 dataOut.flagNoData = True
765
792
766 if not self.isConfig:
793 if not self.isConfig:
767 self.setup(dataOut.data, m, 1)
794 self.setup(dataOut.data, m, 1)
768 self.isConfig = True
795 self.isConfig = True
769
796
770 if dataOut.flagDataAsBlock:
797 if dataOut.flagDataAsBlock:
771 raise ValueError, "ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False"
798 raise ValueError, "ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False"
772
799
773 else:
800 else:
774 self.concat(dataOut.data)
801 self.concat(dataOut.data)
775 self.times += 1
802 self.times += 1
776 if self.times > m:
803 if self.times > m:
777 dataOut.data = self.buffer
804 dataOut.data = self.buffer
778 self.reset()
805 self.reset()
779 dataOut.flagNoData = False
806 dataOut.flagNoData = False
780 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
807 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
781 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
808 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
782 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
809 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
783 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
810 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
784 dataOut.ippSeconds *= m
811 dataOut.ippSeconds *= m
785
812
786 class ProfileSelector(Operation):
813 class ProfileSelector(Operation):
787
814
788 profileIndex = None
815 profileIndex = None
789 # Tamanho total de los perfiles
816 # Tamanho total de los perfiles
790 nProfiles = None
817 nProfiles = None
791
818
792 def __init__(self, **kwargs):
819 def __init__(self, **kwargs):
793
820
794 Operation.__init__(self, **kwargs)
821 Operation.__init__(self, **kwargs)
795 self.profileIndex = 0
822 self.profileIndex = 0
796
823
797 def incProfileIndex(self):
824 def incProfileIndex(self):
798
825
799 self.profileIndex += 1
826 self.profileIndex += 1
800
827
801 if self.profileIndex >= self.nProfiles:
828 if self.profileIndex >= self.nProfiles:
802 self.profileIndex = 0
829 self.profileIndex = 0
803
830
804 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
831 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
805
832
806 if profileIndex < minIndex:
833 if profileIndex < minIndex:
807 return False
834 return False
808
835
809 if profileIndex > maxIndex:
836 if profileIndex > maxIndex:
810 return False
837 return False
811
838
812 return True
839 return True
813
840
814 def isThisProfileInList(self, profileIndex, profileList):
841 def isThisProfileInList(self, profileIndex, profileList):
815
842
816 if profileIndex not in profileList:
843 if profileIndex not in profileList:
817 return False
844 return False
818
845
819 return True
846 return True
820
847
821 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
848 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
822
849
823 """
850 """
824 ProfileSelector:
851 ProfileSelector:
825
852
826 Inputs:
853 Inputs:
827 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
854 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
828
855
829 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
856 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
830
857
831 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
858 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
832
859
833 """
860 """
834
861
835 if rangeList is not None:
862 if rangeList is not None:
836 if type(rangeList[0]) not in (tuple, list):
863 if type(rangeList[0]) not in (tuple, list):
837 rangeList = [rangeList]
864 rangeList = [rangeList]
838
865
839 dataOut.flagNoData = True
866 dataOut.flagNoData = True
840
867
841 if dataOut.flagDataAsBlock:
868 if dataOut.flagDataAsBlock:
842 """
869 """
843 data dimension = [nChannels, nProfiles, nHeis]
870 data dimension = [nChannels, nProfiles, nHeis]
844 """
871 """
845 if profileList != None:
872 if profileList != None:
846 dataOut.data = dataOut.data[:,profileList,:]
873 dataOut.data = dataOut.data[:,profileList,:]
847
874
848 if profileRangeList != None:
875 if profileRangeList != None:
849 minIndex = profileRangeList[0]
876 minIndex = profileRangeList[0]
850 maxIndex = profileRangeList[1]
877 maxIndex = profileRangeList[1]
851 profileList = range(minIndex, maxIndex+1)
878 profileList = range(minIndex, maxIndex+1)
852
879
853 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
880 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
854
881
855 if rangeList != None:
882 if rangeList != None:
856
883
857 profileList = []
884 profileList = []
858
885
859 for thisRange in rangeList:
886 for thisRange in rangeList:
860 minIndex = thisRange[0]
887 minIndex = thisRange[0]
861 maxIndex = thisRange[1]
888 maxIndex = thisRange[1]
862
889
863 profileList.extend(range(minIndex, maxIndex+1))
890 profileList.extend(range(minIndex, maxIndex+1))
864
891
865 dataOut.data = dataOut.data[:,profileList,:]
892 dataOut.data = dataOut.data[:,profileList,:]
866
893
867 dataOut.nProfiles = len(profileList)
894 dataOut.nProfiles = len(profileList)
868 dataOut.profileIndex = dataOut.nProfiles - 1
895 dataOut.profileIndex = dataOut.nProfiles - 1
869 dataOut.flagNoData = False
896 dataOut.flagNoData = False
870
897
871 return True
898 return True
872
899
873 """
900 """
874 data dimension = [nChannels, nHeis]
901 data dimension = [nChannels, nHeis]
875 """
902 """
876
903
877 if profileList != None:
904 if profileList != None:
878
905
879 if self.isThisProfileInList(dataOut.profileIndex, profileList):
906 if self.isThisProfileInList(dataOut.profileIndex, profileList):
880
907
881 self.nProfiles = len(profileList)
908 self.nProfiles = len(profileList)
882 dataOut.nProfiles = self.nProfiles
909 dataOut.nProfiles = self.nProfiles
883 dataOut.profileIndex = self.profileIndex
910 dataOut.profileIndex = self.profileIndex
884 dataOut.flagNoData = False
911 dataOut.flagNoData = False
885
912
886 self.incProfileIndex()
913 self.incProfileIndex()
887 return True
914 return True
888
915
889 if profileRangeList != None:
916 if profileRangeList != None:
890
917
891 minIndex = profileRangeList[0]
918 minIndex = profileRangeList[0]
892 maxIndex = profileRangeList[1]
919 maxIndex = profileRangeList[1]
893
920
894 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
921 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
895
922
896 self.nProfiles = maxIndex - minIndex + 1
923 self.nProfiles = maxIndex - minIndex + 1
897 dataOut.nProfiles = self.nProfiles
924 dataOut.nProfiles = self.nProfiles
898 dataOut.profileIndex = self.profileIndex
925 dataOut.profileIndex = self.profileIndex
899 dataOut.flagNoData = False
926 dataOut.flagNoData = False
900
927
901 self.incProfileIndex()
928 self.incProfileIndex()
902 return True
929 return True
903
930
904 if rangeList != None:
931 if rangeList != None:
905
932
906 nProfiles = 0
933 nProfiles = 0
907
934
908 for thisRange in rangeList:
935 for thisRange in rangeList:
909 minIndex = thisRange[0]
936 minIndex = thisRange[0]
910 maxIndex = thisRange[1]
937 maxIndex = thisRange[1]
911
938
912 nProfiles += maxIndex - minIndex + 1
939 nProfiles += maxIndex - minIndex + 1
913
940
914 for thisRange in rangeList:
941 for thisRange in rangeList:
915
942
916 minIndex = thisRange[0]
943 minIndex = thisRange[0]
917 maxIndex = thisRange[1]
944 maxIndex = thisRange[1]
918
945
919 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
946 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
920
947
921 self.nProfiles = nProfiles
948 self.nProfiles = nProfiles
922 dataOut.nProfiles = self.nProfiles
949 dataOut.nProfiles = self.nProfiles
923 dataOut.profileIndex = self.profileIndex
950 dataOut.profileIndex = self.profileIndex
924 dataOut.flagNoData = False
951 dataOut.flagNoData = False
925
952
926 self.incProfileIndex()
953 self.incProfileIndex()
927
954
928 break
955 break
929
956
930 return True
957 return True
931
958
932
959
933 if beam != None: #beam is only for AMISR data
960 if beam != None: #beam is only for AMISR data
934 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
961 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
935 dataOut.flagNoData = False
962 dataOut.flagNoData = False
936 dataOut.profileIndex = self.profileIndex
963 dataOut.profileIndex = self.profileIndex
937
964
938 self.incProfileIndex()
965 self.incProfileIndex()
939
966
940 return True
967 return True
941
968
942 raise ValueError, "ProfileSelector needs profileList, profileRangeList or rangeList parameter"
969 raise ValueError, "ProfileSelector needs profileList, profileRangeList or rangeList parameter"
943
970
944 return False
971 return False
945
972
946 class Reshaper(Operation):
973 class Reshaper(Operation):
947
974
948 def __init__(self, **kwargs):
975 def __init__(self, **kwargs):
949
976
950 Operation.__init__(self, **kwargs)
977 Operation.__init__(self, **kwargs)
951
978
952 self.__buffer = None
979 self.__buffer = None
953 self.__nitems = 0
980 self.__nitems = 0
954
981
955 def __appendProfile(self, dataOut, nTxs):
982 def __appendProfile(self, dataOut, nTxs):
956
983
957 if self.__buffer is None:
984 if self.__buffer is None:
958 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
985 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
959 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
986 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
960
987
961 ini = dataOut.nHeights * self.__nitems
988 ini = dataOut.nHeights * self.__nitems
962 end = ini + dataOut.nHeights
989 end = ini + dataOut.nHeights
963
990
964 self.__buffer[:, ini:end] = dataOut.data
991 self.__buffer[:, ini:end] = dataOut.data
965
992
966 self.__nitems += 1
993 self.__nitems += 1
967
994
968 return int(self.__nitems*nTxs)
995 return int(self.__nitems*nTxs)
969
996
970 def __getBuffer(self):
997 def __getBuffer(self):
971
998
972 if self.__nitems == int(1./self.__nTxs):
999 if self.__nitems == int(1./self.__nTxs):
973
1000
974 self.__nitems = 0
1001 self.__nitems = 0
975
1002
976 return self.__buffer.copy()
1003 return self.__buffer.copy()
977
1004
978 return None
1005 return None
979
1006
980 def __checkInputs(self, dataOut, shape, nTxs):
1007 def __checkInputs(self, dataOut, shape, nTxs):
981
1008
982 if shape is None and nTxs is None:
1009 if shape is None and nTxs is None:
983 raise ValueError, "Reshaper: shape of factor should be defined"
1010 raise ValueError, "Reshaper: shape of factor should be defined"
984
1011
985 if nTxs:
1012 if nTxs:
986 if nTxs < 0:
1013 if nTxs < 0:
987 raise ValueError, "nTxs should be greater than 0"
1014 raise ValueError, "nTxs should be greater than 0"
988
1015
989 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1016 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
990 raise ValueError, "nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs))
1017 raise ValueError, "nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs))
991
1018
992 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1019 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
993
1020
994 return shape, nTxs
1021 return shape, nTxs
995
1022
996 if len(shape) != 2 and len(shape) != 3:
1023 if len(shape) != 2 and len(shape) != 3:
997 raise ValueError, "shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights)
1024 raise ValueError, "shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights)
998
1025
999 if len(shape) == 2:
1026 if len(shape) == 2:
1000 shape_tuple = [dataOut.nChannels]
1027 shape_tuple = [dataOut.nChannels]
1001 shape_tuple.extend(shape)
1028 shape_tuple.extend(shape)
1002 else:
1029 else:
1003 shape_tuple = list(shape)
1030 shape_tuple = list(shape)
1004
1031
1005 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1032 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1006
1033
1007 return shape_tuple, nTxs
1034 return shape_tuple, nTxs
1008
1035
1009 def run(self, dataOut, shape=None, nTxs=None):
1036 def run(self, dataOut, shape=None, nTxs=None):
1010
1037
1011 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1038 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1012
1039
1013 dataOut.flagNoData = True
1040 dataOut.flagNoData = True
1014 profileIndex = None
1041 profileIndex = None
1015
1042
1016 if dataOut.flagDataAsBlock:
1043 if dataOut.flagDataAsBlock:
1017
1044
1018 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1045 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1019 dataOut.flagNoData = False
1046 dataOut.flagNoData = False
1020
1047
1021 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1048 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1022
1049
1023 else:
1050 else:
1024
1051
1025 if self.__nTxs < 1:
1052 if self.__nTxs < 1:
1026
1053
1027 self.__appendProfile(dataOut, self.__nTxs)
1054 self.__appendProfile(dataOut, self.__nTxs)
1028 new_data = self.__getBuffer()
1055 new_data = self.__getBuffer()
1029
1056
1030 if new_data is not None:
1057 if new_data is not None:
1031 dataOut.data = new_data
1058 dataOut.data = new_data
1032 dataOut.flagNoData = False
1059 dataOut.flagNoData = False
1033
1060
1034 profileIndex = dataOut.profileIndex*nTxs
1061 profileIndex = dataOut.profileIndex*nTxs
1035
1062
1036 else:
1063 else:
1037 raise ValueError, "nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)"
1064 raise ValueError, "nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)"
1038
1065
1039 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1066 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1040
1067
1041 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1068 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1042
1069
1043 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1070 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1044
1071
1045 dataOut.profileIndex = profileIndex
1072 dataOut.profileIndex = profileIndex
1046
1073
1047 dataOut.ippSeconds /= self.__nTxs
1074 dataOut.ippSeconds /= self.__nTxs
1048
1075
1049 class SplitProfiles(Operation):
1076 class SplitProfiles(Operation):
1050
1077
1051 def __init__(self, **kwargs):
1078 def __init__(self, **kwargs):
1052
1079
1053 Operation.__init__(self, **kwargs)
1080 Operation.__init__(self, **kwargs)
1054
1081
1055 def run(self, dataOut, n):
1082 def run(self, dataOut, n):
1056
1083
1057 dataOut.flagNoData = True
1084 dataOut.flagNoData = True
1058 profileIndex = None
1085 profileIndex = None
1059
1086
1060 if dataOut.flagDataAsBlock:
1087 if dataOut.flagDataAsBlock:
1061
1088
1062 #nchannels, nprofiles, nsamples
1089 #nchannels, nprofiles, nsamples
1063 shape = dataOut.data.shape
1090 shape = dataOut.data.shape
1064
1091
1065 if shape[2] % n != 0:
1092 if shape[2] % n != 0:
1066 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[2])
1093 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[2])
1067
1094
1068 new_shape = shape[0], shape[1]*n, shape[2]/n
1095 new_shape = shape[0], shape[1]*n, shape[2]/n
1069
1096
1070 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1097 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1071 dataOut.flagNoData = False
1098 dataOut.flagNoData = False
1072
1099
1073 profileIndex = int(dataOut.nProfiles/n) - 1
1100 profileIndex = int(dataOut.nProfiles/n) - 1
1074
1101
1075 else:
1102 else:
1076
1103
1077 raise ValueError, "Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)"
1104 raise ValueError, "Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)"
1078
1105
1079 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1106 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1080
1107
1081 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1108 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1082
1109
1083 dataOut.nProfiles = int(dataOut.nProfiles*n)
1110 dataOut.nProfiles = int(dataOut.nProfiles*n)
1084
1111
1085 dataOut.profileIndex = profileIndex
1112 dataOut.profileIndex = profileIndex
1086
1113
1087 dataOut.ippSeconds /= n
1114 dataOut.ippSeconds /= n
1088
1115
1089 class CombineProfiles(Operation):
1116 class CombineProfiles(Operation):
1090
1117
1091 def __init__(self, **kwargs):
1118 def __init__(self, **kwargs):
1092
1119
1093 Operation.__init__(self, **kwargs)
1120 Operation.__init__(self, **kwargs)
1094
1121
1095 self.__remData = None
1122 self.__remData = None
1096 self.__profileIndex = 0
1123 self.__profileIndex = 0
1097
1124
1098 def run(self, dataOut, n):
1125 def run(self, dataOut, n):
1099
1126
1100 dataOut.flagNoData = True
1127 dataOut.flagNoData = True
1101 profileIndex = None
1128 profileIndex = None
1102
1129
1103 if dataOut.flagDataAsBlock:
1130 if dataOut.flagDataAsBlock:
1104
1131
1105 #nchannels, nprofiles, nsamples
1132 #nchannels, nprofiles, nsamples
1106 shape = dataOut.data.shape
1133 shape = dataOut.data.shape
1107 new_shape = shape[0], shape[1]/n, shape[2]*n
1134 new_shape = shape[0], shape[1]/n, shape[2]*n
1108
1135
1109 if shape[1] % n != 0:
1136 if shape[1] % n != 0:
1110 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[1])
1137 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[1])
1111
1138
1112 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1139 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1113 dataOut.flagNoData = False
1140 dataOut.flagNoData = False
1114
1141
1115 profileIndex = int(dataOut.nProfiles*n) - 1
1142 profileIndex = int(dataOut.nProfiles*n) - 1
1116
1143
1117 else:
1144 else:
1118
1145
1119 #nchannels, nsamples
1146 #nchannels, nsamples
1120 if self.__remData is None:
1147 if self.__remData is None:
1121 newData = dataOut.data
1148 newData = dataOut.data
1122 else:
1149 else:
1123 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1150 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1124
1151
1125 self.__profileIndex += 1
1152 self.__profileIndex += 1
1126
1153
1127 if self.__profileIndex < n:
1154 if self.__profileIndex < n:
1128 self.__remData = newData
1155 self.__remData = newData
1129 #continue
1156 #continue
1130 return
1157 return
1131
1158
1132 self.__profileIndex = 0
1159 self.__profileIndex = 0
1133 self.__remData = None
1160 self.__remData = None
1134
1161
1135 dataOut.data = newData
1162 dataOut.data = newData
1136 dataOut.flagNoData = False
1163 dataOut.flagNoData = False
1137
1164
1138 profileIndex = dataOut.profileIndex/n
1165 profileIndex = dataOut.profileIndex/n
1139
1166
1140
1167
1141 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1168 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1142
1169
1143 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1170 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1144
1171
1145 dataOut.nProfiles = int(dataOut.nProfiles/n)
1172 dataOut.nProfiles = int(dataOut.nProfiles/n)
1146
1173
1147 dataOut.profileIndex = profileIndex
1174 dataOut.profileIndex = profileIndex
1148
1175
1149 dataOut.ippSeconds *= n
1176 dataOut.ippSeconds *= n
1150
1177
1151 # import collections
1178 # import collections
1152 # from scipy.stats import mode
1179 # from scipy.stats import mode
1153 #
1180 #
1154 # class Synchronize(Operation):
1181 # class Synchronize(Operation):
1155 #
1182 #
1156 # isConfig = False
1183 # isConfig = False
1157 # __profIndex = 0
1184 # __profIndex = 0
1158 #
1185 #
1159 # def __init__(self, **kwargs):
1186 # def __init__(self, **kwargs):
1160 #
1187 #
1161 # Operation.__init__(self, **kwargs)
1188 # Operation.__init__(self, **kwargs)
1162 # # self.isConfig = False
1189 # # self.isConfig = False
1163 # self.__powBuffer = None
1190 # self.__powBuffer = None
1164 # self.__startIndex = 0
1191 # self.__startIndex = 0
1165 # self.__pulseFound = False
1192 # self.__pulseFound = False
1166 #
1193 #
1167 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1194 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1168 #
1195 #
1169 # #Read data
1196 # #Read data
1170 #
1197 #
1171 # powerdB = dataOut.getPower(channel = channel)
1198 # powerdB = dataOut.getPower(channel = channel)
1172 # noisedB = dataOut.getNoise(channel = channel)[0]
1199 # noisedB = dataOut.getNoise(channel = channel)[0]
1173 #
1200 #
1174 # self.__powBuffer.extend(powerdB.flatten())
1201 # self.__powBuffer.extend(powerdB.flatten())
1175 #
1202 #
1176 # dataArray = numpy.array(self.__powBuffer)
1203 # dataArray = numpy.array(self.__powBuffer)
1177 #
1204 #
1178 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1205 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1179 #
1206 #
1180 # maxValue = numpy.nanmax(filteredPower)
1207 # maxValue = numpy.nanmax(filteredPower)
1181 #
1208 #
1182 # if maxValue < noisedB + 10:
1209 # if maxValue < noisedB + 10:
1183 # #No se encuentra ningun pulso de transmision
1210 # #No se encuentra ningun pulso de transmision
1184 # return None
1211 # return None
1185 #
1212 #
1186 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1213 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1187 #
1214 #
1188 # if len(maxValuesIndex) < 2:
1215 # if len(maxValuesIndex) < 2:
1189 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1216 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1190 # return None
1217 # return None
1191 #
1218 #
1192 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1219 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1193 #
1220 #
1194 # #Seleccionar solo valores con un espaciamiento de nSamples
1221 # #Seleccionar solo valores con un espaciamiento de nSamples
1195 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1222 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1196 #
1223 #
1197 # if len(pulseIndex) < 2:
1224 # if len(pulseIndex) < 2:
1198 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1225 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1199 # return None
1226 # return None
1200 #
1227 #
1201 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1228 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1202 #
1229 #
1203 # #remover senales que se distancien menos de 10 unidades o muestras
1230 # #remover senales que se distancien menos de 10 unidades o muestras
1204 # #(No deberian existir IPP menor a 10 unidades)
1231 # #(No deberian existir IPP menor a 10 unidades)
1205 #
1232 #
1206 # realIndex = numpy.where(spacing > 10 )[0]
1233 # realIndex = numpy.where(spacing > 10 )[0]
1207 #
1234 #
1208 # if len(realIndex) < 2:
1235 # if len(realIndex) < 2:
1209 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1236 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1210 # return None
1237 # return None
1211 #
1238 #
1212 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1239 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1213 # realPulseIndex = pulseIndex[realIndex]
1240 # realPulseIndex = pulseIndex[realIndex]
1214 #
1241 #
1215 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1242 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1216 #
1243 #
1217 # print "IPP = %d samples" %period
1244 # print "IPP = %d samples" %period
1218 #
1245 #
1219 # self.__newNSamples = dataOut.nHeights #int(period)
1246 # self.__newNSamples = dataOut.nHeights #int(period)
1220 # self.__startIndex = int(realPulseIndex[0])
1247 # self.__startIndex = int(realPulseIndex[0])
1221 #
1248 #
1222 # return 1
1249 # return 1
1223 #
1250 #
1224 #
1251 #
1225 # def setup(self, nSamples, nChannels, buffer_size = 4):
1252 # def setup(self, nSamples, nChannels, buffer_size = 4):
1226 #
1253 #
1227 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1254 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1228 # maxlen = buffer_size*nSamples)
1255 # maxlen = buffer_size*nSamples)
1229 #
1256 #
1230 # bufferList = []
1257 # bufferList = []
1231 #
1258 #
1232 # for i in range(nChannels):
1259 # for i in range(nChannels):
1233 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1260 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1234 # maxlen = buffer_size*nSamples)
1261 # maxlen = buffer_size*nSamples)
1235 #
1262 #
1236 # bufferList.append(bufferByChannel)
1263 # bufferList.append(bufferByChannel)
1237 #
1264 #
1238 # self.__nSamples = nSamples
1265 # self.__nSamples = nSamples
1239 # self.__nChannels = nChannels
1266 # self.__nChannels = nChannels
1240 # self.__bufferList = bufferList
1267 # self.__bufferList = bufferList
1241 #
1268 #
1242 # def run(self, dataOut, channel = 0):
1269 # def run(self, dataOut, channel = 0):
1243 #
1270 #
1244 # if not self.isConfig:
1271 # if not self.isConfig:
1245 # nSamples = dataOut.nHeights
1272 # nSamples = dataOut.nHeights
1246 # nChannels = dataOut.nChannels
1273 # nChannels = dataOut.nChannels
1247 # self.setup(nSamples, nChannels)
1274 # self.setup(nSamples, nChannels)
1248 # self.isConfig = True
1275 # self.isConfig = True
1249 #
1276 #
1250 # #Append new data to internal buffer
1277 # #Append new data to internal buffer
1251 # for thisChannel in range(self.__nChannels):
1278 # for thisChannel in range(self.__nChannels):
1252 # bufferByChannel = self.__bufferList[thisChannel]
1279 # bufferByChannel = self.__bufferList[thisChannel]
1253 # bufferByChannel.extend(dataOut.data[thisChannel])
1280 # bufferByChannel.extend(dataOut.data[thisChannel])
1254 #
1281 #
1255 # if self.__pulseFound:
1282 # if self.__pulseFound:
1256 # self.__startIndex -= self.__nSamples
1283 # self.__startIndex -= self.__nSamples
1257 #
1284 #
1258 # #Finding Tx Pulse
1285 # #Finding Tx Pulse
1259 # if not self.__pulseFound:
1286 # if not self.__pulseFound:
1260 # indexFound = self.__findTxPulse(dataOut, channel)
1287 # indexFound = self.__findTxPulse(dataOut, channel)
1261 #
1288 #
1262 # if indexFound == None:
1289 # if indexFound == None:
1263 # dataOut.flagNoData = True
1290 # dataOut.flagNoData = True
1264 # return
1291 # return
1265 #
1292 #
1266 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1293 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1267 # self.__pulseFound = True
1294 # self.__pulseFound = True
1268 # self.__startIndex = indexFound
1295 # self.__startIndex = indexFound
1269 #
1296 #
1270 # #If pulse was found ...
1297 # #If pulse was found ...
1271 # for thisChannel in range(self.__nChannels):
1298 # for thisChannel in range(self.__nChannels):
1272 # bufferByChannel = self.__bufferList[thisChannel]
1299 # bufferByChannel = self.__bufferList[thisChannel]
1273 # #print self.__startIndex
1300 # #print self.__startIndex
1274 # x = numpy.array(bufferByChannel)
1301 # x = numpy.array(bufferByChannel)
1275 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1302 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1276 #
1303 #
1277 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1304 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1278 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1305 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1279 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1306 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1280 #
1307 #
1281 # dataOut.data = self.__arrayBuffer
1308 # dataOut.data = self.__arrayBuffer
1282 #
1309 #
1283 # self.__startIndex += self.__newNSamples
1310 # self.__startIndex += self.__newNSamples
1284 #
1311 #
1285 # return
1312 # return
Show file before | Show file after | Hide comments
@@ -1,85 +1,85
1
1
2 import os, sys
2 import os, sys
3
3
4 from schainpy.controller import Project
4 from schainpy.controller import Project
5
5
6 controllerObj = Project()
6 controllerObj = Project()
7 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
7 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
8
8
9 #-------------------------------------- Setup -----------------------------------------
9 #-------------------------------------- Setup -----------------------------------------
10 #Verificar estas variables
10 #Verificar estas variables
11
11
12 #Path para los archivos
12 #Path para los archivos
13 # path = '/mnt/jars/2016_08/NOCHE'
13 # path = '/mnt/jars/2016_08/NOCHE'
14 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
14 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
15 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
15 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
16 path = '/media/nanosat/NewVolumen/JASMET/2016_08/DIA'
16 path = '/home/nanosat/data/jasmet'
17
17
18 #Path para los graficos
18 #Path para los graficos
19 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
19 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
20
20
21 #Path para los archivos HDF5 de meteoros
21 #Path para los archivos HDF5 de meteoros
22 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
22 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
23
23
24 #Fechas para busqueda de archivos
24 #Fechas para busqueda de archivos
25 startDate = '2016/08/29'
25 startDate = '2010/08/29'
26 endDate = '2016/09/11'
26 endDate = '2017/09/11'
27 #Horas para busqueda de archivos
27 #Horas para busqueda de archivos
28 startTime = '00:00:00'
28 startTime = '00:00:00'
29 endTime = '23:59:59'
29 endTime = '23:59:59'
30
30
31
31
32 #------------------------------ Voltage Reading Unit ----------------------------------
32 #------------------------------ Voltage Reading Unit ----------------------------------
33
33
34 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
34 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
35 path=path,
35 path=path,
36 startDate=startDate,
36 startDate=startDate,
37 endDate=endDate,
37 endDate=endDate,
38 startTime=startTime,
38 startTime=startTime,
39 endTime=endTime,
39 endTime=endTime,
40 online=0,
40 online=0,
41 delay=30,
41 delay=30,
42 walk=1,
42 walk=1,
43 getblock=1,
43 getblock=1,
44 blocktime=100)
44 blocktime=100)
45
45
46 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
46 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
47
47
48 #-------------------------- Voltage Processing Unit ------------------------------------
48 #-------------------------- Voltage Processing Unit ------------------------------------
49
49
50 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
50 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
51
51
52 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
52 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
53 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
53 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
54
54
55 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
55 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
56 opObj01.addParameter(name='frequency', value='30.e6', format='float')
56 opObj01.addParameter(name='frequency', value='30.e6', format='float')
57
57
58 opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
58 opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
59 opObj01.addParameter(name='topLim', value='73', format='int')
59 opObj01.addParameter(name='topLim', value='73', format='int')
60 opObj01.addParameter(name='botLim', value='71', format='int')
60 opObj01.addParameter(name='botLim', value='71', format='int')
61
61
62 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
62 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
63
63
64 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
64 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
65 opObj03.addParameter(name='n', value='2', format='int')
65 opObj03.addParameter(name='n', value='2', format='int')
66
66
67 #--------------------------- Parameters Processing Unit ------------------------------------
67 #--------------------------- Parameters Processing Unit ------------------------------------
68
68
69 procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
69 procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
70 #
70 #
71 opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
71 opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
72 opObj10.addParameter(name='azimuth', value='45', format='float')
72 opObj10.addParameter(name='azimuth', value='45', format='float')
73 opObj10.addParameter(name='hmin', value='60', format='float')
73 opObj10.addParameter(name='hmin', value='60', format='float')
74 opObj10.addParameter(name='hmax', value='120', format='float')
74 opObj10.addParameter(name='hmax', value='120', format='float')
75
75
76 opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
76 opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
77 opObj12.addParameter(name='path', value=pathfile)
77 opObj12.addParameter(name='path', value=pathfile)
78 opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
78 opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
79 opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
79 opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
80 opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
80 opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
81 opObj12.addParameter(name='mode',value='2',format='int')
81 opObj12.addParameter(name='mode',value='2',format='int')
82
82
83 #--------------------------------------------------------------------------------------------------
83 #--------------------------------------------------------------------------------------------------
84
84
85 controllerObj.start()
85 controllerObj.start()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now