##// END OF EJS Templates
schain
RSS
formatting
José Chávez -
r1082:831a23834344
parent child
Show More
Side by Side Unified
Context file:
r1082:831a23834344 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1297 +1,1313
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, cpu_count
12 from multiprocessing import Process, cpu_count
13
13
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 from xml.dom import minidom
15 from xml.dom import minidom
16
16
17 import schainpy
17 import schainpy
18 import schainpy.admin
18 import schainpy.admin
19 from schainpy.model import *
19 from schainpy.model import *
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 DTYPES = {
22 DTYPES = {
23 'Voltage': '.r',
23 'Voltage': '.r',
24 'Spectra': '.pdata'
24 'Spectra': '.pdata'
25 }
25 }
26
26
27
27 def MPProject(project, n=cpu_count()):
28 def MPProject(project, n=cpu_count()):
28 '''
29 '''
29 Project wrapper to run schain in n processes
30 Project wrapper to run schain in n processes
30 '''
31 '''
31
32
32 rconf = project.getReadUnitObj()
33 rconf = project.getReadUnitObj()
33 op = rconf.getOperationObj('run')
34 op = rconf.getOperationObj('run')
34 dt1 = op.getParameterValue('startDate')
35 dt1 = op.getParameterValue('startDate')
35 dt2 = op.getParameterValue('endDate')
36 dt2 = op.getParameterValue('endDate')
36 days = (dt2 - dt1).days
37 days = (dt2 - dt1).days
37
38
38 for day in range(days+1):
39 for day in range(days + 1):
39 skip = 0
40 skip = 0
40 cursor = 0
41 cursor = 0
41 processes = []
42 processes = []
42 dt = dt1 + datetime.timedelta(day)
43 dt = dt1 + datetime.timedelta(day)
43 dt_str = dt.strftime('%Y/%m/%d')
44 dt_str = dt.strftime('%Y/%m/%d')
44 reader = JRODataReader()
45 reader = JRODataReader()
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 startDate=dt,
47 startDate=dt,
47 endDate=dt,
48 endDate=dt,
48 ext=DTYPES[rconf.datatype])
49 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
50 nFiles = len(files)
50 if nFiles == 0:
51 if nFiles == 0:
51 continue
52 continue
52 skip = int(math.ceil(nFiles/n))
53 skip = int(math.ceil(nFiles / n))
53 while nFiles > cursor*skip:
54 while nFiles > cursor * skip:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 skip=skip)
56 skip=skip)
56 p = project.clone()
57 p = project.clone()
57 p.start()
58 p.start()
58 processes.append(p)
59 processes.append(p)
59 cursor += 1
60 cursor += 1
60
61
61 def beforeExit(exctype, value, trace):
62 def beforeExit(exctype, value, trace):
62 for process in processes:
63 for process in processes:
63 process.terminate()
64 process.terminate()
64 process.join()
65 process.join()
65 print traceback.print_tb(trace)
66 print traceback.print_tb(trace)
66
67
67 sys.excepthook = beforeExit
68 sys.excepthook = beforeExit
68
69
69 for process in processes:
70 for process in processes:
70 process.join()
71 process.join()
71 process.terminate()
72 process.terminate()
72
73
73 time.sleep(3)
74 time.sleep(3)
74
75
76
75 class ParameterConf():
77 class ParameterConf():
76
78
77 id = None
79 id = None
78 name = None
80 name = None
79 value = None
81 value = None
80 format = None
82 format = None
81
83
82 __formated_value = None
84 __formated_value = None
83
85
84 ELEMENTNAME = 'Parameter'
86 ELEMENTNAME = 'Parameter'
85
87
86 def __init__(self):
88 def __init__(self):
87
89
88 self.format = 'str'
90 self.format = 'str'
89
91
90 def getElementName(self):
92 def getElementName(self):
91
93
92 return self.ELEMENTNAME
94 return self.ELEMENTNAME
93
95
94 def getValue(self):
96 def getValue(self):
95
97
96 value = self.value
98 value = self.value
97 format = self.format
99 format = self.format
98
100
99 if self.__formated_value != None:
101 if self.__formated_value != None:
100
102
101 return self.__formated_value
103 return self.__formated_value
102
104
103 if format == 'obj':
105 if format == 'obj':
104 return value
106 return value
105
107
106 if format == 'str':
108 if format == 'str':
107 self.__formated_value = str(value)
109 self.__formated_value = str(value)
108 return self.__formated_value
110 return self.__formated_value
109
111
110 if value == '':
112 if value == '':
111 raise ValueError, '%s: This parameter value is empty' %self.name
113 raise ValueError, '%s: This parameter value is empty' % self.name
112
114
113 if format == 'list':
115 if format == 'list':
114 strList = value.split(',')
116 strList = value.split(',')
115
117
116 self.__formated_value = strList
118 self.__formated_value = strList
117
119
118 return self.__formated_value
120 return self.__formated_value
119
121
120 if format == 'intlist':
122 if format == 'intlist':
121 '''
123 '''
122 Example:
124 Example:
123 value = (0,1,2)
125 value = (0,1,2)
124 '''
126 '''
125
127
126 new_value = ast.literal_eval(value)
128 new_value = ast.literal_eval(value)
127
129
128 if type(new_value) not in (tuple, list):
130 if type(new_value) not in (tuple, list):
129 new_value = [int(new_value)]
131 new_value = [int(new_value)]
130
132
131 self.__formated_value = new_value
133 self.__formated_value = new_value
132
134
133 return self.__formated_value
135 return self.__formated_value
134
136
135 if format == 'floatlist':
137 if format == 'floatlist':
136 '''
138 '''
137 Example:
139 Example:
138 value = (0.5, 1.4, 2.7)
140 value = (0.5, 1.4, 2.7)
139 '''
141 '''
140
142
141 new_value = ast.literal_eval(value)
143 new_value = ast.literal_eval(value)
142
144
143 if type(new_value) not in (tuple, list):
145 if type(new_value) not in (tuple, list):
144 new_value = [float(new_value)]
146 new_value = [float(new_value)]
145
147
146 self.__formated_value = new_value
148 self.__formated_value = new_value
147
149
148 return self.__formated_value
150 return self.__formated_value
149
151
150 if format == 'date':
152 if format == 'date':
151 strList = value.split('/')
153 strList = value.split('/')
152 intList = [int(x) for x in strList]
154 intList = [int(x) for x in strList]
153 date = datetime.date(intList[0], intList[1], intList[2])
155 date = datetime.date(intList[0], intList[1], intList[2])
154
156
155 self.__formated_value = date
157 self.__formated_value = date
156
158
157 return self.__formated_value
159 return self.__formated_value
158
160
159 if format == 'time':
161 if format == 'time':
160 strList = value.split(':')
162 strList = value.split(':')
161 intList = [int(x) for x in strList]
163 intList = [int(x) for x in strList]
162 time = datetime.time(intList[0], intList[1], intList[2])
164 time = datetime.time(intList[0], intList[1], intList[2])
163
165
164 self.__formated_value = time
166 self.__formated_value = time
165
167
166 return self.__formated_value
168 return self.__formated_value
167
169
168 if format == 'pairslist':
170 if format == 'pairslist':
169 '''
171 '''
170 Example:
172 Example:
171 value = (0,1),(1,2)
173 value = (0,1),(1,2)
172 '''
174 '''
173
175
174 new_value = ast.literal_eval(value)
176 new_value = ast.literal_eval(value)
175
177
176 if type(new_value) not in (tuple, list):
178 if type(new_value) not in (tuple, list):
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
179 raise ValueError, '%s has to be a tuple or list of pairs' % value
178
180
179 if type(new_value[0]) not in (tuple, list):
181 if type(new_value[0]) not in (tuple, list):
180 if len(new_value) != 2:
182 if len(new_value) != 2:
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
182 new_value = [new_value]
184 new_value = [new_value]
183
185
184 for thisPair in new_value:
186 for thisPair in new_value:
185 if len(thisPair) != 2:
187 if len(thisPair) != 2:
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
188 raise ValueError, '%s has to be a tuple or list of pairs' % value
187
189
188 self.__formated_value = new_value
190 self.__formated_value = new_value
189
191
190 return self.__formated_value
192 return self.__formated_value
191
193
192 if format == 'multilist':
194 if format == 'multilist':
193 '''
195 '''
194 Example:
196 Example:
195 value = (0,1,2),(3,4,5)
197 value = (0,1,2),(3,4,5)
196 '''
198 '''
197 multiList = ast.literal_eval(value)
199 multiList = ast.literal_eval(value)
198
200
199 if type(multiList[0]) == int:
201 if type(multiList[0]) == int:
200 multiList = ast.literal_eval('(' + value + ')')
202 multiList = ast.literal_eval('(' + value + ')')
201
203
202 self.__formated_value = multiList
204 self.__formated_value = multiList
203
205
204 return self.__formated_value
206 return self.__formated_value
205
207
206 if format == 'bool':
208 if format == 'bool':
207 value = int(value)
209 value = int(value)
208
210
209 if format == 'int':
211 if format == 'int':
210 value = float(value)
212 value = float(value)
211
213
212 format_func = eval(format)
214 format_func = eval(format)
213
215
214 self.__formated_value = format_func(value)
216 self.__formated_value = format_func(value)
215
217
216 return self.__formated_value
218 return self.__formated_value
217
219
218 def updateId(self, new_id):
220 def updateId(self, new_id):
219
221
220 self.id = str(new_id)
222 self.id = str(new_id)
221
223
222 def setup(self, id, name, value, format='str'):
224 def setup(self, id, name, value, format='str'):
223 self.id = str(id)
225 self.id = str(id)
224 self.name = name
226 self.name = name
225 if format == 'obj':
227 if format == 'obj':
226 self.value = value
228 self.value = value
227 else:
229 else:
228 self.value = str(value)
230 self.value = str(value)
229 self.format = str.lower(format)
231 self.format = str.lower(format)
230
232
231 self.getValue()
233 self.getValue()
232
234
233 return 1
235 return 1
234
236
235 def update(self, name, value, format='str'):
237 def update(self, name, value, format='str'):
236
238
237 self.name = name
239 self.name = name
238 self.value = str(value)
240 self.value = str(value)
239 self.format = format
241 self.format = format
240
242
241 def makeXml(self, opElement):
243 def makeXml(self, opElement):
242 if self.name not in ('queue',):
244 if self.name not in ('queue',):
243 parmElement = SubElement(opElement, self.ELEMENTNAME)
245 parmElement = SubElement(opElement, self.ELEMENTNAME)
244 parmElement.set('id', str(self.id))
246 parmElement.set('id', str(self.id))
245 parmElement.set('name', self.name)
247 parmElement.set('name', self.name)
246 parmElement.set('value', self.value)
248 parmElement.set('value', self.value)
247 parmElement.set('format', self.format)
249 parmElement.set('format', self.format)
248
250
249 def readXml(self, parmElement):
251 def readXml(self, parmElement):
250
252
251 self.id = parmElement.get('id')
253 self.id = parmElement.get('id')
252 self.name = parmElement.get('name')
254 self.name = parmElement.get('name')
253 self.value = parmElement.get('value')
255 self.value = parmElement.get('value')
254 self.format = str.lower(parmElement.get('format'))
256 self.format = str.lower(parmElement.get('format'))
255
257
256 #Compatible with old signal chain version
258 # Compatible with old signal chain version
257 if self.format == 'int' and self.name == 'idfigure':
259 if self.format == 'int' and self.name == 'idfigure':
258 self.name = 'id'
260 self.name = 'id'
259
261
260 def printattr(self):
262 def printattr(self):
261
263
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
264 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
263
265
266
264 class OperationConf():
267 class OperationConf():
265
268
266 id = None
269 id = None
267 name = None
270 name = None
268 priority = None
271 priority = None
269 type = None
272 type = None
270
273
271 parmConfObjList = []
274 parmConfObjList = []
272
275
273 ELEMENTNAME = 'Operation'
276 ELEMENTNAME = 'Operation'
274
277
275 def __init__(self):
278 def __init__(self):
276
279
277 self.id = '0'
280 self.id = '0'
278 self.name = None
281 self.name = None
279 self.priority = None
282 self.priority = None
280 self.type = 'self'
283 self.type = 'self'
281
284
282
283 def __getNewId(self):
285 def __getNewId(self):
284
286
285 return int(self.id)*10 + len(self.parmConfObjList) + 1
287 return int(self.id) * 10 + len(self.parmConfObjList) + 1
286
288
287 def updateId(self, new_id):
289 def updateId(self, new_id):
288
290
289 self.id = str(new_id)
291 self.id = str(new_id)
290
292
291 n = 1
293 n = 1
292 for parmObj in self.parmConfObjList:
294 for parmObj in self.parmConfObjList:
293
295
294 idParm = str(int(new_id)*10 + n)
296 idParm = str(int(new_id) * 10 + n)
295 parmObj.updateId(idParm)
297 parmObj.updateId(idParm)
296
298
297 n += 1
299 n += 1
298
300
299 def getElementName(self):
301 def getElementName(self):
300
302
301 return self.ELEMENTNAME
303 return self.ELEMENTNAME
302
304
303 def getParameterObjList(self):
305 def getParameterObjList(self):
304
306
305 return self.parmConfObjList
307 return self.parmConfObjList
306
308
307 def getParameterObj(self, parameterName):
309 def getParameterObj(self, parameterName):
308
310
309 for parmConfObj in self.parmConfObjList:
311 for parmConfObj in self.parmConfObjList:
310
312
311 if parmConfObj.name != parameterName:
313 if parmConfObj.name != parameterName:
312 continue
314 continue
313
315
314 return parmConfObj
316 return parmConfObj
315
317
316 return None
318 return None
317
319
318 def getParameterObjfromValue(self, parameterValue):
320 def getParameterObjfromValue(self, parameterValue):
319
321
320 for parmConfObj in self.parmConfObjList:
322 for parmConfObj in self.parmConfObjList:
321
323
322 if parmConfObj.getValue() != parameterValue:
324 if parmConfObj.getValue() != parameterValue:
323 continue
325 continue
324
326
325 return parmConfObj.getValue()
327 return parmConfObj.getValue()
326
328
327 return None
329 return None
328
330
329 def getParameterValue(self, parameterName):
331 def getParameterValue(self, parameterName):
330
332
331 parameterObj = self.getParameterObj(parameterName)
333 parameterObj = self.getParameterObj(parameterName)
332
334
333 # if not parameterObj:
335 # if not parameterObj:
334 # return None
336 # return None
335
337
336 value = parameterObj.getValue()
338 value = parameterObj.getValue()
337
339
338 return value
340 return value
339
341
340
341 def getKwargs(self):
342 def getKwargs(self):
342
343
343 kwargs = {}
344 kwargs = {}
344
345
345 for parmConfObj in self.parmConfObjList:
346 for parmConfObj in self.parmConfObjList:
346 if self.name == 'run' and parmConfObj.name == 'datatype':
347 if self.name == 'run' and parmConfObj.name == 'datatype':
347 continue
348 continue
348
349
349 kwargs[parmConfObj.name] = parmConfObj.getValue()
350 kwargs[parmConfObj.name] = parmConfObj.getValue()
350
351
351 return kwargs
352 return kwargs
352
353
353 def setup(self, id, name, priority, type):
354 def setup(self, id, name, priority, type):
354
355
355 self.id = str(id)
356 self.id = str(id)
356 self.name = name
357 self.name = name
357 self.type = type
358 self.type = type
358 self.priority = priority
359 self.priority = priority
359
360
360 self.parmConfObjList = []
361 self.parmConfObjList = []
361
362
362 def removeParameters(self):
363 def removeParameters(self):
363
364
364 for obj in self.parmConfObjList:
365 for obj in self.parmConfObjList:
365 del obj
366 del obj
366
367
367 self.parmConfObjList = []
368 self.parmConfObjList = []
368
369
369 def addParameter(self, name, value, format='str'):
370 def addParameter(self, name, value, format='str'):
370
371
371 if value is None:
372 if value is None:
372 return None
373 return None
373 id = self.__getNewId()
374 id = self.__getNewId()
374
375
375 parmConfObj = ParameterConf()
376 parmConfObj = ParameterConf()
376 if not parmConfObj.setup(id, name, value, format):
377 if not parmConfObj.setup(id, name, value, format):
377 return None
378 return None
378
379
379 self.parmConfObjList.append(parmConfObj)
380 self.parmConfObjList.append(parmConfObj)
380
381
381 return parmConfObj
382 return parmConfObj
382
383
383 def changeParameter(self, name, value, format='str'):
384 def changeParameter(self, name, value, format='str'):
384
385
385 parmConfObj = self.getParameterObj(name)
386 parmConfObj = self.getParameterObj(name)
386 parmConfObj.update(name, value, format)
387 parmConfObj.update(name, value, format)
387
388
388 return parmConfObj
389 return parmConfObj
389
390
390 def makeXml(self, procUnitElement):
391 def makeXml(self, procUnitElement):
391
392
392 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
393 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
393 opElement.set('id', str(self.id))
394 opElement.set('id', str(self.id))
394 opElement.set('name', self.name)
395 opElement.set('name', self.name)
395 opElement.set('type', self.type)
396 opElement.set('type', self.type)
396 opElement.set('priority', str(self.priority))
397 opElement.set('priority', str(self.priority))
397
398
398 for parmConfObj in self.parmConfObjList:
399 for parmConfObj in self.parmConfObjList:
399 parmConfObj.makeXml(opElement)
400 parmConfObj.makeXml(opElement)
400
401
401 def readXml(self, opElement):
402 def readXml(self, opElement):
402
403
403 self.id = opElement.get('id')
404 self.id = opElement.get('id')
404 self.name = opElement.get('name')
405 self.name = opElement.get('name')
405 self.type = opElement.get('type')
406 self.type = opElement.get('type')
406 self.priority = opElement.get('priority')
407 self.priority = opElement.get('priority')
407
408
408 #Compatible with old signal chain version
409 # Compatible with old signal chain version
409 #Use of 'run' method instead 'init'
410 # Use of 'run' method instead 'init'
410 if self.type == 'self' and self.name == 'init':
411 if self.type == 'self' and self.name == 'init':
411 self.name = 'run'
412 self.name = 'run'
412
413
413 self.parmConfObjList = []
414 self.parmConfObjList = []
414
415
415 parmElementList = opElement.iter(ParameterConf().getElementName())
416 parmElementList = opElement.iter(ParameterConf().getElementName())
416
417
417 for parmElement in parmElementList:
418 for parmElement in parmElementList:
418 parmConfObj = ParameterConf()
419 parmConfObj = ParameterConf()
419 parmConfObj.readXml(parmElement)
420 parmConfObj.readXml(parmElement)
420
421
421 #Compatible with old signal chain version
422 # Compatible with old signal chain version
422 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
423 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
423 if self.type != 'self' and self.name == 'Plot':
424 if self.type != 'self' and self.name == 'Plot':
424 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
425 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
425 self.name = parmConfObj.value
426 self.name = parmConfObj.value
426 continue
427 continue
427
428
428 self.parmConfObjList.append(parmConfObj)
429 self.parmConfObjList.append(parmConfObj)
429
430
430 def printattr(self):
431 def printattr(self):
431
432
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
433 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
433 self.id,
434 self.id,
434 self.name,
435 self.name,
435 self.type,
436 self.type,
436 self.priority)
437 self.priority)
437
438
438 for parmConfObj in self.parmConfObjList:
439 for parmConfObj in self.parmConfObjList:
439 parmConfObj.printattr()
440 parmConfObj.printattr()
440
441
441 def createObject(self, plotter_queue=None):
442 def createObject(self, plotter_queue=None):
442
443
443
444 if self.type == 'self':
444 if self.type == 'self':
445 raise ValueError, 'This operation type cannot be created'
445 raise ValueError, 'This operation type cannot be created'
446
446
447 if self.type == 'plotter':
447 if self.type == 'plotter':
448 if not plotter_queue:
448 if not plotter_queue:
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
450
450
451 opObj = Plotter(self.name, plotter_queue)
451 opObj = Plotter(self.name, plotter_queue)
452
452
453 if self.type == 'external' or self.type == 'other':
453 if self.type == 'external' or self.type == 'other':
454
454
455 className = eval(self.name)
455 className = eval(self.name)
456 kwargs = self.getKwargs()
456 kwargs = self.getKwargs()
457
457
458 opObj = className(**kwargs)
458 opObj = className(**kwargs)
459
459
460 return opObj
460 return opObj
461
461
462
462
463 class ProcUnitConf():
463 class ProcUnitConf():
464
464
465 id = None
465 id = None
466 name = None
466 name = None
467 datatype = None
467 datatype = None
468 inputId = None
468 inputId = None
469 parentId = None
469 parentId = None
470
470
471 opConfObjList = []
471 opConfObjList = []
472
472
473 procUnitObj = None
473 procUnitObj = None
474 opObjList = []
474 opObjList = []
475
475
476 ELEMENTNAME = 'ProcUnit'
476 ELEMENTNAME = 'ProcUnit'
477
477
478 def __init__(self):
478 def __init__(self):
479
479
480 self.id = None
480 self.id = None
481 self.datatype = None
481 self.datatype = None
482 self.name = None
482 self.name = None
483 self.inputId = None
483 self.inputId = None
484
484
485 self.opConfObjList = []
485 self.opConfObjList = []
486
486
487 self.procUnitObj = None
487 self.procUnitObj = None
488 self.opObjDict = {}
488 self.opObjDict = {}
489
489
490 def __getPriority(self):
490 def __getPriority(self):
491
491
492 return len(self.opConfObjList)+1
492 return len(self.opConfObjList) + 1
493
493
494 def __getNewId(self):
494 def __getNewId(self):
495
495
496 return int(self.id)*10 + len(self.opConfObjList) + 1
496 return int(self.id) * 10 + len(self.opConfObjList) + 1
497
497
498 def getElementName(self):
498 def getElementName(self):
499
499
500 return self.ELEMENTNAME
500 return self.ELEMENTNAME
501
501
502 def getId(self):
502 def getId(self):
503
503
504 return self.id
504 return self.id
505
505
506 def updateId(self, new_id, parentId=parentId):
506 def updateId(self, new_id, parentId=parentId):
507
507
508
509 new_id = int(parentId)*10 + (int(self.id) % 10)
508 new_id = int(parentId) * 10 + (int(self.id) % 10)
510 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
509 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
511
510
512 #If this proc unit has not inputs
511 # If this proc unit has not inputs
513 if self.inputId == '0':
512 if self.inputId == '0':
514 new_inputId = 0
513 new_inputId = 0
515
514
516 n = 1
515 n = 1
517 for opConfObj in self.opConfObjList:
516 for opConfObj in self.opConfObjList:
518
517
519 idOp = str(int(new_id)*10 + n)
518 idOp = str(int(new_id) * 10 + n)
520 opConfObj.updateId(idOp)
519 opConfObj.updateId(idOp)
521
520
522 n += 1
521 n += 1
523
522
524 self.parentId = str(parentId)
523 self.parentId = str(parentId)
525 self.id = str(new_id)
524 self.id = str(new_id)
526 self.inputId = str(new_inputId)
525 self.inputId = str(new_inputId)
527
526
528
529 def getInputId(self):
527 def getInputId(self):
530
528
531 return self.inputId
529 return self.inputId
532
530
533 def getOperationObjList(self):
531 def getOperationObjList(self):
534
532
535 return self.opConfObjList
533 return self.opConfObjList
536
534
537 def getOperationObj(self, name=None):
535 def getOperationObj(self, name=None):
538
536
539 for opConfObj in self.opConfObjList:
537 for opConfObj in self.opConfObjList:
540
538
541 if opConfObj.name != name:
539 if opConfObj.name != name:
542 continue
540 continue
543
541
544 return opConfObj
542 return opConfObj
545
543
546 return None
544 return None
547
545
548 def getOpObjfromParamValue(self, value=None):
546 def getOpObjfromParamValue(self, value=None):
549
547
550 for opConfObj in self.opConfObjList:
548 for opConfObj in self.opConfObjList:
551 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
549 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
552 continue
550 continue
553 return opConfObj
551 return opConfObj
554 return None
552 return None
555
553
556 def getProcUnitObj(self):
554 def getProcUnitObj(self):
557
555
558 return self.procUnitObj
556 return self.procUnitObj
559
557
560 def setup(self, id, name, datatype, inputId, parentId=None):
558 def setup(self, id, name, datatype, inputId, parentId=None):
561
559
562 #Compatible with old signal chain version
560 # Compatible with old signal chain version
563 if datatype==None and name==None:
561 if datatype == None and name == None:
564 raise ValueError, 'datatype or name should be defined'
562 raise ValueError, 'datatype or name should be defined'
565
563
566 if name==None:
564 if name == None:
567 if 'Proc' in datatype:
565 if 'Proc' in datatype:
568 name = datatype
566 name = datatype
569 else:
567 else:
570 name = '%sProc' %(datatype)
568 name = '%sProc' % (datatype)
571
569
572 if datatype==None:
570 if datatype == None:
573 datatype = name.replace('Proc','')
571 datatype = name.replace('Proc', '')
574
572
575 self.id = str(id)
573 self.id = str(id)
576 self.name = name
574 self.name = name
577 self.datatype = datatype
575 self.datatype = datatype
578 self.inputId = inputId
576 self.inputId = inputId
579 self.parentId = parentId
577 self.parentId = parentId
580
578
581 self.opConfObjList = []
579 self.opConfObjList = []
582
580
583 self.addOperation(name='run', optype='self')
581 self.addOperation(name='run', optype='self')
584
582
585 def removeOperations(self):
583 def removeOperations(self):
586
584
587 for obj in self.opConfObjList:
585 for obj in self.opConfObjList:
588 del obj
586 del obj
589
587
590 self.opConfObjList = []
588 self.opConfObjList = []
591 self.addOperation(name='run')
589 self.addOperation(name='run')
592
590
593 def addParameter(self, **kwargs):
591 def addParameter(self, **kwargs):
594 '''
592 '''
595 Add parameters to 'run' operation
593 Add parameters to 'run' operation
596 '''
594 '''
597 opObj = self.opConfObjList[0]
595 opObj = self.opConfObjList[0]
598
596
599 opObj.addParameter(**kwargs)
597 opObj.addParameter(**kwargs)
600
598
601 return opObj
599 return opObj
602
600
603 def addOperation(self, name, optype='self'):
601 def addOperation(self, name, optype='self'):
604
602
605 id = self.__getNewId()
603 id = self.__getNewId()
606 priority = self.__getPriority()
604 priority = self.__getPriority()
607
605
608 opConfObj = OperationConf()
606 opConfObj = OperationConf()
609 opConfObj.setup(id, name=name, priority=priority, type=optype)
607 opConfObj.setup(id, name=name, priority=priority, type=optype)
610
608
611 self.opConfObjList.append(opConfObj)
609 self.opConfObjList.append(opConfObj)
612
610
613 return opConfObj
611 return opConfObj
614
612
615 def makeXml(self, projectElement):
613 def makeXml(self, projectElement):
616
614
617 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
615 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
618 procUnitElement.set('id', str(self.id))
616 procUnitElement.set('id', str(self.id))
619 procUnitElement.set('name', self.name)
617 procUnitElement.set('name', self.name)
620 procUnitElement.set('datatype', self.datatype)
618 procUnitElement.set('datatype', self.datatype)
621 procUnitElement.set('inputId', str(self.inputId))
619 procUnitElement.set('inputId', str(self.inputId))
622
620
623 for opConfObj in self.opConfObjList:
621 for opConfObj in self.opConfObjList:
624 opConfObj.makeXml(procUnitElement)
622 opConfObj.makeXml(procUnitElement)
625
623
626 def readXml(self, upElement):
624 def readXml(self, upElement):
627
625
628 self.id = upElement.get('id')
626 self.id = upElement.get('id')
629 self.name = upElement.get('name')
627 self.name = upElement.get('name')
630 self.datatype = upElement.get('datatype')
628 self.datatype = upElement.get('datatype')
631 self.inputId = upElement.get('inputId')
629 self.inputId = upElement.get('inputId')
632
630
633 if self.ELEMENTNAME == 'ReadUnit':
631 if self.ELEMENTNAME == 'ReadUnit':
634 self.datatype = self.datatype.replace('Reader', '')
632 self.datatype = self.datatype.replace('Reader', '')
635
633
636 if self.ELEMENTNAME == 'ProcUnit':
634 if self.ELEMENTNAME == 'ProcUnit':
637 self.datatype = self.datatype.replace('Proc', '')
635 self.datatype = self.datatype.replace('Proc', '')
638
636
639 if self.inputId == 'None':
637 if self.inputId == 'None':
640 self.inputId = '0'
638 self.inputId = '0'
641
639
642 self.opConfObjList = []
640 self.opConfObjList = []
643
641
644 opElementList = upElement.iter(OperationConf().getElementName())
642 opElementList = upElement.iter(OperationConf().getElementName())
645
643
646 for opElement in opElementList:
644 for opElement in opElementList:
647 opConfObj = OperationConf()
645 opConfObj = OperationConf()
648 opConfObj.readXml(opElement)
646 opConfObj.readXml(opElement)
649 self.opConfObjList.append(opConfObj)
647 self.opConfObjList.append(opConfObj)
650
648
651 def printattr(self):
649 def printattr(self):
652
650
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
651 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
654 self.id,
652 self.id,
655 self.name,
653 self.name,
656 self.datatype,
654 self.datatype,
657 self.inputId)
655 self.inputId)
658
656
659 for opConfObj in self.opConfObjList:
657 for opConfObj in self.opConfObjList:
660 opConfObj.printattr()
658 opConfObj.printattr()
661
659
662
663 def getKwargs(self):
660 def getKwargs(self):
664
661
665 opObj = self.opConfObjList[0]
662 opObj = self.opConfObjList[0]
666 kwargs = opObj.getKwargs()
663 kwargs = opObj.getKwargs()
667
664
668 return kwargs
665 return kwargs
669
666
670 def createObjects(self, plotter_queue=None):
667 def createObjects(self, plotter_queue=None):
671
668
672 className = eval(self.name)
669 className = eval(self.name)
673 kwargs = self.getKwargs()
670 kwargs = self.getKwargs()
674 procUnitObj = className(**kwargs)
671 procUnitObj = className(**kwargs)
675
672
676 for opConfObj in self.opConfObjList:
673 for opConfObj in self.opConfObjList:
677
674
678 if opConfObj.type=='self' and self.name=='run':
675 if opConfObj.type == 'self' and self.name == 'run':
679 continue
676 continue
680 elif opConfObj.type=='self':
677 elif opConfObj.type == 'self':
681 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
678 procUnitObj.addOperationKwargs(
679 opConfObj.id, **opConfObj.getKwargs())
682 continue
680 continue
683
681
684 opObj = opConfObj.createObject(plotter_queue)
682 opObj = opConfObj.createObject(plotter_queue)
685
683
686 self.opObjDict[opConfObj.id] = opObj
684 self.opObjDict[opConfObj.id] = opObj
687
685
688 procUnitObj.addOperation(opObj, opConfObj.id)
686 procUnitObj.addOperation(opObj, opConfObj.id)
689
687
690 self.procUnitObj = procUnitObj
688 self.procUnitObj = procUnitObj
691
689
692 return procUnitObj
690 return procUnitObj
693
691
694 def run(self):
692 def run(self):
695
693
696 is_ok = False
694 is_ok = False
697
695
698 for opConfObj in self.opConfObjList:
696 for opConfObj in self.opConfObjList:
699
697
700 kwargs = {}
698 kwargs = {}
701 for parmConfObj in opConfObj.getParameterObjList():
699 for parmConfObj in opConfObj.getParameterObjList():
702 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
700 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
703 continue
701 continue
704
702
705 kwargs[parmConfObj.name] = parmConfObj.getValue()
703 kwargs[parmConfObj.name] = parmConfObj.getValue()
706
704
707 sts = self.procUnitObj.call(opType = opConfObj.type,
705 sts = self.procUnitObj.call(opType=opConfObj.type,
708 opName = opConfObj.name,
706 opName=opConfObj.name,
709 opId = opConfObj.id)
707 opId=opConfObj.id)
710
708
711 is_ok = is_ok or sts
709 is_ok = is_ok or sts
712
710
713 return is_ok
711 return is_ok
714
712
715 def close(self):
713 def close(self):
716
714
717 for opConfObj in self.opConfObjList:
715 for opConfObj in self.opConfObjList:
718 if opConfObj.type == 'self':
716 if opConfObj.type == 'self':
719 continue
717 continue
720
718
721 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
719 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
722 opObj.close()
720 opObj.close()
723
721
724 self.procUnitObj.close()
722 self.procUnitObj.close()
725
723
726 return
724 return
727
725
726
728 class ReadUnitConf(ProcUnitConf):
727 class ReadUnitConf(ProcUnitConf):
729
728
730 path = None
729 path = None
731 startDate = None
730 startDate = None
732 endDate = None
731 endDate = None
733 startTime = None
732 startTime = None
734 endTime = None
733 endTime = None
735
734
736 ELEMENTNAME = 'ReadUnit'
735 ELEMENTNAME = 'ReadUnit'
737
736
738 def __init__(self):
737 def __init__(self):
739
738
740 self.id = None
739 self.id = None
741 self.datatype = None
740 self.datatype = None
742 self.name = None
741 self.name = None
743 self.inputId = None
742 self.inputId = None
744
743
745 self.parentId = None
744 self.parentId = None
746
745
747 self.opConfObjList = []
746 self.opConfObjList = []
748 self.opObjList = []
747 self.opObjList = []
749
748
750 def getElementName(self):
749 def getElementName(self):
751
750
752 return self.ELEMENTNAME
751 return self.ELEMENTNAME
753
752
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
753 def setup(self, id, name, datatype, path='', startDate='', endDate='',
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
754 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
755
757 #Compatible with old signal chain version
756 # Compatible with old signal chain version
758 if datatype==None and name==None:
757 if datatype == None and name == None:
759 raise ValueError, 'datatype or name should be defined'
758 raise ValueError, 'datatype or name should be defined'
760
759
761 if name==None:
760 if name == None:
762 if 'Reader' in datatype:
761 if 'Reader' in datatype:
763 name = datatype
762 name = datatype
764 else:
763 else:
765 name = '%sReader' %(datatype)
764 name = '%sReader' % (datatype)
766 if datatype==None:
765 if datatype == None:
767 datatype = name.replace('Reader','')
766 datatype = name.replace('Reader', '')
768
767
769 self.id = id
768 self.id = id
770 self.name = name
769 self.name = name
771 self.datatype = datatype
770 self.datatype = datatype
772 if path != '':
771 if path != '':
773 self.path = os.path.abspath(path)
772 self.path = os.path.abspath(path)
774 self.startDate = startDate
773 self.startDate = startDate
775 self.endDate = endDate
774 self.endDate = endDate
776 self.startTime = startTime
775 self.startTime = startTime
777 self.endTime = endTime
776 self.endTime = endTime
778 self.inputId = '0'
777 self.inputId = '0'
779 self.parentId = parentId
778 self.parentId = parentId
780 self.server = server
779 self.server = server
781 self.addRunOperation(**kwargs)
780 self.addRunOperation(**kwargs)
782
781
783 def update(self, **kwargs):
782 def update(self, **kwargs):
784
783
785 if 'datatype' in kwargs:
784 if 'datatype' in kwargs:
786 datatype = kwargs.pop('datatype')
785 datatype = kwargs.pop('datatype')
787 if 'Reader' in datatype:
786 if 'Reader' in datatype:
788 self.name = datatype
787 self.name = datatype
789 else:
788 else:
790 self.name = '%sReader' %(datatype)
789 self.name = '%sReader' % (datatype)
791 self.datatype = self.name.replace('Reader', '')
790 self.datatype = self.name.replace('Reader', '')
792
791
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
792 attrs = ('path', 'startDate', 'endDate',
793 'startTime', 'endTime', 'parentId')
794
794
795 for attr in attrs:
795 for attr in attrs:
796 if attr in kwargs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
797 setattr(self, attr, kwargs.pop(attr))
798
798
799 self.inputId = '0'
799 self.inputId = '0'
800 self.updateRunOperation(**kwargs)
800 self.updateRunOperation(**kwargs)
801
801
802 def removeOperations(self):
802 def removeOperations(self):
803
803
804 for obj in self.opConfObjList:
804 for obj in self.opConfObjList:
805 del obj
805 del obj
806
806
807 self.opConfObjList = []
807 self.opConfObjList = []
808
808
809 def addRunOperation(self, **kwargs):
809 def addRunOperation(self, **kwargs):
810
810
811 opObj = self.addOperation(name = 'run', optype = 'self')
811 opObj = self.addOperation(name='run', optype='self')
812
812
813 if self.server is None:
813 if self.server is None:
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
814 opObj.addParameter(
815 name='datatype', value=self.datatype, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
817 opObj.addParameter(
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
818 name='startDate', value=self.startDate, format='date')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
819 opObj.addParameter(
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
820 name='endDate', value=self.endDate, format='date')
821 opObj.addParameter(
822 name='startTime', value=self.startTime, format='time')
823 opObj.addParameter(
824 name='endTime', value=self.endTime, format='time')
820
825
821 for key, value in kwargs.items():
826 for key, value in kwargs.items():
822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
827 opObj.addParameter(name=key, value=value,
828 format=type(value).__name__)
823 else:
829 else:
824 opObj.addParameter(name='server' , value=self.server, format='str')
830 opObj.addParameter(name='server', value=self.server, format='str')
825
831
826
827 return opObj
832 return opObj
828
833
829 def updateRunOperation(self, **kwargs):
834 def updateRunOperation(self, **kwargs):
830
835
831 opObj = self.getOperationObj(name='run')
836 opObj = self.getOperationObj(name='run')
832 opObj.removeParameters()
837 opObj.removeParameters()
833
838
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
839 opObj.addParameter(name='datatype', value=self.datatype, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
840 opObj.addParameter(name='path', value=self.path, format='str')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
841 opObj.addParameter(
842 name='startDate', value=self.startDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
843 opObj.addParameter(name='endDate', value=self.endDate, format='date')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
844 opObj.addParameter(
845 name='startTime', value=self.startTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
846 opObj.addParameter(name='endTime', value=self.endTime, format='time')
840
847
841 for key, value in kwargs.items():
848 for key, value in kwargs.items():
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
849 opObj.addParameter(name=key, value=value,
850 format=type(value).__name__)
843
851
844 return opObj
852 return opObj
845
853
846 def readXml(self, upElement):
854 def readXml(self, upElement):
847
855
848 self.id = upElement.get('id')
856 self.id = upElement.get('id')
849 self.name = upElement.get('name')
857 self.name = upElement.get('name')
850 self.datatype = upElement.get('datatype')
858 self.datatype = upElement.get('datatype')
851 self.inputId = upElement.get('inputId')
859 self.inputId = upElement.get('inputId')
852
860
853 if self.ELEMENTNAME == 'ReadUnit':
861 if self.ELEMENTNAME == 'ReadUnit':
854 self.datatype = self.datatype.replace('Reader', '')
862 self.datatype = self.datatype.replace('Reader', '')
855
863
856 if self.inputId == 'None':
864 if self.inputId == 'None':
857 self.inputId = '0'
865 self.inputId = '0'
858
866
859 self.opConfObjList = []
867 self.opConfObjList = []
860
868
861 opElementList = upElement.iter(OperationConf().getElementName())
869 opElementList = upElement.iter(OperationConf().getElementName())
862
870
863 for opElement in opElementList:
871 for opElement in opElementList:
864 opConfObj = OperationConf()
872 opConfObj = OperationConf()
865 opConfObj.readXml(opElement)
873 opConfObj.readXml(opElement)
866 self.opConfObjList.append(opConfObj)
874 self.opConfObjList.append(opConfObj)
867
875
868 if opConfObj.name == 'run':
876 if opConfObj.name == 'run':
869 self.path = opConfObj.getParameterValue('path')
877 self.path = opConfObj.getParameterValue('path')
870 self.startDate = opConfObj.getParameterValue('startDate')
878 self.startDate = opConfObj.getParameterValue('startDate')
871 self.endDate = opConfObj.getParameterValue('endDate')
879 self.endDate = opConfObj.getParameterValue('endDate')
872 self.startTime = opConfObj.getParameterValue('startTime')
880 self.startTime = opConfObj.getParameterValue('startTime')
873 self.endTime = opConfObj.getParameterValue('endTime')
881 self.endTime = opConfObj.getParameterValue('endTime')
874
882
883
875 class Project(Process):
884 class Project(Process):
876
885
877 id = None
886 id = None
878 # name = None
887 # name = None
879 description = None
888 description = None
880 filename = None
889 filename = None
881
890
882 procUnitConfObjDict = None
891 procUnitConfObjDict = None
883
892
884 ELEMENTNAME = 'Project'
893 ELEMENTNAME = 'Project'
885
894
886 plotterQueue = None
895 plotterQueue = None
887
896
888 def __init__(self, plotter_queue=None):
897 def __init__(self, plotter_queue=None):
889
898
890 Process.__init__(self)
899 Process.__init__(self)
891 self.id = None
900 self.id = None
892 # self.name = None
901 # self.name = None
893 self.description = None
902 self.description = None
894
903
895 self.plotterQueue = plotter_queue
904 self.plotterQueue = plotter_queue
896
905
897 self.procUnitConfObjDict = {}
906 self.procUnitConfObjDict = {}
898
907
899 def __getNewId(self):
908 def __getNewId(self):
900
909
901 idList = self.procUnitConfObjDict.keys()
910 idList = self.procUnitConfObjDict.keys()
902
911
903 id = int(self.id)*10
912 id = int(self.id) * 10
904
913
905 while True:
914 while True:
906 id += 1
915 id += 1
907
916
908 if str(id) in idList:
917 if str(id) in idList:
909 continue
918 continue
910
919
911 break
920 break
912
921
913 return str(id)
922 return str(id)
914
923
915 def getElementName(self):
924 def getElementName(self):
916
925
917 return self.ELEMENTNAME
926 return self.ELEMENTNAME
918
927
919 def getId(self):
928 def getId(self):
920
929
921 return self.id
930 return self.id
922
931
923 def updateId(self, new_id):
932 def updateId(self, new_id):
924
933
925 self.id = str(new_id)
934 self.id = str(new_id)
926
935
927 keyList = self.procUnitConfObjDict.keys()
936 keyList = self.procUnitConfObjDict.keys()
928 keyList.sort()
937 keyList.sort()
929
938
930 n = 1
939 n = 1
931 newProcUnitConfObjDict = {}
940 newProcUnitConfObjDict = {}
932
941
933 for procKey in keyList:
942 for procKey in keyList:
934
943
935 procUnitConfObj = self.procUnitConfObjDict[procKey]
944 procUnitConfObj = self.procUnitConfObjDict[procKey]
936 idProcUnit = str(int(self.id)*10 + n)
945 idProcUnit = str(int(self.id) * 10 + n)
937 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
946 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
938
947
939 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
948 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
940 n += 1
949 n += 1
941
950
942 self.procUnitConfObjDict = newProcUnitConfObjDict
951 self.procUnitConfObjDict = newProcUnitConfObjDict
943
952
944 def setup(self, id, name='', description=''):
953 def setup(self, id, name='', description=''):
945
954
946 print
955 print
947 print '*'*60
956 print '*' * 60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
957 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
958 print '*' * 60
950 print
959 print
951 self.id = str(id)
960 self.id = str(id)
952 self.description = description
961 self.description = description
953
962
954 def update(self, name, description):
963 def update(self, name, description):
955
964
956 self.description = description
965 self.description = description
957
966
958 def clone(self):
967 def clone(self):
959
968
960 p = Project()
969 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
970 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
971 return p
963
972
964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
973 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
974
966 if id is None:
975 if id is None:
967 idReadUnit = self.__getNewId()
976 idReadUnit = self.__getNewId()
968 else:
977 else:
969 idReadUnit = str(id)
978 idReadUnit = str(id)
970
979
971 readUnitConfObj = ReadUnitConf()
980 readUnitConfObj = ReadUnitConf()
972 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
981 readUnitConfObj.setup(idReadUnit, name, datatype,
982 parentId=self.id, **kwargs)
973
983
974 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
984 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
975
985
976 return readUnitConfObj
986 return readUnitConfObj
977
987
978 def addProcUnit(self, inputId='0', datatype=None, name=None):
988 def addProcUnit(self, inputId='0', datatype=None, name=None):
979
989
980 idProcUnit = self.__getNewId()
990 idProcUnit = self.__getNewId()
981
991
982 procUnitConfObj = ProcUnitConf()
992 procUnitConfObj = ProcUnitConf()
983 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
993 procUnitConfObj.setup(idProcUnit, name, datatype,
994 inputId, parentId=self.id)
984
995
985 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
996 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
986
997
987 return procUnitConfObj
998 return procUnitConfObj
988
999
989 def removeProcUnit(self, id):
1000 def removeProcUnit(self, id):
990
1001
991 if id in self.procUnitConfObjDict.keys():
1002 if id in self.procUnitConfObjDict.keys():
992 self.procUnitConfObjDict.pop(id)
1003 self.procUnitConfObjDict.pop(id)
993
1004
994 def getReadUnitId(self):
1005 def getReadUnitId(self):
995
1006
996 readUnitConfObj = self.getReadUnitObj()
1007 readUnitConfObj = self.getReadUnitObj()
997
1008
998 return readUnitConfObj.id
1009 return readUnitConfObj.id
999
1010
1000 def getReadUnitObj(self):
1011 def getReadUnitObj(self):
1001
1012
1002 for obj in self.procUnitConfObjDict.values():
1013 for obj in self.procUnitConfObjDict.values():
1003 if obj.getElementName() == 'ReadUnit':
1014 if obj.getElementName() == 'ReadUnit':
1004 return obj
1015 return obj
1005
1016
1006 return None
1017 return None
1007
1018
1008 def getProcUnitObj(self, id=None, name=None):
1019 def getProcUnitObj(self, id=None, name=None):
1009
1020
1010 if id != None:
1021 if id != None:
1011 return self.procUnitConfObjDict[id]
1022 return self.procUnitConfObjDict[id]
1012
1023
1013 if name != None:
1024 if name != None:
1014 return self.getProcUnitObjByName(name)
1025 return self.getProcUnitObjByName(name)
1015
1026
1016 return None
1027 return None
1017
1028
1018 def getProcUnitObjByName(self, name):
1029 def getProcUnitObjByName(self, name):
1019
1030
1020 for obj in self.procUnitConfObjDict.values():
1031 for obj in self.procUnitConfObjDict.values():
1021 if obj.name == name:
1032 if obj.name == name:
1022 return obj
1033 return obj
1023
1034
1024 return None
1035 return None
1025
1036
1026 def procUnitItems(self):
1037 def procUnitItems(self):
1027
1038
1028 return self.procUnitConfObjDict.items()
1039 return self.procUnitConfObjDict.items()
1029
1040
1030 def makeXml(self):
1041 def makeXml(self):
1031
1042
1032 projectElement = Element('Project')
1043 projectElement = Element('Project')
1033 projectElement.set('id', str(self.id))
1044 projectElement.set('id', str(self.id))
1034 projectElement.set('name', self.name)
1045 projectElement.set('name', self.name)
1035 projectElement.set('description', self.description)
1046 projectElement.set('description', self.description)
1036
1047
1037 for procUnitConfObj in self.procUnitConfObjDict.values():
1048 for procUnitConfObj in self.procUnitConfObjDict.values():
1038 procUnitConfObj.makeXml(projectElement)
1049 procUnitConfObj.makeXml(projectElement)
1039
1050
1040 self.projectElement = projectElement
1051 self.projectElement = projectElement
1041
1052
1042 def writeXml(self, filename=None):
1053 def writeXml(self, filename=None):
1043
1054
1044 if filename == None:
1055 if filename == None:
1045 if self.filename:
1056 if self.filename:
1046 filename = self.filename
1057 filename = self.filename
1047 else:
1058 else:
1048 filename = 'schain.xml'
1059 filename = 'schain.xml'
1049
1060
1050 if not filename:
1061 if not filename:
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1062 print 'filename has not been defined. Use setFilename(filename) for do it.'
1052 return 0
1063 return 0
1053
1064
1054 abs_file = os.path.abspath(filename)
1065 abs_file = os.path.abspath(filename)
1055
1066
1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1067 if not os.access(os.path.dirname(abs_file), os.W_OK):
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1068 print 'No write permission on %s' % os.path.dirname(abs_file)
1058 return 0
1069 return 0
1059
1070
1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1071 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1072 print 'File %s already exists and it could not be overwriten' % abs_file
1062 return 0
1073 return 0
1063
1074
1064 self.makeXml()
1075 self.makeXml()
1065
1076
1066 ElementTree(self.projectElement).write(abs_file, method='xml')
1077 ElementTree(self.projectElement).write(abs_file, method='xml')
1067
1078
1068 self.filename = abs_file
1079 self.filename = abs_file
1069
1080
1070 return 1
1081 return 1
1071
1082
1072 def readXml(self, filename = None):
1083 def readXml(self, filename=None):
1073
1084
1074 if not filename:
1085 if not filename:
1075 print 'filename is not defined'
1086 print 'filename is not defined'
1076 return 0
1087 return 0
1077
1088
1078 abs_file = os.path.abspath(filename)
1089 abs_file = os.path.abspath(filename)
1079
1090
1080 if not os.path.isfile(abs_file):
1091 if not os.path.isfile(abs_file):
1081 print '%s file does not exist' %abs_file
1092 print '%s file does not exist' % abs_file
1082 return 0
1093 return 0
1083
1094
1084 self.projectElement = None
1095 self.projectElement = None
1085 self.procUnitConfObjDict = {}
1096 self.procUnitConfObjDict = {}
1086
1097
1087 try:
1098 try:
1088 self.projectElement = ElementTree().parse(abs_file)
1099 self.projectElement = ElementTree().parse(abs_file)
1089 except:
1100 except:
1090 print 'Error reading %s, verify file format' %filename
1101 print 'Error reading %s, verify file format' % filename
1091 return 0
1102 return 0
1092
1103
1093 self.project = self.projectElement.tag
1104 self.project = self.projectElement.tag
1094
1105
1095 self.id = self.projectElement.get('id')
1106 self.id = self.projectElement.get('id')
1096 self.name = self.projectElement.get('name')
1107 self.name = self.projectElement.get('name')
1097 self.description = self.projectElement.get('description')
1108 self.description = self.projectElement.get('description')
1098
1109
1099 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1110 readUnitElementList = self.projectElement.iter(
1111 ReadUnitConf().getElementName())
1100
1112
1101 for readUnitElement in readUnitElementList:
1113 for readUnitElement in readUnitElementList:
1102 readUnitConfObj = ReadUnitConf()
1114 readUnitConfObj = ReadUnitConf()
1103 readUnitConfObj.readXml(readUnitElement)
1115 readUnitConfObj.readXml(readUnitElement)
1104
1116
1105 if readUnitConfObj.parentId == None:
1117 if readUnitConfObj.parentId == None:
1106 readUnitConfObj.parentId = self.id
1118 readUnitConfObj.parentId = self.id
1107
1119
1108 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1120 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1109
1121
1110 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1122 procUnitElementList = self.projectElement.iter(
1123 ProcUnitConf().getElementName())
1111
1124
1112 for procUnitElement in procUnitElementList:
1125 for procUnitElement in procUnitElementList:
1113 procUnitConfObj = ProcUnitConf()
1126 procUnitConfObj = ProcUnitConf()
1114 procUnitConfObj.readXml(procUnitElement)
1127 procUnitConfObj.readXml(procUnitElement)
1115
1128
1116 if procUnitConfObj.parentId == None:
1129 if procUnitConfObj.parentId == None:
1117 procUnitConfObj.parentId = self.id
1130 procUnitConfObj.parentId = self.id
1118
1131
1119 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1132 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1120
1133
1121 self.filename = abs_file
1134 self.filename = abs_file
1122
1135
1123 return 1
1136 return 1
1124
1137
1125 def printattr(self):
1138 def printattr(self):
1126
1139
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1140 print 'Project[%s]: name = %s, description = %s' % (self.id,
1128 self.name,
1141 self.name,
1129 self.description)
1142 self.description)
1130
1143
1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1144 for procUnitConfObj in self.procUnitConfObjDict.values():
1132 procUnitConfObj.printattr()
1145 procUnitConfObj.printattr()
1133
1146
1134 def createObjects(self):
1147 def createObjects(self):
1135
1148
1136 for procUnitConfObj in self.procUnitConfObjDict.values():
1149 for procUnitConfObj in self.procUnitConfObjDict.values():
1137 procUnitConfObj.createObjects(self.plotterQueue)
1150 procUnitConfObj.createObjects(self.plotterQueue)
1138
1151
1139 def __connect(self, objIN, thisObj):
1152 def __connect(self, objIN, thisObj):
1140
1153
1141 thisObj.setInput(objIN.getOutputObj())
1154 thisObj.setInput(objIN.getOutputObj())
1142
1155
1143 def connectObjects(self):
1156 def connectObjects(self):
1144
1157
1145 for thisPUConfObj in self.procUnitConfObjDict.values():
1158 for thisPUConfObj in self.procUnitConfObjDict.values():
1146
1159
1147 inputId = thisPUConfObj.getInputId()
1160 inputId = thisPUConfObj.getInputId()
1148
1161
1149 if int(inputId) == 0:
1162 if int(inputId) == 0:
1150 continue
1163 continue
1151
1164
1152 #Get input object
1165 # Get input object
1153 puConfINObj = self.procUnitConfObjDict[inputId]
1166 puConfINObj = self.procUnitConfObjDict[inputId]
1154 puObjIN = puConfINObj.getProcUnitObj()
1167 puObjIN = puConfINObj.getProcUnitObj()
1155
1168
1156 #Get current object
1169 # Get current object
1157 thisPUObj = thisPUConfObj.getProcUnitObj()
1170 thisPUObj = thisPUConfObj.getProcUnitObj()
1158
1171
1159 self.__connect(puObjIN, thisPUObj)
1172 self.__connect(puObjIN, thisPUObj)
1160
1173
1161 def __handleError(self, procUnitConfObj, send_email=False):
1174 def __handleError(self, procUnitConfObj, send_email=False):
1162
1175
1163 import socket
1176 import socket
1164
1177
1165 err = traceback.format_exception(sys.exc_info()[0],
1178 err = traceback.format_exception(sys.exc_info()[0],
1166 sys.exc_info()[1],
1179 sys.exc_info()[1],
1167 sys.exc_info()[2])
1180 sys.exc_info()[2])
1168
1181
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1182 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1170 print '***** %s' %err[-1]
1183 print '***** %s' % err[-1]
1171
1184
1172 message = ''.join(err)
1185 message = ''.join(err)
1173
1186
1174 sys.stderr.write(message)
1187 sys.stderr.write(message)
1175
1188
1176 if not send_email:
1189 if not send_email:
1177 return
1190 return
1178
1191
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1192 subject = 'SChain v%s: Error running %s\n' % (
1193 schainpy.__version__, procUnitConfObj.name)
1180
1194
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1195 subtitle = '%s: %s\n' % (
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1196 procUnitConfObj.getElementName(), procUnitConfObj.name)
1197 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1198 socket.gethostname())
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1199 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1184 subtitle += 'Configuration file: %s\n' %self.filename
1200 subtitle += 'Configuration file: %s\n' % self.filename
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1201 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1186
1202
1187 readUnitConfObj = self.getReadUnitObj()
1203 readUnitConfObj = self.getReadUnitObj()
1188 if readUnitConfObj:
1204 if readUnitConfObj:
1189 subtitle += '\nInput parameters:\n'
1205 subtitle += '\nInput parameters:\n'
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1206 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1207 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1208 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1209 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1210 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1211 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1196
1212
1197 adminObj = schainpy.admin.SchainNotify()
1213 adminObj = schainpy.admin.SchainNotify()
1198 adminObj.sendAlert(message=message,
1214 adminObj.sendAlert(message=message,
1199 subject=subject,
1215 subject=subject,
1200 subtitle=subtitle,
1216 subtitle=subtitle,
1201 filename=self.filename)
1217 filename=self.filename)
1202
1218
1203 def isPaused(self):
1219 def isPaused(self):
1204 return 0
1220 return 0
1205
1221
1206 def isStopped(self):
1222 def isStopped(self):
1207 return 0
1223 return 0
1208
1224
1209 def runController(self):
1225 def runController(self):
1210 '''
1226 '''
1211 returns 0 when this process has been stopped, 1 otherwise
1227 returns 0 when this process has been stopped, 1 otherwise
1212 '''
1228 '''
1213
1229
1214 if self.isPaused():
1230 if self.isPaused():
1215 print 'Process suspended'
1231 print 'Process suspended'
1216
1232
1217 while True:
1233 while True:
1218 time.sleep(0.1)
1234 time.sleep(0.1)
1219
1235
1220 if not self.isPaused():
1236 if not self.isPaused():
1221 break
1237 break
1222
1238
1223 if self.isStopped():
1239 if self.isStopped():
1224 break
1240 break
1225
1241
1226 print 'Process reinitialized'
1242 print 'Process reinitialized'
1227
1243
1228 if self.isStopped():
1244 if self.isStopped():
1229 print 'Process stopped'
1245 print 'Process stopped'
1230 return 0
1246 return 0
1231
1247
1232 return 1
1248 return 1
1233
1249
1234 def setFilename(self, filename):
1250 def setFilename(self, filename):
1235
1251
1236 self.filename = filename
1252 self.filename = filename
1237
1253
1238 def setPlotterQueue(self, plotter_queue):
1254 def setPlotterQueue(self, plotter_queue):
1239
1255
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1256 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1241
1257
1242 def getPlotterQueue(self):
1258 def getPlotterQueue(self):
1243
1259
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1260 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1245
1261
1246 def useExternalPlotter(self):
1262 def useExternalPlotter(self):
1247
1263
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1249
1265
1250 def run(self):
1266 def run(self):
1251
1267
1252 log.success('Starting {}'.format(self.name))
1268 log.success('Starting {}'.format(self.name))
1253
1269
1254 self.createObjects()
1270 self.createObjects()
1255 self.connectObjects()
1271 self.connectObjects()
1256
1272
1257 keyList = self.procUnitConfObjDict.keys()
1273 keyList = self.procUnitConfObjDict.keys()
1258 keyList.sort()
1274 keyList.sort()
1259
1275
1260 while(True):
1276 while(True):
1261
1277
1262 is_ok = False
1278 is_ok = False
1263
1279
1264 for procKey in keyList:
1280 for procKey in keyList:
1265
1281
1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1282 procUnitConfObj = self.procUnitConfObjDict[procKey]
1267
1283
1268 try:
1284 try:
1269 sts = procUnitConfObj.run()
1285 sts = procUnitConfObj.run()
1270 is_ok = is_ok or sts
1286 is_ok = is_ok or sts
1271 except KeyboardInterrupt:
1287 except KeyboardInterrupt:
1272 is_ok = False
1288 is_ok = False
1273 break
1289 break
1274 except ValueError, e:
1290 except ValueError, e:
1275 time.sleep(0.5)
1291 time.sleep(0.5)
1276 self.__handleError(procUnitConfObj, send_email=True)
1292 self.__handleError(procUnitConfObj, send_email=True)
1277 is_ok = False
1293 is_ok = False
1278 break
1294 break
1279 except:
1295 except:
1280 time.sleep(0.5)
1296 time.sleep(0.5)
1281 self.__handleError(procUnitConfObj)
1297 self.__handleError(procUnitConfObj)
1282 is_ok = False
1298 is_ok = False
1283 break
1299 break
1284
1300
1285 #If every process unit finished so end process
1301 # If every process unit finished so end process
1286 if not(is_ok):
1302 if not(is_ok):
1287 break
1303 break
1288
1304
1289 if not self.runController():
1305 if not self.runController():
1290 break
1306 break
1291
1307
1292 #Closing every process
1308 # Closing every process
1293 for procKey in keyList:
1309 for procKey in keyList:
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1310 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 procUnitConfObj.close()
1311 procUnitConfObj.close()
1296
1312
1297 log.success('{} finished'.format(self.name))
1313 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
@@ -1,736 +1,762
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import numpy
7 import numpy
8
8
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13 import zmq
13 import zmq
14 import tempfile
14 import tempfile
15 from StringIO import StringIO
15 from StringIO import StringIO
16 # from _sha import blocksize
16 # from _sha import blocksize
17
17
18
18 class VoltageReader(JRODataReader, ProcessingUnit):
19 class VoltageReader(JRODataReader, ProcessingUnit):
19 """
20 """
20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
22 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
22 perfiles*alturas*canales) son almacenados en la variable "buffer".
23 perfiles*alturas*canales) son almacenados en la variable "buffer".
23
24
24 perfiles * alturas * canales
25 perfiles * alturas * canales
25
26
26 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
27 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
27 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
28 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
28 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
29 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
29 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
30 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
30
31
31 Example:
32 Example:
32
33
33 dpath = "/home/myuser/data"
34 dpath = "/home/myuser/data"
34
35
35 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
36 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
36
37
37 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
38 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
38
39
39 readerObj = VoltageReader()
40 readerObj = VoltageReader()
40
41
41 readerObj.setup(dpath, startTime, endTime)
42 readerObj.setup(dpath, startTime, endTime)
42
43
43 while(True):
44 while(True):
44
45
45 #to get one profile
46 #to get one profile
46 profile = readerObj.getData()
47 profile = readerObj.getData()
47
48
48 #print the profile
49 #print the profile
49 print profile
50 print profile
50
51
51 #If you want to see all datablock
52 #If you want to see all datablock
52 print readerObj.datablock
53 print readerObj.datablock
53
54
54 if readerObj.flagNoMoreFiles:
55 if readerObj.flagNoMoreFiles:
55 break
56 break
56
57
57 """
58 """
58
59
59 ext = ".r"
60 ext = ".r"
60
61
61 optchar = "D"
62 optchar = "D"
62 dataOut = None
63 dataOut = None
63
64
64 def __init__(self, **kwargs):
65 def __init__(self, **kwargs):
65 """
66 """
66 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
67
68
68 Input:
69 Input:
69 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
70 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
70 almacenar un perfil de datos cada vez que se haga un requerimiento
71 almacenar un perfil de datos cada vez que se haga un requerimiento
71 (getData). El perfil sera obtenido a partir del buffer de datos,
72 (getData). El perfil sera obtenido a partir del buffer de datos,
72 si el buffer esta vacio se hara un nuevo proceso de lectura de un
73 si el buffer esta vacio se hara un nuevo proceso de lectura de un
73 bloque de datos.
74 bloque de datos.
74 Si este parametro no es pasado se creara uno internamente.
75 Si este parametro no es pasado se creara uno internamente.
75
76
76 Variables afectadas:
77 Variables afectadas:
77 self.dataOut
78 self.dataOut
78
79
79 Return:
80 Return:
80 None
81 None
81 """
82 """
82
83
83 ProcessingUnit.__init__(self, **kwargs)
84 ProcessingUnit.__init__(self, **kwargs)
84
85
85 self.isConfig = False
86 self.isConfig = False
86
87
87 self.datablock = None
88 self.datablock = None
88
89
89 self.utc = 0
90 self.utc = 0
90
91
91 self.ext = ".r"
92 self.ext = ".r"
92
93
93 self.optchar = "D"
94 self.optchar = "D"
94
95
95 self.basicHeaderObj = BasicHeader(LOCALTIME)
96 self.basicHeaderObj = BasicHeader(LOCALTIME)
96
97
97 self.systemHeaderObj = SystemHeader()
98 self.systemHeaderObj = SystemHeader()
98
99
99 self.radarControllerHeaderObj = RadarControllerHeader()
100 self.radarControllerHeaderObj = RadarControllerHeader()
100
101
101 self.processingHeaderObj = ProcessingHeader()
102 self.processingHeaderObj = ProcessingHeader()
102
103
103 self.online = 0
104 self.online = 0
104
105
105 self.fp = None
106 self.fp = None
106
107
107 self.idFile = None
108 self.idFile = None
108
109
109 self.dtype = None
110 self.dtype = None
110
111
111 self.fileSizeByHeader = None
112 self.fileSizeByHeader = None
112
113
113 self.filenameList = []
114 self.filenameList = []
114
115
115 self.filename = None
116 self.filename = None
116
117
117 self.fileSize = None
118 self.fileSize = None
118
119
119 self.firstHeaderSize = 0
120 self.firstHeaderSize = 0
120
121
121 self.basicHeaderSize = 24
122 self.basicHeaderSize = 24
122
123
123 self.pathList = []
124 self.pathList = []
124
125
125 self.filenameList = []
126 self.filenameList = []
126
127
127 self.lastUTTime = 0
128 self.lastUTTime = 0
128
129
129 self.maxTimeStep = 30
130 self.maxTimeStep = 30
130
131
131 self.flagNoMoreFiles = 0
132 self.flagNoMoreFiles = 0
132
133
133 self.set = 0
134 self.set = 0
134
135
135 self.path = None
136 self.path = None
136
137
137 self.profileIndex = 2**32-1
138 self.profileIndex = 2**32 - 1
138
139
139 self.delay = 3 #seconds
140 self.delay = 3 # seconds
140
141
141 self.nTries = 3 #quantity tries
142 self.nTries = 3 # quantity tries
142
143
143 self.nFiles = 3 #number of files for searching
144 self.nFiles = 3 # number of files for searching
144
145
145 self.nReadBlocks = 0
146 self.nReadBlocks = 0
146
147
147 self.flagIsNewFile = 1
148 self.flagIsNewFile = 1
148
149
149 self.__isFirstTimeOnline = 1
150 self.__isFirstTimeOnline = 1
150
151
151 # self.ippSeconds = 0
152 # self.ippSeconds = 0
152
153
153 self.flagDiscontinuousBlock = 0
154 self.flagDiscontinuousBlock = 0
154
155
155 self.flagIsNewBlock = 0
156 self.flagIsNewBlock = 0
156
157
157 self.nTotalBlocks = 0
158 self.nTotalBlocks = 0
158
159
159 self.blocksize = 0
160 self.blocksize = 0
160
161
161 self.dataOut = self.createObjByDefault()
162 self.dataOut = self.createObjByDefault()
162
163
163 self.nTxs = 1
164 self.nTxs = 1
164
165
165 self.txIndex = 0
166 self.txIndex = 0
166
167
167 def createObjByDefault(self):
168 def createObjByDefault(self):
168
169
169 dataObj = Voltage()
170 dataObj = Voltage()
170
171
171 return dataObj
172 return dataObj
172
173
173 def __hasNotDataInBuffer(self):
174 def __hasNotDataInBuffer(self):
174
175
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
176 return 1
177 return 1
177
178
178 return 0
179 return 0
179
180
180
181 def getBlockDimension(self):
181 def getBlockDimension(self):
182 """
182 """
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184
184
185 Affected:
185 Affected:
186 self.blocksize
186 self.blocksize
187
187
188 Return:
188 Return:
189 None
189 None
190 """
190 """
191 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
191 pts2read = self.processingHeaderObj.profilesPerBlock * \
192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
192 self.blocksize = pts2read
193 self.blocksize = pts2read
193
194
194
195
196 def readBlock(self):
195 def readBlock(self):
197 """
196 """
198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
199 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
200 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
201 es seteado a 0
200 es seteado a 0
202
201
203 Inputs:
202 Inputs:
204 None
203 None
205
204
206 Return:
205 Return:
207 None
206 None
208
207
209 Affected:
208 Affected:
210 self.profileIndex
209 self.profileIndex
211 self.datablock
210 self.datablock
212 self.flagIsNewFile
211 self.flagIsNewFile
213 self.flagIsNewBlock
212 self.flagIsNewBlock
214 self.nTotalBlocks
213 self.nTotalBlocks
215
214
216 Exceptions:
215 Exceptions:
217 Si un bloque leido no es un bloque valido
216 Si un bloque leido no es un bloque valido
218 """
217 """
219
218
220 # if self.server is not None:
219 # if self.server is not None:
221 # self.zBlock = self.receiver.recv()
220 # self.zBlock = self.receiver.recv()
222 # self.zHeader = self.zBlock[:24]
221 # self.zHeader = self.zBlock[:24]
223 # self.zDataBlock = self.zBlock[24:]
222 # self.zDataBlock = self.zBlock[24:]
224 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
223 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
225 # self.processingHeaderObj.profilesPerBlock = 240
224 # self.processingHeaderObj.profilesPerBlock = 240
226 # self.processingHeaderObj.nHeights = 248
225 # self.processingHeaderObj.nHeights = 248
227 # self.systemHeaderObj.nChannels
226 # self.systemHeaderObj.nChannels
228 # else:
227 # else:
229 current_pointer_location = self.fp.tell()
228 current_pointer_location = self.fp.tell()
230 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
231
230
232 try:
231 try:
233 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
234 except:
234 except:
235 #print "The read block (%3d) has not enough data" %self.nReadBlocks
235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
236
236
237 if self.waitDataBlock(pointer_location=current_pointer_location):
237 if self.waitDataBlock(pointer_location=current_pointer_location):
238 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
239 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
240 # return 0
241 # return 0
241
242
242 #Dimensions : nChannels, nProfiles, nSamples
243 # Dimensions : nChannels, nProfiles, nSamples
243
244
244 junk = numpy.transpose(junk, (2,0,1))
245 junk = numpy.transpose(junk, (2, 0, 1))
245 self.datablock = junk['real'] + junk['imag']*1j
246 self.datablock = junk['real'] + junk['imag'] * 1j
246
247
247 self.profileIndex = 0
248 self.profileIndex = 0
248
249
249 self.flagIsNewFile = 0
250 self.flagIsNewFile = 0
250 self.flagIsNewBlock = 1
251 self.flagIsNewBlock = 1
251
252
252 self.nTotalBlocks += 1
253 self.nTotalBlocks += 1
253 self.nReadBlocks += 1
254 self.nReadBlocks += 1
254
255
255 return 1
256 return 1
256
257
257 def getFirstHeader(self):
258 def getFirstHeader(self):
258
259
259 self.getBasicHeader()
260 self.getBasicHeader()
260
261
261 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
262 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
262
263
263 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
264 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
264
265
265 if self.nTxs > 1:
266 if self.nTxs > 1:
266 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
267 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
267 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
268 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
268
269
269 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
270 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
270 #
271 #
271 # if self.radarControllerHeaderObj.code is not None:
272 # if self.radarControllerHeaderObj.code is not None:
272 #
273 #
273 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
274 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
274 #
275 #
275 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
276 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
276 #
277 #
277 # self.dataOut.code = self.radarControllerHeaderObj.code
278 # self.dataOut.code = self.radarControllerHeaderObj.code
278
279
279 self.dataOut.dtype = self.dtype
280 self.dataOut.dtype = self.dtype
280
281
281 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
282 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
282
283
283 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
284 self.dataOut.heightList = numpy.arange(
285 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
284
286
285 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
287 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
286
288
287 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
289 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
288
290
289 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
291 # asumo q la data no esta decodificada
292 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
290
293
291 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
294 # asumo q la data no esta sin flip
295 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
292
296
293 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
297 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
294
298
295 def reshapeData(self):
299 def reshapeData(self):
296
300
297 if self.nTxs < 0:
301 if self.nTxs < 0:
298 return
302 return
299
303
300 if self.nTxs == 1:
304 if self.nTxs == 1:
301 return
305 return
302
306
303 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
307 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
304 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
308 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
309 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock)
305
310
306 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
311 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
307 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
312 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" % (
313 self.nTxs, self.processingHeaderObj.nHeights)
308
314
309 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
315 self.datablock = self.datablock.reshape(
316 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
310
317
311 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
318 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
312 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
319 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
320 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
313 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
321 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
314
322
315 return
323 return
316
324
317 def readFirstHeaderFromServer(self):
325 def readFirstHeaderFromServer(self):
318
326
319 self.getFirstHeader()
327 self.getFirstHeader()
320
328
321 self.firstHeaderSize = self.basicHeaderObj.size
329 self.firstHeaderSize = self.basicHeaderObj.size
322
330
323 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
331 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
332 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
324 if datatype == 0:
333 if datatype == 0:
325 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
334 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
326 elif datatype == 1:
335 elif datatype == 1:
327 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
336 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
328 elif datatype == 2:
337 elif datatype == 2:
329 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
338 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
330 elif datatype == 3:
339 elif datatype == 3:
331 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
340 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
332 elif datatype == 4:
341 elif datatype == 4:
333 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
342 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
334 elif datatype == 5:
343 elif datatype == 5:
335 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
344 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
336 else:
345 else:
337 raise ValueError, 'Data type was not defined'
346 raise ValueError, 'Data type was not defined'
338
347
339 self.dtype = datatype_str
348 self.dtype = datatype_str
340 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
349 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
341 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
350 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
351 self.firstHeaderSize + self.basicHeaderSize * \
352 (self.processingHeaderObj.dataBlocksPerFile - 1)
342 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
353 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
343 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
354 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
344 self.getBlockDimension()
355 self.getBlockDimension()
345
356
346
347 def getFromServer(self):
357 def getFromServer(self):
348 self.flagDiscontinuousBlock = 0
358 self.flagDiscontinuousBlock = 0
349 self.profileIndex = 0
359 self.profileIndex = 0
350 self.flagIsNewBlock = 1
360 self.flagIsNewBlock = 1
351 self.dataOut.flagNoData = False
361 self.dataOut.flagNoData = False
352 self.nTotalBlocks += 1
362 self.nTotalBlocks += 1
353 self.nReadBlocks += 1
363 self.nReadBlocks += 1
354 self.blockPointer = 0
364 self.blockPointer = 0
355
365
356 block = self.receiver.recv()
366 block = self.receiver.recv()
357
367
358 self.basicHeaderObj.read(block[self.blockPointer:])
368 self.basicHeaderObj.read(block[self.blockPointer:])
359 self.blockPointer += self.basicHeaderObj.length
369 self.blockPointer += self.basicHeaderObj.length
360 self.systemHeaderObj.read(block[self.blockPointer:])
370 self.systemHeaderObj.read(block[self.blockPointer:])
361 self.blockPointer += self.systemHeaderObj.length
371 self.blockPointer += self.systemHeaderObj.length
362 self.radarControllerHeaderObj.read(block[self.blockPointer:])
372 self.radarControllerHeaderObj.read(block[self.blockPointer:])
363 self.blockPointer += self.radarControllerHeaderObj.length
373 self.blockPointer += self.radarControllerHeaderObj.length
364 self.processingHeaderObj.read(block[self.blockPointer:])
374 self.processingHeaderObj.read(block[self.blockPointer:])
365 self.blockPointer += self.processingHeaderObj.length
375 self.blockPointer += self.processingHeaderObj.length
366 self.readFirstHeaderFromServer()
376 self.readFirstHeaderFromServer()
367
377
368 timestamp = self.basicHeaderObj.get_datatime()
378 timestamp = self.basicHeaderObj.get_datatime()
369 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
379 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
370 current_pointer_location = self.blockPointer
380 current_pointer_location = self.blockPointer
371 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
381 junk = numpy.fromstring(
382 block[self.blockPointer:], self.dtype, self.blocksize)
372
383
373 try:
384 try:
374 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
385 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
386 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
375 except:
387 except:
376 #print "The read block (%3d) has not enough data" %self.nReadBlocks
388 # print "The read block (%3d) has not enough data" %self.nReadBlocks
377 if self.waitDataBlock(pointer_location=current_pointer_location):
389 if self.waitDataBlock(pointer_location=current_pointer_location):
378 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
390 junk = numpy.fromstring(
379 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
391 block[self.blockPointer:], self.dtype, self.blocksize)
392 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
393 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
380 # return 0
394 # return 0
381
395
382 #Dimensions : nChannels, nProfiles, nSamples
396 # Dimensions : nChannels, nProfiles, nSamples
383
397
384 junk = numpy.transpose(junk, (2,0,1))
398 junk = numpy.transpose(junk, (2, 0, 1))
385 self.datablock = junk['real'] + junk['imag'] * 1j
399 self.datablock = junk['real'] + junk['imag'] * 1j
386 self.profileIndex = 0
400 self.profileIndex = 0
387 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
401 if self.selBlocksize == None:
402 self.selBlocksize = self.dataOut.nProfiles
388 if self.selBlocktime != None:
403 if self.selBlocktime != None:
389 if self.dataOut.nCohInt is not None:
404 if self.dataOut.nCohInt is not None:
390 nCohInt = self.dataOut.nCohInt
405 nCohInt = self.dataOut.nCohInt
391 else:
406 else:
392 nCohInt = 1
407 nCohInt = 1
393 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
408 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
394 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
409 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
410 self.dataOut.data = self.datablock[:,
411 self.profileIndex:self.profileIndex + self.selBlocksize, :]
395 datasize = self.dataOut.data.shape[1]
412 datasize = self.dataOut.data.shape[1]
396 if datasize < self.selBlocksize:
413 if datasize < self.selBlocksize:
397 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
414 buffer = numpy.zeros(
415 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
398 buffer[:,:datasize,:] = self.dataOut.data
416 buffer[:, :datasize, :] = self.dataOut.data
399 self.dataOut.data = buffer
417 self.dataOut.data = buffer
400 self.profileIndex = blockIndex
418 self.profileIndex = blockIndex
401
419
402 self.dataOut.flagDataAsBlock = True
420 self.dataOut.flagDataAsBlock = True
403 self.flagIsNewBlock = 1
421 self.flagIsNewBlock = 1
404 self.dataOut.realtime = self.online
422 self.dataOut.realtime = self.online
405
423
406 return self.dataOut.data
424 return self.dataOut.data
407
425
408 def getData(self):
426 def getData(self):
409 """
427 """
410 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
428 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
411 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
429 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
412 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
430 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
413 "readNextBlock"
431 "readNextBlock"
414
432
415 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
433 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
416
434
417 Return:
435 Return:
418
436
419 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
437 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
420 es igual al total de perfiles leidos desde el archivo.
438 es igual al total de perfiles leidos desde el archivo.
421
439
422 Si self.getByBlock == False:
440 Si self.getByBlock == False:
423
441
424 self.dataOut.data = buffer[:, thisProfile, :]
442 self.dataOut.data = buffer[:, thisProfile, :]
425
443
426 shape = [nChannels, nHeis]
444 shape = [nChannels, nHeis]
427
445
428 Si self.getByBlock == True:
446 Si self.getByBlock == True:
429
447
430 self.dataOut.data = buffer[:, :, :]
448 self.dataOut.data = buffer[:, :, :]
431
449
432 shape = [nChannels, nProfiles, nHeis]
450 shape = [nChannels, nProfiles, nHeis]
433
451
434 Variables afectadas:
452 Variables afectadas:
435 self.dataOut
453 self.dataOut
436 self.profileIndex
454 self.profileIndex
437
455
438 Affected:
456 Affected:
439 self.dataOut
457 self.dataOut
440 self.profileIndex
458 self.profileIndex
441 self.flagDiscontinuousBlock
459 self.flagDiscontinuousBlock
442 self.flagIsNewBlock
460 self.flagIsNewBlock
443 """
461 """
444 if self.flagNoMoreFiles:
462 if self.flagNoMoreFiles:
445 self.dataOut.flagNoData = True
463 self.dataOut.flagNoData = True
446 print 'Process finished'
464 print 'Process finished'
447 return 0
465 return 0
448 self.flagDiscontinuousBlock = 0
466 self.flagDiscontinuousBlock = 0
449 self.flagIsNewBlock = 0
467 self.flagIsNewBlock = 0
450 if self.__hasNotDataInBuffer():
468 if self.__hasNotDataInBuffer():
451 if not( self.readNextBlock() ):
469 if not(self.readNextBlock()):
452 return 0
470 return 0
453
471
454 self.getFirstHeader()
472 self.getFirstHeader()
455
473
456 self.reshapeData()
474 self.reshapeData()
457 if self.datablock is None:
475 if self.datablock is None:
458 self.dataOut.flagNoData = True
476 self.dataOut.flagNoData = True
459 return 0
477 return 0
460
478
461 if not self.getByBlock:
479 if not self.getByBlock:
462
480
463 """
481 """
464 Return profile by profile
482 Return profile by profile
465
483
466 If nTxs > 1 then one profile is divided by nTxs and number of total
484 If nTxs > 1 then one profile is divided by nTxs and number of total
467 blocks is increased by nTxs (nProfiles *= nTxs)
485 blocks is increased by nTxs (nProfiles *= nTxs)
468 """
486 """
469 self.dataOut.flagDataAsBlock = False
487 self.dataOut.flagDataAsBlock = False
470 self.dataOut.data = self.datablock[:,self.profileIndex,:]
488 self.dataOut.data = self.datablock[:, self.profileIndex, :]
471 self.dataOut.profileIndex = self.profileIndex
489 self.dataOut.profileIndex = self.profileIndex
472
490
473 self.profileIndex += 1
491 self.profileIndex += 1
474
492
475 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
493 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
476 # """
494 # """
477 # Return all block
495 # Return all block
478 # """
496 # """
479 # self.dataOut.flagDataAsBlock = True
497 # self.dataOut.flagDataAsBlock = True
480 # self.dataOut.data = self.datablock
498 # self.dataOut.data = self.datablock
481 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
499 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
482 #
500 #
483 # self.profileIndex = self.dataOut.nProfiles
501 # self.profileIndex = self.dataOut.nProfiles
484
502
485 else:
503 else:
486 """
504 """
487 Return a block
505 Return a block
488 """
506 """
489 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
507 if self.selBlocksize == None:
508 self.selBlocksize = self.dataOut.nProfiles
490 if self.selBlocktime != None:
509 if self.selBlocktime != None:
491 if self.dataOut.nCohInt is not None:
510 if self.dataOut.nCohInt is not None:
492 nCohInt = self.dataOut.nCohInt
511 nCohInt = self.dataOut.nCohInt
493 else:
512 else:
494 nCohInt = 1
513 nCohInt = 1
495 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
514 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
515 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
496
516
497 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
517 self.dataOut.data = self.datablock[:,
518 self.profileIndex:self.profileIndex + self.selBlocksize, :]
498 self.profileIndex += self.selBlocksize
519 self.profileIndex += self.selBlocksize
499 datasize = self.dataOut.data.shape[1]
520 datasize = self.dataOut.data.shape[1]
500
521
501 if datasize < self.selBlocksize:
522 if datasize < self.selBlocksize:
502 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
523 buffer = numpy.zeros(
524 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
503 buffer[:,:datasize,:] = self.dataOut.data
525 buffer[:, :datasize, :] = self.dataOut.data
504
526
505 while datasize < self.selBlocksize: #Not enough profiles to fill the block
527 while datasize < self.selBlocksize: # Not enough profiles to fill the block
506 if not( self.readNextBlock() ):
528 if not(self.readNextBlock()):
507 return 0
529 return 0
508 self.getFirstHeader()
530 self.getFirstHeader()
509 self.reshapeData()
531 self.reshapeData()
510 if self.datablock is None:
532 if self.datablock is None:
511 self.dataOut.flagNoData = True
533 self.dataOut.flagNoData = True
512 return 0
534 return 0
513 #stack data
535 # stack data
514 blockIndex = self.selBlocksize - datasize
536 blockIndex = self.selBlocksize - datasize
515 datablock1 = self.datablock[:,:blockIndex,:]
537 datablock1 = self.datablock[:, :blockIndex, :]
516
538
517 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
539 buffer[:, datasize:datasize +
540 datablock1.shape[1], :] = datablock1
518 datasize += datablock1.shape[1]
541 datasize += datablock1.shape[1]
519
542
520 self.dataOut.data = buffer
543 self.dataOut.data = buffer
521 self.profileIndex = blockIndex
544 self.profileIndex = blockIndex
522
545
523 self.dataOut.flagDataAsBlock = True
546 self.dataOut.flagDataAsBlock = True
524 self.dataOut.nProfiles = self.dataOut.data.shape[1]
547 self.dataOut.nProfiles = self.dataOut.data.shape[1]
525
548
526 self.dataOut.flagNoData = False
549 self.dataOut.flagNoData = False
527
550
528 self.getBasicHeader()
551 self.getBasicHeader()
529
552
530 self.dataOut.realtime = self.online
553 self.dataOut.realtime = self.online
531
554
532 return self.dataOut.data
555 return self.dataOut.data
533
556
557
534 class VoltageWriter(JRODataWriter, Operation):
558 class VoltageWriter(JRODataWriter, Operation):
535 """
559 """
536 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
560 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
537 de los datos siempre se realiza por bloques.
561 de los datos siempre se realiza por bloques.
538 """
562 """
539
563
540 ext = ".r"
564 ext = ".r"
541
565
542 optchar = "D"
566 optchar = "D"
543
567
544 shapeBuffer = None
568 shapeBuffer = None
545
569
546
547 def __init__(self, **kwargs):
570 def __init__(self, **kwargs):
548 """
571 """
549 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
572 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
550
573
551 Affected:
574 Affected:
552 self.dataOut
575 self.dataOut
553
576
554 Return: None
577 Return: None
555 """
578 """
556 Operation.__init__(self, **kwargs)
579 Operation.__init__(self, **kwargs)
557
580
558 self.nTotalBlocks = 0
581 self.nTotalBlocks = 0
559
582
560 self.profileIndex = 0
583 self.profileIndex = 0
561
584
562 self.isConfig = False
585 self.isConfig = False
563
586
564 self.fp = None
587 self.fp = None
565
588
566 self.flagIsNewFile = 1
589 self.flagIsNewFile = 1
567
590
568 self.blockIndex = 0
591 self.blockIndex = 0
569
592
570 self.flagIsNewBlock = 0
593 self.flagIsNewBlock = 0
571
594
572 self.setFile = None
595 self.setFile = None
573
596
574 self.dtype = None
597 self.dtype = None
575
598
576 self.path = None
599 self.path = None
577
600
578 self.filename = None
601 self.filename = None
579
602
580 self.basicHeaderObj = BasicHeader(LOCALTIME)
603 self.basicHeaderObj = BasicHeader(LOCALTIME)
581
604
582 self.systemHeaderObj = SystemHeader()
605 self.systemHeaderObj = SystemHeader()
583
606
584 self.radarControllerHeaderObj = RadarControllerHeader()
607 self.radarControllerHeaderObj = RadarControllerHeader()
585
608
586 self.processingHeaderObj = ProcessingHeader()
609 self.processingHeaderObj = ProcessingHeader()
587
610
588 def hasAllDataInBuffer(self):
611 def hasAllDataInBuffer(self):
589 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
612 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
590 return 1
613 return 1
591 return 0
614 return 0
592
615
593
594 def setBlockDimension(self):
616 def setBlockDimension(self):
595 """
617 """
596 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
618 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
597
619
598 Affected:
620 Affected:
599 self.shape_spc_Buffer
621 self.shape_spc_Buffer
600 self.shape_cspc_Buffer
622 self.shape_cspc_Buffer
601 self.shape_dc_Buffer
623 self.shape_dc_Buffer
602
624
603 Return: None
625 Return: None
604 """
626 """
605 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
627 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
606 self.processingHeaderObj.nHeights,
628 self.processingHeaderObj.nHeights,
607 self.systemHeaderObj.nChannels)
629 self.systemHeaderObj.nChannels)
608
630
609 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
631 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
610 self.processingHeaderObj.profilesPerBlock,
632 self.processingHeaderObj.profilesPerBlock,
611 self.processingHeaderObj.nHeights),
633 self.processingHeaderObj.nHeights),
612 dtype=numpy.dtype('complex64'))
634 dtype=numpy.dtype('complex64'))
613
635
614 def writeBlock(self):
636 def writeBlock(self):
615 """
637 """
616 Escribe el buffer en el file designado
638 Escribe el buffer en el file designado
617
639
618 Affected:
640 Affected:
619 self.profileIndex
641 self.profileIndex
620 self.flagIsNewFile
642 self.flagIsNewFile
621 self.flagIsNewBlock
643 self.flagIsNewBlock
622 self.nTotalBlocks
644 self.nTotalBlocks
623 self.blockIndex
645 self.blockIndex
624
646
625 Return: None
647 Return: None
626 """
648 """
627 data = numpy.zeros( self.shapeBuffer, self.dtype )
649 data = numpy.zeros(self.shapeBuffer, self.dtype)
628
650
629 junk = numpy.transpose(self.datablock, (1,2,0))
651 junk = numpy.transpose(self.datablock, (1, 2, 0))
630
652
631 data['real'] = junk.real
653 data['real'] = junk.real
632 data['imag'] = junk.imag
654 data['imag'] = junk.imag
633
655
634 data = data.reshape( (-1) )
656 data = data.reshape((-1))
635
657
636 data.tofile( self.fp )
658 data.tofile(self.fp)
637
659
638 self.datablock.fill(0)
660 self.datablock.fill(0)
639
661
640 self.profileIndex = 0
662 self.profileIndex = 0
641 self.flagIsNewFile = 0
663 self.flagIsNewFile = 0
642 self.flagIsNewBlock = 1
664 self.flagIsNewBlock = 1
643
665
644 self.blockIndex += 1
666 self.blockIndex += 1
645 self.nTotalBlocks += 1
667 self.nTotalBlocks += 1
646
668
647 # print "[Writing] Block = %04d" %self.blockIndex
669 # print "[Writing] Block = %04d" %self.blockIndex
648
670
649 def putData(self):
671 def putData(self):
650 """
672 """
651 Setea un bloque de datos y luego los escribe en un file
673 Setea un bloque de datos y luego los escribe en un file
652
674
653 Affected:
675 Affected:
654 self.flagIsNewBlock
676 self.flagIsNewBlock
655 self.profileIndex
677 self.profileIndex
656
678
657 Return:
679 Return:
658 0 : Si no hay data o no hay mas files que puedan escribirse
680 0 : Si no hay data o no hay mas files que puedan escribirse
659 1 : Si se escribio la data de un bloque en un file
681 1 : Si se escribio la data de un bloque en un file
660 """
682 """
661 if self.dataOut.flagNoData:
683 if self.dataOut.flagNoData:
662 return 0
684 return 0
663
685
664 self.flagIsNewBlock = 0
686 self.flagIsNewBlock = 0
665
687
666 if self.dataOut.flagDiscontinuousBlock:
688 if self.dataOut.flagDiscontinuousBlock:
667 self.datablock.fill(0)
689 self.datablock.fill(0)
668 self.profileIndex = 0
690 self.profileIndex = 0
669 self.setNextFile()
691 self.setNextFile()
670
692
671 if self.profileIndex == 0:
693 if self.profileIndex == 0:
672 self.setBasicHeader()
694 self.setBasicHeader()
673
695
674 self.datablock[:,self.profileIndex,:] = self.dataOut.data
696 self.datablock[:, self.profileIndex, :] = self.dataOut.data
675
697
676 self.profileIndex += 1
698 self.profileIndex += 1
677
699
678 if self.hasAllDataInBuffer():
700 if self.hasAllDataInBuffer():
679 #if self.flagIsNewFile:
701 # if self.flagIsNewFile:
680 self.writeNextBlock()
702 self.writeNextBlock()
681 # self.setFirstHeader()
703 # self.setFirstHeader()
682
704
683 return 1
705 return 1
684
706
685 def __getBlockSize(self):
707 def __getBlockSize(self):
686 '''
708 '''
687 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
709 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
688 '''
710 '''
689
711
690 dtype_width = self.getDtypeWidth()
712 dtype_width = self.getDtypeWidth()
691
713
692 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
714 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
715 self.profilesPerBlock * dtype_width * 2)
693
716
694 return blocksize
717 return blocksize
695
718
696 def setFirstHeader(self):
719 def setFirstHeader(self):
697
698 """
720 """
699 Obtiene una copia del First Header
721 Obtiene una copia del First Header
700
722
701 Affected:
723 Affected:
702 self.systemHeaderObj
724 self.systemHeaderObj
703 self.radarControllerHeaderObj
725 self.radarControllerHeaderObj
704 self.dtype
726 self.dtype
705
727
706 Return:
728 Return:
707 None
729 None
708 """
730 """
709
731
710 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
732 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
711 self.systemHeaderObj.nChannels = self.dataOut.nChannels
733 self.systemHeaderObj.nChannels = self.dataOut.nChannels
712 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
734 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
713
735
714 self.processingHeaderObj.dtype = 0 # Voltage
736 self.processingHeaderObj.dtype = 0 # Voltage
715 self.processingHeaderObj.blockSize = self.__getBlockSize()
737 self.processingHeaderObj.blockSize = self.__getBlockSize()
716 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
738 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
717 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
739 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
718 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
740 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
741 self.processingHeaderObj.nWindows = 1
719 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
742 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
720 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
743 # Cuando la data de origen es de tipo Voltage
721 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
744 self.processingHeaderObj.nIncohInt = 1
745 # Cuando la data de origen es de tipo Voltage
746 self.processingHeaderObj.totalSpectra = 0
722
747
723 if self.dataOut.code is not None:
748 if self.dataOut.code is not None:
724 self.processingHeaderObj.code = self.dataOut.code
749 self.processingHeaderObj.code = self.dataOut.code
725 self.processingHeaderObj.nCode = self.dataOut.nCode
750 self.processingHeaderObj.nCode = self.dataOut.nCode
726 self.processingHeaderObj.nBaud = self.dataOut.nBaud
751 self.processingHeaderObj.nBaud = self.dataOut.nBaud
727
752
728 if self.processingHeaderObj.nWindows != 0:
753 if self.processingHeaderObj.nWindows != 0:
729 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
754 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
730 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
755 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
756 self.dataOut.heightList[0]
731 self.processingHeaderObj.nHeights = self.dataOut.nHeights
757 self.processingHeaderObj.nHeights = self.dataOut.nHeights
732 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
758 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
733
759
734 self.processingHeaderObj.processFlags = self.getProcessFlags()
760 self.processingHeaderObj.processFlags = self.getProcessFlags()
735
761
736 self.setBasicHeader()
762 self.setBasicHeader()
General Comments 0
You need to be logged in to leave comments. Login now