##// END OF EJS Templates
schain-jc
RSS
set new matplotlib version in setup, improved searchFiles for multiprocessing
Juan C. Espinoza -
r1112:4bbef2dc4ffc
parent child
Show More
Side by Side Unified
Context file:
r1112:4bbef2dc4ffc Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1317 +1,1323
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, cpu_count
12 from multiprocessing import Process, cpu_count
13
13
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 from xml.dom import minidom
15 from xml.dom import minidom
16
16
17 import schainpy
17 import schainpy
18 import schainpy.admin
18 import schainpy.admin
19 from schainpy.model import *
19 from schainpy.model import *
20 from schainpy.utils import log
20 from schainpy.utils import log
21
21
22 DTYPES = {
22 DTYPES = {
23 'Voltage': '.r',
23 'Voltage': '.r',
24 'Spectra': '.pdata'
24 'Spectra': '.pdata'
25 }
25 }
26
26
27
27
28 def MPProject(project, n=cpu_count()):
28 def MPProject(project, n=cpu_count()):
29 '''
29 '''
30 Project wrapper to run schain in n processes
30 Project wrapper to run schain in n processes
31 '''
31 '''
32
32
33 rconf = project.getReadUnitObj()
33 rconf = project.getReadUnitObj()
34 op = rconf.getOperationObj('run')
34 op = rconf.getOperationObj('run')
35 dt1 = op.getParameterValue('startDate')
35 dt1 = op.getParameterValue('startDate')
36 dt2 = op.getParameterValue('endDate')
36 dt2 = op.getParameterValue('endDate')
37 tm1 = op.getParameterValue('startTime')
38 tm2 = op.getParameterValue('endTime')
37 days = (dt2 - dt1).days
39 days = (dt2 - dt1).days
38
40
39 for day in range(days + 1):
41 for day in range(days + 1):
40 skip = 0
42 skip = 0
41 cursor = 0
43 cursor = 0
42 processes = []
44 processes = []
43 dt = dt1 + datetime.timedelta(day)
45 dt = dt1 + datetime.timedelta(day)
44 dt_str = dt.strftime('%Y/%m/%d')
46 dt_str = dt.strftime('%Y/%m/%d')
45 reader = JRODataReader()
47 reader = JRODataReader()
46 paths, files = reader.searchFilesOffLine(path=rconf.path,
48 paths, files = reader.searchFilesOffLine(path=rconf.path,
47 startDate=dt,
49 startDate=dt,
48 endDate=dt,
50 endDate=dt,
51 startTime=tm1,
52 endTime=tm2,
49 ext=DTYPES[rconf.datatype])
53 ext=DTYPES[rconf.datatype])
50 nFiles = len(files)
54 nFiles = len(files)
51 if nFiles == 0:
55 if nFiles == 0:
52 continue
56 continue
53 skip = int(math.ceil(nFiles / n))
57 skip = int(math.ceil(nFiles / n))
54 while nFiles > cursor * skip:
58 while nFiles > cursor * skip:
55 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
56 skip=skip)
60 skip=skip)
57 p = project.clone()
61 p = project.clone()
58 p.start()
62 p.start()
59 processes.append(p)
63 processes.append(p)
60 cursor += 1
64 cursor += 1
61
65
62 def beforeExit(exctype, value, trace):
66 def beforeExit(exctype, value, trace):
63 for process in processes:
67 for process in processes:
64 process.terminate()
68 process.terminate()
65 process.join()
69 process.join()
66 print traceback.print_tb(trace)
70 print traceback.print_tb(trace)
67
71
68 sys.excepthook = beforeExit
72 sys.excepthook = beforeExit
69
73
70 for process in processes:
74 for process in processes:
71 process.join()
75 process.join()
72 process.terminate()
76 process.terminate()
73
77
74 time.sleep(3)
78 time.sleep(3)
75
79
76
80
77 class ParameterConf():
81 class ParameterConf():
78
82
79 id = None
83 id = None
80 name = None
84 name = None
81 value = None
85 value = None
82 format = None
86 format = None
83
87
84 __formated_value = None
88 __formated_value = None
85
89
86 ELEMENTNAME = 'Parameter'
90 ELEMENTNAME = 'Parameter'
87
91
88 def __init__(self):
92 def __init__(self):
89
93
90 self.format = 'str'
94 self.format = 'str'
91
95
92 def getElementName(self):
96 def getElementName(self):
93
97
94 return self.ELEMENTNAME
98 return self.ELEMENTNAME
95
99
96 def getValue(self):
100 def getValue(self):
97
101
98 value = self.value
102 value = self.value
99 format = self.format
103 format = self.format
100
104
101 if self.__formated_value != None:
105 if self.__formated_value != None:
102
106
103 return self.__formated_value
107 return self.__formated_value
104
108
105 if format == 'obj':
109 if format == 'obj':
106 return value
110 return value
107
111
108 if format == 'str':
112 if format == 'str':
109 self.__formated_value = str(value)
113 self.__formated_value = str(value)
110 return self.__formated_value
114 return self.__formated_value
111
115
112 if value == '':
116 if value == '':
113 raise ValueError, '%s: This parameter value is empty' % self.name
117 raise ValueError, '%s: This parameter value is empty' % self.name
114
118
115 if format == 'list':
119 if format == 'list':
116 strList = value.split(',')
120 strList = value.split(',')
117
121
118 self.__formated_value = strList
122 self.__formated_value = strList
119
123
120 return self.__formated_value
124 return self.__formated_value
121
125
122 if format == 'intlist':
126 if format == 'intlist':
123 '''
127 '''
124 Example:
128 Example:
125 value = (0,1,2)
129 value = (0,1,2)
126 '''
130 '''
127
131
128 new_value = ast.literal_eval(value)
132 new_value = ast.literal_eval(value)
129
133
130 if type(new_value) not in (tuple, list):
134 if type(new_value) not in (tuple, list):
131 new_value = [int(new_value)]
135 new_value = [int(new_value)]
132
136
133 self.__formated_value = new_value
137 self.__formated_value = new_value
134
138
135 return self.__formated_value
139 return self.__formated_value
136
140
137 if format == 'floatlist':
141 if format == 'floatlist':
138 '''
142 '''
139 Example:
143 Example:
140 value = (0.5, 1.4, 2.7)
144 value = (0.5, 1.4, 2.7)
141 '''
145 '''
142
146
143 new_value = ast.literal_eval(value)
147 new_value = ast.literal_eval(value)
144
148
145 if type(new_value) not in (tuple, list):
149 if type(new_value) not in (tuple, list):
146 new_value = [float(new_value)]
150 new_value = [float(new_value)]
147
151
148 self.__formated_value = new_value
152 self.__formated_value = new_value
149
153
150 return self.__formated_value
154 return self.__formated_value
151
155
152 if format == 'date':
156 if format == 'date':
153 strList = value.split('/')
157 strList = value.split('/')
154 intList = [int(x) for x in strList]
158 intList = [int(x) for x in strList]
155 date = datetime.date(intList[0], intList[1], intList[2])
159 date = datetime.date(intList[0], intList[1], intList[2])
156
160
157 self.__formated_value = date
161 self.__formated_value = date
158
162
159 return self.__formated_value
163 return self.__formated_value
160
164
161 if format == 'time':
165 if format == 'time':
162 strList = value.split(':')
166 strList = value.split(':')
163 intList = [int(x) for x in strList]
167 intList = [int(x) for x in strList]
164 time = datetime.time(intList[0], intList[1], intList[2])
168 time = datetime.time(intList[0], intList[1], intList[2])
165
169
166 self.__formated_value = time
170 self.__formated_value = time
167
171
168 return self.__formated_value
172 return self.__formated_value
169
173
170 if format == 'pairslist':
174 if format == 'pairslist':
171 '''
175 '''
172 Example:
176 Example:
173 value = (0,1),(1,2)
177 value = (0,1),(1,2)
174 '''
178 '''
175
179
176 new_value = ast.literal_eval(value)
180 new_value = ast.literal_eval(value)
177
181
178 if type(new_value) not in (tuple, list):
182 if type(new_value) not in (tuple, list):
179 raise ValueError, '%s has to be a tuple or list of pairs' % value
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
180
184
181 if type(new_value[0]) not in (tuple, list):
185 if type(new_value[0]) not in (tuple, list):
182 if len(new_value) != 2:
186 if len(new_value) != 2:
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
187 raise ValueError, '%s has to be a tuple or list of pairs' % value
184 new_value = [new_value]
188 new_value = [new_value]
185
189
186 for thisPair in new_value:
190 for thisPair in new_value:
187 if len(thisPair) != 2:
191 if len(thisPair) != 2:
188 raise ValueError, '%s has to be a tuple or list of pairs' % value
192 raise ValueError, '%s has to be a tuple or list of pairs' % value
189
193
190 self.__formated_value = new_value
194 self.__formated_value = new_value
191
195
192 return self.__formated_value
196 return self.__formated_value
193
197
194 if format == 'multilist':
198 if format == 'multilist':
195 '''
199 '''
196 Example:
200 Example:
197 value = (0,1,2),(3,4,5)
201 value = (0,1,2),(3,4,5)
198 '''
202 '''
199 multiList = ast.literal_eval(value)
203 multiList = ast.literal_eval(value)
200
204
201 if type(multiList[0]) == int:
205 if type(multiList[0]) == int:
202 multiList = ast.literal_eval('(' + value + ')')
206 multiList = ast.literal_eval('(' + value + ')')
203
207
204 self.__formated_value = multiList
208 self.__formated_value = multiList
205
209
206 return self.__formated_value
210 return self.__formated_value
207
211
208 if format == 'bool':
212 if format == 'bool':
209 value = int(value)
213 value = int(value)
210
214
211 if format == 'int':
215 if format == 'int':
212 value = float(value)
216 value = float(value)
213
217
214 format_func = eval(format)
218 format_func = eval(format)
215
219
216 self.__formated_value = format_func(value)
220 self.__formated_value = format_func(value)
217
221
218 return self.__formated_value
222 return self.__formated_value
219
223
220 def updateId(self, new_id):
224 def updateId(self, new_id):
221
225
222 self.id = str(new_id)
226 self.id = str(new_id)
223
227
224 def setup(self, id, name, value, format='str'):
228 def setup(self, id, name, value, format='str'):
225 self.id = str(id)
229 self.id = str(id)
226 self.name = name
230 self.name = name
227 if format == 'obj':
231 if format == 'obj':
228 self.value = value
232 self.value = value
229 else:
233 else:
230 self.value = str(value)
234 self.value = str(value)
231 self.format = str.lower(format)
235 self.format = str.lower(format)
232
236
233 self.getValue()
237 self.getValue()
234
238
235 return 1
239 return 1
236
240
237 def update(self, name, value, format='str'):
241 def update(self, name, value, format='str'):
238
242
239 self.name = name
243 self.name = name
240 self.value = str(value)
244 self.value = str(value)
241 self.format = format
245 self.format = format
242
246
243 def makeXml(self, opElement):
247 def makeXml(self, opElement):
244 if self.name not in ('queue',):
248 if self.name not in ('queue',):
245 parmElement = SubElement(opElement, self.ELEMENTNAME)
249 parmElement = SubElement(opElement, self.ELEMENTNAME)
246 parmElement.set('id', str(self.id))
250 parmElement.set('id', str(self.id))
247 parmElement.set('name', self.name)
251 parmElement.set('name', self.name)
248 parmElement.set('value', self.value)
252 parmElement.set('value', self.value)
249 parmElement.set('format', self.format)
253 parmElement.set('format', self.format)
250
254
251 def readXml(self, parmElement):
255 def readXml(self, parmElement):
252
256
253 self.id = parmElement.get('id')
257 self.id = parmElement.get('id')
254 self.name = parmElement.get('name')
258 self.name = parmElement.get('name')
255 self.value = parmElement.get('value')
259 self.value = parmElement.get('value')
256 self.format = str.lower(parmElement.get('format'))
260 self.format = str.lower(parmElement.get('format'))
257
261
258 # Compatible with old signal chain version
262 # Compatible with old signal chain version
259 if self.format == 'int' and self.name == 'idfigure':
263 if self.format == 'int' and self.name == 'idfigure':
260 self.name = 'id'
264 self.name = 'id'
261
265
262 def printattr(self):
266 def printattr(self):
263
267
264 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
265
269
266
270
267 class OperationConf():
271 class OperationConf():
268
272
269 id = None
273 id = None
270 name = None
274 name = None
271 priority = None
275 priority = None
272 type = None
276 type = None
273
277
274 parmConfObjList = []
278 parmConfObjList = []
275
279
276 ELEMENTNAME = 'Operation'
280 ELEMENTNAME = 'Operation'
277
281
278 def __init__(self):
282 def __init__(self):
279
283
280 self.id = '0'
284 self.id = '0'
281 self.name = None
285 self.name = None
282 self.priority = None
286 self.priority = None
283 self.type = 'self'
287 self.type = 'self'
284
288
285 def __getNewId(self):
289 def __getNewId(self):
286
290
287 return int(self.id) * 10 + len(self.parmConfObjList) + 1
291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
288
292
289 def updateId(self, new_id):
293 def updateId(self, new_id):
290
294
291 self.id = str(new_id)
295 self.id = str(new_id)
292
296
293 n = 1
297 n = 1
294 for parmObj in self.parmConfObjList:
298 for parmObj in self.parmConfObjList:
295
299
296 idParm = str(int(new_id) * 10 + n)
300 idParm = str(int(new_id) * 10 + n)
297 parmObj.updateId(idParm)
301 parmObj.updateId(idParm)
298
302
299 n += 1
303 n += 1
300
304
301 def getElementName(self):
305 def getElementName(self):
302
306
303 return self.ELEMENTNAME
307 return self.ELEMENTNAME
304
308
305 def getParameterObjList(self):
309 def getParameterObjList(self):
306
310
307 return self.parmConfObjList
311 return self.parmConfObjList
308
312
309 def getParameterObj(self, parameterName):
313 def getParameterObj(self, parameterName):
310
314
311 for parmConfObj in self.parmConfObjList:
315 for parmConfObj in self.parmConfObjList:
312
316
313 if parmConfObj.name != parameterName:
317 if parmConfObj.name != parameterName:
314 continue
318 continue
315
319
316 return parmConfObj
320 return parmConfObj
317
321
318 return None
322 return None
319
323
320 def getParameterObjfromValue(self, parameterValue):
324 def getParameterObjfromValue(self, parameterValue):
321
325
322 for parmConfObj in self.parmConfObjList:
326 for parmConfObj in self.parmConfObjList:
323
327
324 if parmConfObj.getValue() != parameterValue:
328 if parmConfObj.getValue() != parameterValue:
325 continue
329 continue
326
330
327 return parmConfObj.getValue()
331 return parmConfObj.getValue()
328
332
329 return None
333 return None
330
334
331 def getParameterValue(self, parameterName):
335 def getParameterValue(self, parameterName):
332
336
333 parameterObj = self.getParameterObj(parameterName)
337 parameterObj = self.getParameterObj(parameterName)
334
338
335 # if not parameterObj:
339 # if not parameterObj:
336 # return None
340 # return None
337
341
338 value = parameterObj.getValue()
342 value = parameterObj.getValue()
339
343
340 return value
344 return value
341
345
342 def getKwargs(self):
346 def getKwargs(self):
343
347
344 kwargs = {}
348 kwargs = {}
345
349
346 for parmConfObj in self.parmConfObjList:
350 for parmConfObj in self.parmConfObjList:
347 if self.name == 'run' and parmConfObj.name == 'datatype':
351 if self.name == 'run' and parmConfObj.name == 'datatype':
348 continue
352 continue
349
353
350 kwargs[parmConfObj.name] = parmConfObj.getValue()
354 kwargs[parmConfObj.name] = parmConfObj.getValue()
351
355
352 return kwargs
356 return kwargs
353
357
354 def setup(self, id, name, priority, type):
358 def setup(self, id, name, priority, type):
355
359
356 self.id = str(id)
360 self.id = str(id)
357 self.name = name
361 self.name = name
358 self.type = type
362 self.type = type
359 self.priority = priority
363 self.priority = priority
360
364
361 self.parmConfObjList = []
365 self.parmConfObjList = []
362
366
363 def removeParameters(self):
367 def removeParameters(self):
364
368
365 for obj in self.parmConfObjList:
369 for obj in self.parmConfObjList:
366 del obj
370 del obj
367
371
368 self.parmConfObjList = []
372 self.parmConfObjList = []
369
373
370 def addParameter(self, name, value, format='str'):
374 def addParameter(self, name, value, format='str'):
371
375
372 if value is None:
376 if value is None:
373 return None
377 return None
374 id = self.__getNewId()
378 id = self.__getNewId()
375
379
376 parmConfObj = ParameterConf()
380 parmConfObj = ParameterConf()
377 if not parmConfObj.setup(id, name, value, format):
381 if not parmConfObj.setup(id, name, value, format):
378 return None
382 return None
379
383
380 self.parmConfObjList.append(parmConfObj)
384 self.parmConfObjList.append(parmConfObj)
381
385
382 return parmConfObj
386 return parmConfObj
383
387
384 def changeParameter(self, name, value, format='str'):
388 def changeParameter(self, name, value, format='str'):
385
389
386 parmConfObj = self.getParameterObj(name)
390 parmConfObj = self.getParameterObj(name)
387 parmConfObj.update(name, value, format)
391 parmConfObj.update(name, value, format)
388
392
389 return parmConfObj
393 return parmConfObj
390
394
391 def makeXml(self, procUnitElement):
395 def makeXml(self, procUnitElement):
392
396
393 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
394 opElement.set('id', str(self.id))
398 opElement.set('id', str(self.id))
395 opElement.set('name', self.name)
399 opElement.set('name', self.name)
396 opElement.set('type', self.type)
400 opElement.set('type', self.type)
397 opElement.set('priority', str(self.priority))
401 opElement.set('priority', str(self.priority))
398
402
399 for parmConfObj in self.parmConfObjList:
403 for parmConfObj in self.parmConfObjList:
400 parmConfObj.makeXml(opElement)
404 parmConfObj.makeXml(opElement)
401
405
402 def readXml(self, opElement):
406 def readXml(self, opElement):
403
407
404 self.id = opElement.get('id')
408 self.id = opElement.get('id')
405 self.name = opElement.get('name')
409 self.name = opElement.get('name')
406 self.type = opElement.get('type')
410 self.type = opElement.get('type')
407 self.priority = opElement.get('priority')
411 self.priority = opElement.get('priority')
408
412
409 # Compatible with old signal chain version
413 # Compatible with old signal chain version
410 # Use of 'run' method instead 'init'
414 # Use of 'run' method instead 'init'
411 if self.type == 'self' and self.name == 'init':
415 if self.type == 'self' and self.name == 'init':
412 self.name = 'run'
416 self.name = 'run'
413
417
414 self.parmConfObjList = []
418 self.parmConfObjList = []
415
419
416 parmElementList = opElement.iter(ParameterConf().getElementName())
420 parmElementList = opElement.iter(ParameterConf().getElementName())
417
421
418 for parmElement in parmElementList:
422 for parmElement in parmElementList:
419 parmConfObj = ParameterConf()
423 parmConfObj = ParameterConf()
420 parmConfObj.readXml(parmElement)
424 parmConfObj.readXml(parmElement)
421
425
422 # Compatible with old signal chain version
426 # Compatible with old signal chain version
423 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
424 if self.type != 'self' and self.name == 'Plot':
428 if self.type != 'self' and self.name == 'Plot':
425 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
426 self.name = parmConfObj.value
430 self.name = parmConfObj.value
427 continue
431 continue
428
432
429 self.parmConfObjList.append(parmConfObj)
433 self.parmConfObjList.append(parmConfObj)
430
434
431 def printattr(self):
435 def printattr(self):
432
436
433 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
434 self.id,
438 self.id,
435 self.name,
439 self.name,
436 self.type,
440 self.type,
437 self.priority)
441 self.priority)
438
442
439 for parmConfObj in self.parmConfObjList:
443 for parmConfObj in self.parmConfObjList:
440 parmConfObj.printattr()
444 parmConfObj.printattr()
441
445
442 def createObject(self, plotter_queue=None):
446 def createObject(self, plotter_queue=None):
443
447
444 if self.type == 'self':
448 if self.type == 'self':
445 raise ValueError, 'This operation type cannot be created'
449 raise ValueError, 'This operation type cannot be created'
446
450
447 if self.type == 'plotter':
451 if self.type == 'plotter':
448 if not plotter_queue:
452 if not plotter_queue:
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
450
454
451 opObj = Plotter(self.name, plotter_queue)
455 opObj = Plotter(self.name, plotter_queue)
452
456
453 if self.type == 'external' or self.type == 'other':
457 if self.type == 'external' or self.type == 'other':
454
458
455 className = eval(self.name)
459 className = eval(self.name)
456 kwargs = self.getKwargs()
460 kwargs = self.getKwargs()
457
461
458 opObj = className(**kwargs)
462 opObj = className(**kwargs)
459
463
460 return opObj
464 return opObj
461
465
462
466
463 class ProcUnitConf():
467 class ProcUnitConf():
464
468
465 id = None
469 id = None
466 name = None
470 name = None
467 datatype = None
471 datatype = None
468 inputId = None
472 inputId = None
469 parentId = None
473 parentId = None
470
474
471 opConfObjList = []
475 opConfObjList = []
472
476
473 procUnitObj = None
477 procUnitObj = None
474 opObjList = []
478 opObjList = []
475
479
476 ELEMENTNAME = 'ProcUnit'
480 ELEMENTNAME = 'ProcUnit'
477
481
478 def __init__(self):
482 def __init__(self):
479
483
480 self.id = None
484 self.id = None
481 self.datatype = None
485 self.datatype = None
482 self.name = None
486 self.name = None
483 self.inputId = None
487 self.inputId = None
484
488
485 self.opConfObjList = []
489 self.opConfObjList = []
486
490
487 self.procUnitObj = None
491 self.procUnitObj = None
488 self.opObjDict = {}
492 self.opObjDict = {}
489
493
490 def __getPriority(self):
494 def __getPriority(self):
491
495
492 return len(self.opConfObjList) + 1
496 return len(self.opConfObjList) + 1
493
497
494 def __getNewId(self):
498 def __getNewId(self):
495
499
496 return int(self.id) * 10 + len(self.opConfObjList) + 1
500 return int(self.id) * 10 + len(self.opConfObjList) + 1
497
501
498 def getElementName(self):
502 def getElementName(self):
499
503
500 return self.ELEMENTNAME
504 return self.ELEMENTNAME
501
505
502 def getId(self):
506 def getId(self):
503
507
504 return self.id
508 return self.id
505
509
506 def updateId(self, new_id, parentId=parentId):
510 def updateId(self, new_id, parentId=parentId):
507
511
508 new_id = int(parentId) * 10 + (int(self.id) % 10)
512 new_id = int(parentId) * 10 + (int(self.id) % 10)
509 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
510
514
511 # If this proc unit has not inputs
515 # If this proc unit has not inputs
512 if self.inputId == '0':
516 if self.inputId == '0':
513 new_inputId = 0
517 new_inputId = 0
514
518
515 n = 1
519 n = 1
516 for opConfObj in self.opConfObjList:
520 for opConfObj in self.opConfObjList:
517
521
518 idOp = str(int(new_id) * 10 + n)
522 idOp = str(int(new_id) * 10 + n)
519 opConfObj.updateId(idOp)
523 opConfObj.updateId(idOp)
520
524
521 n += 1
525 n += 1
522
526
523 self.parentId = str(parentId)
527 self.parentId = str(parentId)
524 self.id = str(new_id)
528 self.id = str(new_id)
525 self.inputId = str(new_inputId)
529 self.inputId = str(new_inputId)
526
530
527 def getInputId(self):
531 def getInputId(self):
528
532
529 return self.inputId
533 return self.inputId
530
534
531 def getOperationObjList(self):
535 def getOperationObjList(self):
532
536
533 return self.opConfObjList
537 return self.opConfObjList
534
538
535 def getOperationObj(self, name=None):
539 def getOperationObj(self, name=None):
536
540
537 for opConfObj in self.opConfObjList:
541 for opConfObj in self.opConfObjList:
538
542
539 if opConfObj.name != name:
543 if opConfObj.name != name:
540 continue
544 continue
541
545
542 return opConfObj
546 return opConfObj
543
547
544 return None
548 return None
545
549
546 def getOpObjfromParamValue(self, value=None):
550 def getOpObjfromParamValue(self, value=None):
547
551
548 for opConfObj in self.opConfObjList:
552 for opConfObj in self.opConfObjList:
549 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
550 continue
554 continue
551 return opConfObj
555 return opConfObj
552 return None
556 return None
553
557
554 def getProcUnitObj(self):
558 def getProcUnitObj(self):
555
559
556 return self.procUnitObj
560 return self.procUnitObj
557
561
558 def setup(self, id, name, datatype, inputId, parentId=None):
562 def setup(self, id, name, datatype, inputId, parentId=None):
559
563
560 # Compatible with old signal chain version
564 # Compatible with old signal chain version
561 if datatype == None and name == None:
565 if datatype == None and name == None:
562 raise ValueError, 'datatype or name should be defined'
566 raise ValueError, 'datatype or name should be defined'
563
567
564 if name == None:
568 if name == None:
565 if 'Proc' in datatype:
569 if 'Proc' in datatype:
566 name = datatype
570 name = datatype
567 else:
571 else:
568 name = '%sProc' % (datatype)
572 name = '%sProc' % (datatype)
569
573
570 if datatype == None:
574 if datatype == None:
571 datatype = name.replace('Proc', '')
575 datatype = name.replace('Proc', '')
572
576
573 self.id = str(id)
577 self.id = str(id)
574 self.name = name
578 self.name = name
575 self.datatype = datatype
579 self.datatype = datatype
576 self.inputId = inputId
580 self.inputId = inputId
577 self.parentId = parentId
581 self.parentId = parentId
578
582
579 self.opConfObjList = []
583 self.opConfObjList = []
580
584
581 self.addOperation(name='run', optype='self')
585 self.addOperation(name='run', optype='self')
582
586
583 def removeOperations(self):
587 def removeOperations(self):
584
588
585 for obj in self.opConfObjList:
589 for obj in self.opConfObjList:
586 del obj
590 del obj
587
591
588 self.opConfObjList = []
592 self.opConfObjList = []
589 self.addOperation(name='run')
593 self.addOperation(name='run')
590
594
591 def addParameter(self, **kwargs):
595 def addParameter(self, **kwargs):
592 '''
596 '''
593 Add parameters to 'run' operation
597 Add parameters to 'run' operation
594 '''
598 '''
595 opObj = self.opConfObjList[0]
599 opObj = self.opConfObjList[0]
596
600
597 opObj.addParameter(**kwargs)
601 opObj.addParameter(**kwargs)
598
602
599 return opObj
603 return opObj
600
604
601 def addOperation(self, name, optype='self'):
605 def addOperation(self, name, optype='self'):
602
606
603 id = self.__getNewId()
607 id = self.__getNewId()
604 priority = self.__getPriority()
608 priority = self.__getPriority()
605
609
606 opConfObj = OperationConf()
610 opConfObj = OperationConf()
607 opConfObj.setup(id, name=name, priority=priority, type=optype)
611 opConfObj.setup(id, name=name, priority=priority, type=optype)
608
612
609 self.opConfObjList.append(opConfObj)
613 self.opConfObjList.append(opConfObj)
610
614
611 return opConfObj
615 return opConfObj
612
616
613 def makeXml(self, projectElement):
617 def makeXml(self, projectElement):
614
618
615 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
616 procUnitElement.set('id', str(self.id))
620 procUnitElement.set('id', str(self.id))
617 procUnitElement.set('name', self.name)
621 procUnitElement.set('name', self.name)
618 procUnitElement.set('datatype', self.datatype)
622 procUnitElement.set('datatype', self.datatype)
619 procUnitElement.set('inputId', str(self.inputId))
623 procUnitElement.set('inputId', str(self.inputId))
620
624
621 for opConfObj in self.opConfObjList:
625 for opConfObj in self.opConfObjList:
622 opConfObj.makeXml(procUnitElement)
626 opConfObj.makeXml(procUnitElement)
623
627
624 def readXml(self, upElement):
628 def readXml(self, upElement):
625
629
626 self.id = upElement.get('id')
630 self.id = upElement.get('id')
627 self.name = upElement.get('name')
631 self.name = upElement.get('name')
628 self.datatype = upElement.get('datatype')
632 self.datatype = upElement.get('datatype')
629 self.inputId = upElement.get('inputId')
633 self.inputId = upElement.get('inputId')
630
634
631 if self.ELEMENTNAME == 'ReadUnit':
635 if self.ELEMENTNAME == 'ReadUnit':
632 self.datatype = self.datatype.replace('Reader', '')
636 self.datatype = self.datatype.replace('Reader', '')
633
637
634 if self.ELEMENTNAME == 'ProcUnit':
638 if self.ELEMENTNAME == 'ProcUnit':
635 self.datatype = self.datatype.replace('Proc', '')
639 self.datatype = self.datatype.replace('Proc', '')
636
640
637 if self.inputId == 'None':
641 if self.inputId == 'None':
638 self.inputId = '0'
642 self.inputId = '0'
639
643
640 self.opConfObjList = []
644 self.opConfObjList = []
641
645
642 opElementList = upElement.iter(OperationConf().getElementName())
646 opElementList = upElement.iter(OperationConf().getElementName())
643
647
644 for opElement in opElementList:
648 for opElement in opElementList:
645 opConfObj = OperationConf()
649 opConfObj = OperationConf()
646 opConfObj.readXml(opElement)
650 opConfObj.readXml(opElement)
647 self.opConfObjList.append(opConfObj)
651 self.opConfObjList.append(opConfObj)
648
652
649 def printattr(self):
653 def printattr(self):
650
654
651 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
652 self.id,
656 self.id,
653 self.name,
657 self.name,
654 self.datatype,
658 self.datatype,
655 self.inputId)
659 self.inputId)
656
660
657 for opConfObj in self.opConfObjList:
661 for opConfObj in self.opConfObjList:
658 opConfObj.printattr()
662 opConfObj.printattr()
659
663
660 def getKwargs(self):
664 def getKwargs(self):
661
665
662 opObj = self.opConfObjList[0]
666 opObj = self.opConfObjList[0]
663 kwargs = opObj.getKwargs()
667 kwargs = opObj.getKwargs()
664
668
665 return kwargs
669 return kwargs
666
670
667 def createObjects(self, plotter_queue=None):
671 def createObjects(self, plotter_queue=None):
668
672
669 className = eval(self.name)
673 className = eval(self.name)
670 kwargs = self.getKwargs()
674 kwargs = self.getKwargs()
671 procUnitObj = className(**kwargs)
675 procUnitObj = className(**kwargs)
672
676
673 for opConfObj in self.opConfObjList:
677 for opConfObj in self.opConfObjList:
674
678
675 if opConfObj.type == 'self' and self.name == 'run':
679 if opConfObj.type == 'self' and self.name == 'run':
676 continue
680 continue
677 elif opConfObj.type == 'self':
681 elif opConfObj.type == 'self':
678 procUnitObj.addOperationKwargs(
682 procUnitObj.addOperationKwargs(
679 opConfObj.id, **opConfObj.getKwargs())
683 opConfObj.id, **opConfObj.getKwargs())
680 continue
684 continue
681
685
682 opObj = opConfObj.createObject(plotter_queue)
686 opObj = opConfObj.createObject(plotter_queue)
683
687
684 self.opObjDict[opConfObj.id] = opObj
688 self.opObjDict[opConfObj.id] = opObj
685
689
686 procUnitObj.addOperation(opObj, opConfObj.id)
690 procUnitObj.addOperation(opObj, opConfObj.id)
687
691
688 self.procUnitObj = procUnitObj
692 self.procUnitObj = procUnitObj
689
693
690 return procUnitObj
694 return procUnitObj
691
695
692 def run(self):
696 def run(self):
693
697
694 is_ok = False
698 is_ok = False
695
699
696 for opConfObj in self.opConfObjList:
700 for opConfObj in self.opConfObjList:
697
701
698 kwargs = {}
702 kwargs = {}
699 for parmConfObj in opConfObj.getParameterObjList():
703 for parmConfObj in opConfObj.getParameterObjList():
700 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
701 continue
705 continue
702
706
703 kwargs[parmConfObj.name] = parmConfObj.getValue()
707 kwargs[parmConfObj.name] = parmConfObj.getValue()
704
708
705 sts = self.procUnitObj.call(opType=opConfObj.type,
709 sts = self.procUnitObj.call(opType=opConfObj.type,
706 opName=opConfObj.name,
710 opName=opConfObj.name,
707 opId=opConfObj.id)
711 opId=opConfObj.id)
708
712
709 is_ok = is_ok or sts
713 is_ok = is_ok or sts
710
714
711 return is_ok
715 return is_ok
712
716
713 def close(self):
717 def close(self):
714
718
715 for opConfObj in self.opConfObjList:
719 for opConfObj in self.opConfObjList:
716 if opConfObj.type == 'self':
720 if opConfObj.type == 'self':
717 continue
721 continue
718
722
719 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
720 opObj.close()
724 opObj.close()
721
725
722 self.procUnitObj.close()
726 self.procUnitObj.close()
723
727
724 return
728 return
725
729
726
730
727 class ReadUnitConf(ProcUnitConf):
731 class ReadUnitConf(ProcUnitConf):
728
732
729 path = None
733 path = None
730 startDate = None
734 startDate = None
731 endDate = None
735 endDate = None
732 startTime = None
736 startTime = None
733 endTime = None
737 endTime = None
734
738
735 ELEMENTNAME = 'ReadUnit'
739 ELEMENTNAME = 'ReadUnit'
736
740
737 def __init__(self):
741 def __init__(self):
738
742
739 self.id = None
743 self.id = None
740 self.datatype = None
744 self.datatype = None
741 self.name = None
745 self.name = None
742 self.inputId = None
746 self.inputId = None
743
747
744 self.parentId = None
748 self.parentId = None
745
749
746 self.opConfObjList = []
750 self.opConfObjList = []
747 self.opObjList = []
751 self.opObjList = []
748
752
749 def getElementName(self):
753 def getElementName(self):
750
754
751 return self.ELEMENTNAME
755 return self.ELEMENTNAME
752
756
753 def setup(self, id, name, datatype, path='', startDate='', endDate='',
757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
754 startTime='', endTime='', parentId=None, server=None, **kwargs):
758 startTime='', endTime='', parentId=None, server=None, **kwargs):
755
759
756 # Compatible with old signal chain version
760 # Compatible with old signal chain version
757 if datatype == None and name == None:
761 if datatype == None and name == None:
758 raise ValueError, 'datatype or name should be defined'
762 raise ValueError, 'datatype or name should be defined'
759 if name == None:
763 if name == None:
760 if 'Reader' in datatype:
764 if 'Reader' in datatype:
761 name = datatype
765 name = datatype
762 datatype = name.replace('Reader','')
766 datatype = name.replace('Reader','')
763 else:
767 else:
764 name = '{}Reader'.format(datatype)
768 name = '{}Reader'.format(datatype)
765 if datatype == None:
769 if datatype == None:
766 if 'Reader' in name:
770 if 'Reader' in name:
767 datatype = name.replace('Reader','')
771 datatype = name.replace('Reader','')
768 else:
772 else:
769 datatype = name
773 datatype = name
770 name = '{}Reader'.format(name)
774 name = '{}Reader'.format(name)
771
775
772 self.id = id
776 self.id = id
773 self.name = name
777 self.name = name
774 self.datatype = datatype
778 self.datatype = datatype
775 if path != '':
779 if path != '':
776 self.path = os.path.abspath(path)
780 self.path = os.path.abspath(path)
777 self.startDate = startDate
781 self.startDate = startDate
778 self.endDate = endDate
782 self.endDate = endDate
779 self.startTime = startTime
783 self.startTime = startTime
780 self.endTime = endTime
784 self.endTime = endTime
781 self.inputId = '0'
785 self.inputId = '0'
782 self.parentId = parentId
786 self.parentId = parentId
783 self.server = server
787 self.server = server
784 self.addRunOperation(**kwargs)
788 self.addRunOperation(**kwargs)
785
789
786 def update(self, **kwargs):
790 def update(self, **kwargs):
787
791
788 if 'datatype' in kwargs:
792 if 'datatype' in kwargs:
789 datatype = kwargs.pop('datatype')
793 datatype = kwargs.pop('datatype')
790 if 'Reader' in datatype:
794 if 'Reader' in datatype:
791 self.name = datatype
795 self.name = datatype
792 else:
796 else:
793 self.name = '%sReader' % (datatype)
797 self.name = '%sReader' % (datatype)
794 self.datatype = self.name.replace('Reader', '')
798 self.datatype = self.name.replace('Reader', '')
795
799
796 attrs = ('path', 'startDate', 'endDate',
800 attrs = ('path', 'startDate', 'endDate',
797 'startTime', 'endTime', 'parentId')
801 'startTime', 'endTime', 'parentId')
798
802
799 for attr in attrs:
803 for attr in attrs:
800 if attr in kwargs:
804 if attr in kwargs:
801 setattr(self, attr, kwargs.pop(attr))
805 setattr(self, attr, kwargs.pop(attr))
802
806
803 self.inputId = '0'
807 self.inputId = '0'
804 self.updateRunOperation(**kwargs)
808 self.updateRunOperation(**kwargs)
805
809
806 def removeOperations(self):
810 def removeOperations(self):
807
811
808 for obj in self.opConfObjList:
812 for obj in self.opConfObjList:
809 del obj
813 del obj
810
814
811 self.opConfObjList = []
815 self.opConfObjList = []
812
816
813 def addRunOperation(self, **kwargs):
817 def addRunOperation(self, **kwargs):
814
818
815 opObj = self.addOperation(name='run', optype='self')
819 opObj = self.addOperation(name='run', optype='self')
816
820
817 if self.server is None:
821 if self.server is None:
818 opObj.addParameter(
822 opObj.addParameter(
819 name='datatype', value=self.datatype, format='str')
823 name='datatype', value=self.datatype, format='str')
820 opObj.addParameter(name='path', value=self.path, format='str')
824 opObj.addParameter(name='path', value=self.path, format='str')
821 opObj.addParameter(
825 opObj.addParameter(
822 name='startDate', value=self.startDate, format='date')
826 name='startDate', value=self.startDate, format='date')
823 opObj.addParameter(
827 opObj.addParameter(
824 name='endDate', value=self.endDate, format='date')
828 name='endDate', value=self.endDate, format='date')
825 opObj.addParameter(
829 opObj.addParameter(
826 name='startTime', value=self.startTime, format='time')
830 name='startTime', value=self.startTime, format='time')
827 opObj.addParameter(
831 opObj.addParameter(
828 name='endTime', value=self.endTime, format='time')
832 name='endTime', value=self.endTime, format='time')
829
833
830 for key, value in kwargs.items():
834 for key, value in kwargs.items():
831 opObj.addParameter(name=key, value=value,
835 opObj.addParameter(name=key, value=value,
832 format=type(value).__name__)
836 format=type(value).__name__)
833 else:
837 else:
834 opObj.addParameter(name='server', value=self.server, format='str')
838 opObj.addParameter(name='server', value=self.server, format='str')
835
839
836 return opObj
840 return opObj
837
841
838 def updateRunOperation(self, **kwargs):
842 def updateRunOperation(self, **kwargs):
839
843
840 opObj = self.getOperationObj(name='run')
844 opObj = self.getOperationObj(name='run')
841 opObj.removeParameters()
845 opObj.removeParameters()
842
846
843 opObj.addParameter(name='datatype', value=self.datatype, format='str')
847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
844 opObj.addParameter(name='path', value=self.path, format='str')
848 opObj.addParameter(name='path', value=self.path, format='str')
845 opObj.addParameter(
849 opObj.addParameter(
846 name='startDate', value=self.startDate, format='date')
850 name='startDate', value=self.startDate, format='date')
847 opObj.addParameter(name='endDate', value=self.endDate, format='date')
851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
848 opObj.addParameter(
852 opObj.addParameter(
849 name='startTime', value=self.startTime, format='time')
853 name='startTime', value=self.startTime, format='time')
850 opObj.addParameter(name='endTime', value=self.endTime, format='time')
854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
851
855
852 for key, value in kwargs.items():
856 for key, value in kwargs.items():
853 opObj.addParameter(name=key, value=value,
857 opObj.addParameter(name=key, value=value,
854 format=type(value).__name__)
858 format=type(value).__name__)
855
859
856 return opObj
860 return opObj
857
861
858 def readXml(self, upElement):
862 def readXml(self, upElement):
859
863
860 self.id = upElement.get('id')
864 self.id = upElement.get('id')
861 self.name = upElement.get('name')
865 self.name = upElement.get('name')
862 self.datatype = upElement.get('datatype')
866 self.datatype = upElement.get('datatype')
863 self.inputId = upElement.get('inputId')
867 self.inputId = upElement.get('inputId')
864
868
865 if self.ELEMENTNAME == 'ReadUnit':
869 if self.ELEMENTNAME == 'ReadUnit':
866 self.datatype = self.datatype.replace('Reader', '')
870 self.datatype = self.datatype.replace('Reader', '')
867
871
868 if self.inputId == 'None':
872 if self.inputId == 'None':
869 self.inputId = '0'
873 self.inputId = '0'
870
874
871 self.opConfObjList = []
875 self.opConfObjList = []
872
876
873 opElementList = upElement.iter(OperationConf().getElementName())
877 opElementList = upElement.iter(OperationConf().getElementName())
874
878
875 for opElement in opElementList:
879 for opElement in opElementList:
876 opConfObj = OperationConf()
880 opConfObj = OperationConf()
877 opConfObj.readXml(opElement)
881 opConfObj.readXml(opElement)
878 self.opConfObjList.append(opConfObj)
882 self.opConfObjList.append(opConfObj)
879
883
880 if opConfObj.name == 'run':
884 if opConfObj.name == 'run':
881 self.path = opConfObj.getParameterValue('path')
885 self.path = opConfObj.getParameterValue('path')
882 self.startDate = opConfObj.getParameterValue('startDate')
886 self.startDate = opConfObj.getParameterValue('startDate')
883 self.endDate = opConfObj.getParameterValue('endDate')
887 self.endDate = opConfObj.getParameterValue('endDate')
884 self.startTime = opConfObj.getParameterValue('startTime')
888 self.startTime = opConfObj.getParameterValue('startTime')
885 self.endTime = opConfObj.getParameterValue('endTime')
889 self.endTime = opConfObj.getParameterValue('endTime')
886
890
887
891
888 class Project(Process):
892 class Project(Process):
889
893
890 id = None
894 id = None
891 # name = None
895 # name = None
892 description = None
896 description = None
893 filename = None
897 filename = None
894
898
895 procUnitConfObjDict = None
899 procUnitConfObjDict = None
896
900
897 ELEMENTNAME = 'Project'
901 ELEMENTNAME = 'Project'
898
902
899 plotterQueue = None
903 plotterQueue = None
900
904
901 def __init__(self, plotter_queue=None):
905 def __init__(self, plotter_queue=None):
902
906
903 Process.__init__(self)
907 Process.__init__(self)
904 self.id = None
908 self.id = None
905 # self.name = None
909 # self.name = None
906 self.description = None
910 self.description = None
907
911
908 self.plotterQueue = plotter_queue
912 self.plotterQueue = plotter_queue
909
913
910 self.procUnitConfObjDict = {}
914 self.procUnitConfObjDict = {}
911
915
912 def __getNewId(self):
916 def __getNewId(self):
913
917
914 idList = self.procUnitConfObjDict.keys()
918 idList = self.procUnitConfObjDict.keys()
915
919
916 id = int(self.id) * 10
920 id = int(self.id) * 10
917
921
918 while True:
922 while True:
919 id += 1
923 id += 1
920
924
921 if str(id) in idList:
925 if str(id) in idList:
922 continue
926 continue
923
927
924 break
928 break
925
929
926 return str(id)
930 return str(id)
927
931
928 def getElementName(self):
932 def getElementName(self):
929
933
930 return self.ELEMENTNAME
934 return self.ELEMENTNAME
931
935
932 def getId(self):
936 def getId(self):
933
937
934 return self.id
938 return self.id
935
939
936 def updateId(self, new_id):
940 def updateId(self, new_id):
937
941
938 self.id = str(new_id)
942 self.id = str(new_id)
939
943
940 keyList = self.procUnitConfObjDict.keys()
944 keyList = self.procUnitConfObjDict.keys()
941 keyList.sort()
945 keyList.sort()
942
946
943 n = 1
947 n = 1
944 newProcUnitConfObjDict = {}
948 newProcUnitConfObjDict = {}
945
949
946 for procKey in keyList:
950 for procKey in keyList:
947
951
948 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 procUnitConfObj = self.procUnitConfObjDict[procKey]
949 idProcUnit = str(int(self.id) * 10 + n)
953 idProcUnit = str(int(self.id) * 10 + n)
950 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
951
955
952 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
956 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
953 n += 1
957 n += 1
954
958
955 self.procUnitConfObjDict = newProcUnitConfObjDict
959 self.procUnitConfObjDict = newProcUnitConfObjDict
956
960
957 def setup(self, id, name='', description=''):
961 def setup(self, id, name='', description=''):
958
962
959 print
963 print
960 print '*' * 60
964 print '*' * 60
961 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
965 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
962 print '*' * 60
966 print '*' * 60
963 print
967 print
964 self.id = str(id)
968 self.id = str(id)
965 self.description = description
969 self.description = description
966
970
967 def update(self, name, description):
971 def update(self, name, description):
968
972
969 self.description = description
973 self.description = description
970
974
971 def clone(self):
975 def clone(self):
972
976
973 p = Project()
977 p = Project()
974 p.procUnitConfObjDict = self.procUnitConfObjDict
978 p.procUnitConfObjDict = self.procUnitConfObjDict
975 return p
979 return p
976
980
977 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
978
982
979 if id is None:
983 if id is None:
980 idReadUnit = self.__getNewId()
984 idReadUnit = self.__getNewId()
981 else:
985 else:
982 idReadUnit = str(id)
986 idReadUnit = str(id)
983
987
984 readUnitConfObj = ReadUnitConf()
988 readUnitConfObj = ReadUnitConf()
985 readUnitConfObj.setup(idReadUnit, name, datatype,
989 readUnitConfObj.setup(idReadUnit, name, datatype,
986 parentId=self.id, **kwargs)
990 parentId=self.id, **kwargs)
987
991
988 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
992 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
989
993
990 return readUnitConfObj
994 return readUnitConfObj
991
995
992 def addProcUnit(self, inputId='0', datatype=None, name=None):
996 def addProcUnit(self, inputId='0', datatype=None, name=None):
993
997
994 idProcUnit = self.__getNewId()
998 idProcUnit = self.__getNewId()
995
999
996 procUnitConfObj = ProcUnitConf()
1000 procUnitConfObj = ProcUnitConf()
997 procUnitConfObj.setup(idProcUnit, name, datatype,
1001 procUnitConfObj.setup(idProcUnit, name, datatype,
998 inputId, parentId=self.id)
1002 inputId, parentId=self.id)
999
1003
1000 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1004 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1001
1005
1002 return procUnitConfObj
1006 return procUnitConfObj
1003
1007
1004 def removeProcUnit(self, id):
1008 def removeProcUnit(self, id):
1005
1009
1006 if id in self.procUnitConfObjDict.keys():
1010 if id in self.procUnitConfObjDict.keys():
1007 self.procUnitConfObjDict.pop(id)
1011 self.procUnitConfObjDict.pop(id)
1008
1012
1009 def getReadUnitId(self):
1013 def getReadUnitId(self):
1010
1014
1011 readUnitConfObj = self.getReadUnitObj()
1015 readUnitConfObj = self.getReadUnitObj()
1012
1016
1013 return readUnitConfObj.id
1017 return readUnitConfObj.id
1014
1018
1015 def getReadUnitObj(self):
1019 def getReadUnitObj(self):
1016
1020
1017 for obj in self.procUnitConfObjDict.values():
1021 for obj in self.procUnitConfObjDict.values():
1018 if obj.getElementName() == 'ReadUnit':
1022 if obj.getElementName() == 'ReadUnit':
1019 return obj
1023 return obj
1020
1024
1021 return None
1025 return None
1022
1026
1023 def getProcUnitObj(self, id=None, name=None):
1027 def getProcUnitObj(self, id=None, name=None):
1024
1028
1025 if id != None:
1029 if id != None:
1026 return self.procUnitConfObjDict[id]
1030 return self.procUnitConfObjDict[id]
1027
1031
1028 if name != None:
1032 if name != None:
1029 return self.getProcUnitObjByName(name)
1033 return self.getProcUnitObjByName(name)
1030
1034
1031 return None
1035 return None
1032
1036
1033 def getProcUnitObjByName(self, name):
1037 def getProcUnitObjByName(self, name):
1034
1038
1035 for obj in self.procUnitConfObjDict.values():
1039 for obj in self.procUnitConfObjDict.values():
1036 if obj.name == name:
1040 if obj.name == name:
1037 return obj
1041 return obj
1038
1042
1039 return None
1043 return None
1040
1044
1041 def procUnitItems(self):
1045 def procUnitItems(self):
1042
1046
1043 return self.procUnitConfObjDict.items()
1047 return self.procUnitConfObjDict.items()
1044
1048
1045 def makeXml(self):
1049 def makeXml(self):
1046
1050
1047 projectElement = Element('Project')
1051 projectElement = Element('Project')
1048 projectElement.set('id', str(self.id))
1052 projectElement.set('id', str(self.id))
1049 projectElement.set('name', self.name)
1053 projectElement.set('name', self.name)
1050 projectElement.set('description', self.description)
1054 projectElement.set('description', self.description)
1051
1055
1052 for procUnitConfObj in self.procUnitConfObjDict.values():
1056 for procUnitConfObj in self.procUnitConfObjDict.values():
1053 procUnitConfObj.makeXml(projectElement)
1057 procUnitConfObj.makeXml(projectElement)
1054
1058
1055 self.projectElement = projectElement
1059 self.projectElement = projectElement
1056
1060
1057 def writeXml(self, filename=None):
1061 def writeXml(self, filename=None):
1058
1062
1059 if filename == None:
1063 if filename == None:
1060 if self.filename:
1064 if self.filename:
1061 filename = self.filename
1065 filename = self.filename
1062 else:
1066 else:
1063 filename = 'schain.xml'
1067 filename = 'schain.xml'
1064
1068
1065 if not filename:
1069 if not filename:
1066 print 'filename has not been defined. Use setFilename(filename) for do it.'
1070 print 'filename has not been defined. Use setFilename(filename) for do it.'
1067 return 0
1071 return 0
1068
1072
1069 abs_file = os.path.abspath(filename)
1073 abs_file = os.path.abspath(filename)
1070
1074
1071 if not os.access(os.path.dirname(abs_file), os.W_OK):
1075 if not os.access(os.path.dirname(abs_file), os.W_OK):
1072 print 'No write permission on %s' % os.path.dirname(abs_file)
1076 print 'No write permission on %s' % os.path.dirname(abs_file)
1073 return 0
1077 return 0
1074
1078
1075 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1079 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1076 print 'File %s already exists and it could not be overwriten' % abs_file
1080 print 'File %s already exists and it could not be overwriten' % abs_file
1077 return 0
1081 return 0
1078
1082
1079 self.makeXml()
1083 self.makeXml()
1080
1084
1081 ElementTree(self.projectElement).write(abs_file, method='xml')
1085 ElementTree(self.projectElement).write(abs_file, method='xml')
1082
1086
1083 self.filename = abs_file
1087 self.filename = abs_file
1084
1088
1085 return 1
1089 return 1
1086
1090
1087 def readXml(self, filename=None):
1091 def readXml(self, filename=None):
1088
1092
1089 if not filename:
1093 if not filename:
1090 print 'filename is not defined'
1094 print 'filename is not defined'
1091 return 0
1095 return 0
1092
1096
1093 abs_file = os.path.abspath(filename)
1097 abs_file = os.path.abspath(filename)
1094
1098
1095 if not os.path.isfile(abs_file):
1099 if not os.path.isfile(abs_file):
1096 print '%s file does not exist' % abs_file
1100 print '%s file does not exist' % abs_file
1097 return 0
1101 return 0
1098
1102
1099 self.projectElement = None
1103 self.projectElement = None
1100 self.procUnitConfObjDict = {}
1104 self.procUnitConfObjDict = {}
1101
1105
1102 try:
1106 try:
1103 self.projectElement = ElementTree().parse(abs_file)
1107 self.projectElement = ElementTree().parse(abs_file)
1104 except:
1108 except:
1105 print 'Error reading %s, verify file format' % filename
1109 print 'Error reading %s, verify file format' % filename
1106 return 0
1110 return 0
1107
1111
1108 self.project = self.projectElement.tag
1112 self.project = self.projectElement.tag
1109
1113
1110 self.id = self.projectElement.get('id')
1114 self.id = self.projectElement.get('id')
1111 self.name = self.projectElement.get('name')
1115 self.name = self.projectElement.get('name')
1112 self.description = self.projectElement.get('description')
1116 self.description = self.projectElement.get('description')
1113
1117
1114 readUnitElementList = self.projectElement.iter(
1118 readUnitElementList = self.projectElement.iter(
1115 ReadUnitConf().getElementName())
1119 ReadUnitConf().getElementName())
1116
1120
1117 for readUnitElement in readUnitElementList:
1121 for readUnitElement in readUnitElementList:
1118 readUnitConfObj = ReadUnitConf()
1122 readUnitConfObj = ReadUnitConf()
1119 readUnitConfObj.readXml(readUnitElement)
1123 readUnitConfObj.readXml(readUnitElement)
1120
1124
1121 if readUnitConfObj.parentId == None:
1125 if readUnitConfObj.parentId == None:
1122 readUnitConfObj.parentId = self.id
1126 readUnitConfObj.parentId = self.id
1123
1127
1124 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1128 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1125
1129
1126 procUnitElementList = self.projectElement.iter(
1130 procUnitElementList = self.projectElement.iter(
1127 ProcUnitConf().getElementName())
1131 ProcUnitConf().getElementName())
1128
1132
1129 for procUnitElement in procUnitElementList:
1133 for procUnitElement in procUnitElementList:
1130 procUnitConfObj = ProcUnitConf()
1134 procUnitConfObj = ProcUnitConf()
1131 procUnitConfObj.readXml(procUnitElement)
1135 procUnitConfObj.readXml(procUnitElement)
1132
1136
1133 if procUnitConfObj.parentId == None:
1137 if procUnitConfObj.parentId == None:
1134 procUnitConfObj.parentId = self.id
1138 procUnitConfObj.parentId = self.id
1135
1139
1136 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1137
1141
1138 self.filename = abs_file
1142 self.filename = abs_file
1139
1143
1140 return 1
1144 return 1
1141
1145
1142 def printattr(self):
1146 def printattr(self):
1143
1147
1144 print 'Project[%s]: name = %s, description = %s' % (self.id,
1148 print 'Project[%s]: name = %s, description = %s' % (self.id,
1145 self.name,
1149 self.name,
1146 self.description)
1150 self.description)
1147
1151
1148 for procUnitConfObj in self.procUnitConfObjDict.values():
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1149 procUnitConfObj.printattr()
1153 procUnitConfObj.printattr()
1150
1154
1151 def createObjects(self):
1155 def createObjects(self):
1152
1156
1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 procUnitConfObj.createObjects(self.plotterQueue)
1158 procUnitConfObj.createObjects(self.plotterQueue)
1155
1159
1156 def __connect(self, objIN, thisObj):
1160 def __connect(self, objIN, thisObj):
1157
1161
1158 thisObj.setInput(objIN.getOutputObj())
1162 thisObj.setInput(objIN.getOutputObj())
1159
1163
1160 def connectObjects(self):
1164 def connectObjects(self):
1161
1165
1162 for thisPUConfObj in self.procUnitConfObjDict.values():
1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1163
1167
1164 inputId = thisPUConfObj.getInputId()
1168 inputId = thisPUConfObj.getInputId()
1165
1169
1166 if int(inputId) == 0:
1170 if int(inputId) == 0:
1167 continue
1171 continue
1168
1172
1169 # Get input object
1173 # Get input object
1170 puConfINObj = self.procUnitConfObjDict[inputId]
1174 puConfINObj = self.procUnitConfObjDict[inputId]
1171 puObjIN = puConfINObj.getProcUnitObj()
1175 puObjIN = puConfINObj.getProcUnitObj()
1172
1176
1173 # Get current object
1177 # Get current object
1174 thisPUObj = thisPUConfObj.getProcUnitObj()
1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1175
1179
1176 self.__connect(puObjIN, thisPUObj)
1180 self.__connect(puObjIN, thisPUObj)
1177
1181
1178 def __handleError(self, procUnitConfObj, send_email=False):
1182 def __handleError(self, procUnitConfObj, send_email=False):
1179
1183
1180 import socket
1184 import socket
1181
1185
1182 err = traceback.format_exception(sys.exc_info()[0],
1186 err = traceback.format_exception(sys.exc_info()[0],
1183 sys.exc_info()[1],
1187 sys.exc_info()[1],
1184 sys.exc_info()[2])
1188 sys.exc_info()[2])
1185
1189
1186 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1190 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1187 print '***** %s' % err[-1]
1191 print '***** %s' % err[-1]
1188
1192
1189 message = ''.join(err)
1193 message = ''.join(err)
1190
1194
1191 sys.stderr.write(message)
1195 sys.stderr.write(message)
1192
1196
1193 if not send_email:
1197 if not send_email:
1194 return
1198 return
1195
1199
1196 subject = 'SChain v%s: Error running %s\n' % (
1200 subject = 'SChain v%s: Error running %s\n' % (
1197 schainpy.__version__, procUnitConfObj.name)
1201 schainpy.__version__, procUnitConfObj.name)
1198
1202
1199 subtitle = '%s: %s\n' % (
1203 subtitle = '%s: %s\n' % (
1200 procUnitConfObj.getElementName(), procUnitConfObj.name)
1204 procUnitConfObj.getElementName(), procUnitConfObj.name)
1201 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1205 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1202 socket.gethostname())
1206 socket.gethostname())
1203 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1207 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1204 subtitle += 'Configuration file: %s\n' % self.filename
1208 subtitle += 'Configuration file: %s\n' % self.filename
1205 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1209 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1206
1210
1207 readUnitConfObj = self.getReadUnitObj()
1211 readUnitConfObj = self.getReadUnitObj()
1208 if readUnitConfObj:
1212 if readUnitConfObj:
1209 subtitle += '\nInput parameters:\n'
1213 subtitle += '\nInput parameters:\n'
1210 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1214 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1211 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1215 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1212 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1216 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1213 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1217 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1214 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1218 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1215 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1219 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1216
1220
1217 adminObj = schainpy.admin.SchainNotify()
1221 adminObj = schainpy.admin.SchainNotify()
1218 adminObj.sendAlert(message=message,
1222 adminObj.sendAlert(message=message,
1219 subject=subject,
1223 subject=subject,
1220 subtitle=subtitle,
1224 subtitle=subtitle,
1221 filename=self.filename)
1225 filename=self.filename)
1222
1226
1223 def isPaused(self):
1227 def isPaused(self):
1224 return 0
1228 return 0
1225
1229
1226 def isStopped(self):
1230 def isStopped(self):
1227 return 0
1231 return 0
1228
1232
1229 def runController(self):
1233 def runController(self):
1230 '''
1234 '''
1231 returns 0 when this process has been stopped, 1 otherwise
1235 returns 0 when this process has been stopped, 1 otherwise
1232 '''
1236 '''
1233
1237
1234 if self.isPaused():
1238 if self.isPaused():
1235 print 'Process suspended'
1239 print 'Process suspended'
1236
1240
1237 while True:
1241 while True:
1238 time.sleep(0.1)
1242 time.sleep(0.1)
1239
1243
1240 if not self.isPaused():
1244 if not self.isPaused():
1241 break
1245 break
1242
1246
1243 if self.isStopped():
1247 if self.isStopped():
1244 break
1248 break
1245
1249
1246 print 'Process reinitialized'
1250 print 'Process reinitialized'
1247
1251
1248 if self.isStopped():
1252 if self.isStopped():
1249 print 'Process stopped'
1253 print 'Process stopped'
1250 return 0
1254 return 0
1251
1255
1252 return 1
1256 return 1
1253
1257
1254 def setFilename(self, filename):
1258 def setFilename(self, filename):
1255
1259
1256 self.filename = filename
1260 self.filename = filename
1257
1261
1258 def setPlotterQueue(self, plotter_queue):
1262 def setPlotterQueue(self, plotter_queue):
1259
1263
1260 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1261
1265
1262 def getPlotterQueue(self):
1266 def getPlotterQueue(self):
1263
1267
1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1268 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1265
1269
1266 def useExternalPlotter(self):
1270 def useExternalPlotter(self):
1267
1271
1268 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1272 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1269
1273
1270 def run(self):
1274 def run(self):
1271
1275
1272 log.success('Starting {}'.format(self.name))
1276 log.success('Starting {}'.format(self.name))
1273
1277 self.start_time = time.time()
1274 self.createObjects()
1278 self.createObjects()
1275 self.connectObjects()
1279 self.connectObjects()
1276
1280
1277 keyList = self.procUnitConfObjDict.keys()
1281 keyList = self.procUnitConfObjDict.keys()
1278 keyList.sort()
1282 keyList.sort()
1279
1283
1280 while(True):
1284 while(True):
1281
1285
1282 is_ok = False
1286 is_ok = False
1283
1287
1284 for procKey in keyList:
1288 for procKey in keyList:
1285
1289
1286 procUnitConfObj = self.procUnitConfObjDict[procKey]
1290 procUnitConfObj = self.procUnitConfObjDict[procKey]
1287
1291
1288 try:
1292 try:
1289 sts = procUnitConfObj.run()
1293 sts = procUnitConfObj.run()
1290 is_ok = is_ok or sts
1294 is_ok = is_ok or sts
1291 except KeyboardInterrupt:
1295 except KeyboardInterrupt:
1292 is_ok = False
1296 is_ok = False
1293 break
1297 break
1294 except ValueError, e:
1298 except ValueError, e:
1295 time.sleep(0.5)
1299 time.sleep(0.5)
1296 self.__handleError(procUnitConfObj, send_email=True)
1300 self.__handleError(procUnitConfObj, send_email=True)
1297 is_ok = False
1301 is_ok = False
1298 break
1302 break
1299 except:
1303 except:
1300 time.sleep(0.5)
1304 time.sleep(0.5)
1301 self.__handleError(procUnitConfObj)
1305 self.__handleError(procUnitConfObj)
1302 is_ok = False
1306 is_ok = False
1303 break
1307 break
1304
1308
1305 # If every process unit finished so end process
1309 # If every process unit finished so end process
1306 if not(is_ok):
1310 if not(is_ok):
1307 break
1311 break
1308
1312
1309 if not self.runController():
1313 if not self.runController():
1310 break
1314 break
1311
1315
1312 # Closing every process
1316 # Closing every process
1313 for procKey in keyList:
1317 for procKey in keyList:
1314 procUnitConfObj = self.procUnitConfObjDict[procKey]
1318 procUnitConfObj = self.procUnitConfObjDict[procKey]
1315 procUnitConfObj.close()
1319 procUnitConfObj.close()
1316
1320
1317 log.success('{} finished'.format(self.name))
1321 log.success('{} finished (time: {}s)'.format(
1322 self.name,
1323 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,876 +1,876
1 '''
1 '''
2
2
3 $Author: murco $
3 $Author: murco $
4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 '''
5 '''
6 import sys
6 import sys
7 import numpy
7 import numpy
8 import copy
8 import copy
9 import datetime
9 import datetime
10 import inspect
10 import inspect
11
11
12 SPEED_OF_LIGHT = 299792458
12 SPEED_OF_LIGHT = 299792458
13 SPEED_OF_LIGHT = 3e8
13 SPEED_OF_LIGHT = 3e8
14
14
15 BASIC_STRUCTURE = numpy.dtype([
15 BASIC_STRUCTURE = numpy.dtype([
16 ('nSize','<u4'),
16 ('nSize','<u4'),
17 ('nVersion','<u2'),
17 ('nVersion','<u2'),
18 ('nDataBlockId','<u4'),
18 ('nDataBlockId','<u4'),
19 ('nUtime','<u4'),
19 ('nUtime','<u4'),
20 ('nMilsec','<u2'),
20 ('nMilsec','<u2'),
21 ('nTimezone','<i2'),
21 ('nTimezone','<i2'),
22 ('nDstflag','<i2'),
22 ('nDstflag','<i2'),
23 ('nErrorCount','<u4')
23 ('nErrorCount','<u4')
24 ])
24 ])
25
25
26 SYSTEM_STRUCTURE = numpy.dtype([
26 SYSTEM_STRUCTURE = numpy.dtype([
27 ('nSize','<u4'),
27 ('nSize','<u4'),
28 ('nNumSamples','<u4'),
28 ('nNumSamples','<u4'),
29 ('nNumProfiles','<u4'),
29 ('nNumProfiles','<u4'),
30 ('nNumChannels','<u4'),
30 ('nNumChannels','<u4'),
31 ('nADCResolution','<u4'),
31 ('nADCResolution','<u4'),
32 ('nPCDIOBusWidth','<u4'),
32 ('nPCDIOBusWidth','<u4'),
33 ])
33 ])
34
34
35 RADAR_STRUCTURE = numpy.dtype([
35 RADAR_STRUCTURE = numpy.dtype([
36 ('nSize','<u4'),
36 ('nSize','<u4'),
37 ('nExpType','<u4'),
37 ('nExpType','<u4'),
38 ('nNTx','<u4'),
38 ('nNTx','<u4'),
39 ('fIpp','<f4'),
39 ('fIpp','<f4'),
40 ('fTxA','<f4'),
40 ('fTxA','<f4'),
41 ('fTxB','<f4'),
41 ('fTxB','<f4'),
42 ('nNumWindows','<u4'),
42 ('nNumWindows','<u4'),
43 ('nNumTaus','<u4'),
43 ('nNumTaus','<u4'),
44 ('nCodeType','<u4'),
44 ('nCodeType','<u4'),
45 ('nLine6Function','<u4'),
45 ('nLine6Function','<u4'),
46 ('nLine5Function','<u4'),
46 ('nLine5Function','<u4'),
47 ('fClock','<f4'),
47 ('fClock','<f4'),
48 ('nPrePulseBefore','<u4'),
48 ('nPrePulseBefore','<u4'),
49 ('nPrePulseAfter','<u4'),
49 ('nPrePulseAfter','<u4'),
50 ('sRangeIPP','<a20'),
50 ('sRangeIPP','<a20'),
51 ('sRangeTxA','<a20'),
51 ('sRangeTxA','<a20'),
52 ('sRangeTxB','<a20'),
52 ('sRangeTxB','<a20'),
53 ])
53 ])
54
54
55 SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
55 SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
56
56
57
57
58 PROCESSING_STRUCTURE = numpy.dtype([
58 PROCESSING_STRUCTURE = numpy.dtype([
59 ('nSize','<u4'),
59 ('nSize','<u4'),
60 ('nDataType','<u4'),
60 ('nDataType','<u4'),
61 ('nSizeOfDataBlock','<u4'),
61 ('nSizeOfDataBlock','<u4'),
62 ('nProfilesperBlock','<u4'),
62 ('nProfilesperBlock','<u4'),
63 ('nDataBlocksperFile','<u4'),
63 ('nDataBlocksperFile','<u4'),
64 ('nNumWindows','<u4'),
64 ('nNumWindows','<u4'),
65 ('nProcessFlags','<u4'),
65 ('nProcessFlags','<u4'),
66 ('nCoherentIntegrations','<u4'),
66 ('nCoherentIntegrations','<u4'),
67 ('nIncoherentIntegrations','<u4'),
67 ('nIncoherentIntegrations','<u4'),
68 ('nTotalSpectra','<u4')
68 ('nTotalSpectra','<u4')
69 ])
69 ])
70
70
71 class Header(object):
71 class Header(object):
72
72
73 def __init__(self):
73 def __init__(self):
74 raise NotImplementedError
74 raise NotImplementedError
75
75
76 def copy(self):
76 def copy(self):
77 return copy.deepcopy(self)
77 return copy.deepcopy(self)
78
78
79 def read(self):
79 def read(self):
80
80
81 raise NotImplementedError
81 raise NotImplementedError
82
82
83 def write(self):
83 def write(self):
84
84
85 raise NotImplementedError
85 raise NotImplementedError
86
86
87 def getAllowedArgs(self):
87 def getAllowedArgs(self):
88 args = inspect.getargspec(self.__init__).args
88 args = inspect.getargspec(self.__init__).args
89 try:
89 try:
90 args.remove('self')
90 args.remove('self')
91 except:
91 except:
92 pass
92 pass
93 return args
93 return args
94
94
95 def getAsDict(self):
95 def getAsDict(self):
96 args = self.getAllowedArgs()
96 args = self.getAllowedArgs()
97 asDict = {}
97 asDict = {}
98 for x in args:
98 for x in args:
99 asDict[x] = self[x]
99 asDict[x] = self[x]
100 return asDict
100 return asDict
101
101
102 def __getitem__(self, name):
102 def __getitem__(self, name):
103 return getattr(self, name)
103 return getattr(self, name)
104
104
105 def printInfo(self):
105 def printInfo(self):
106
106
107 message = "#"*50 + "\n"
107 message = "#"*50 + "\n"
108 message += self.__class__.__name__.upper() + "\n"
108 message += self.__class__.__name__.upper() + "\n"
109 message += "#"*50 + "\n"
109 message += "#"*50 + "\n"
110
110
111 keyList = self.__dict__.keys()
111 keyList = self.__dict__.keys()
112 keyList.sort()
112 keyList.sort()
113
113
114 for key in keyList:
114 for key in keyList:
115 message += "%s = %s" %(key, self.__dict__[key]) + "\n"
115 message += "%s = %s" %(key, self.__dict__[key]) + "\n"
116
116
117 if "size" not in keyList:
117 if "size" not in keyList:
118 attr = getattr(self, "size")
118 attr = getattr(self, "size")
119
119
120 if attr:
120 if attr:
121 message += "%s = %s" %("size", attr) + "\n"
121 message += "%s = %s" %("size", attr) + "\n"
122
122
123 print message
123 print message
124
124
125 class BasicHeader(Header):
125 class BasicHeader(Header):
126
126
127 size = None
127 size = None
128 version = None
128 version = None
129 dataBlock = None
129 dataBlock = None
130 utc = None
130 utc = None
131 ltc = None
131 ltc = None
132 miliSecond = None
132 miliSecond = None
133 timeZone = None
133 timeZone = None
134 dstFlag = None
134 dstFlag = None
135 errorCount = None
135 errorCount = None
136 datatime = None
136 datatime = None
137 structure = BASIC_STRUCTURE
137 structure = BASIC_STRUCTURE
138 __LOCALTIME = None
138 __LOCALTIME = None
139
139
140 def __init__(self, useLocalTime=True):
140 def __init__(self, useLocalTime=True):
141
141
142 self.size = 24
142 self.size = 24
143 self.version = 0
143 self.version = 0
144 self.dataBlock = 0
144 self.dataBlock = 0
145 self.utc = 0
145 self.utc = 0
146 self.miliSecond = 0
146 self.miliSecond = 0
147 self.timeZone = 0
147 self.timeZone = 0
148 self.dstFlag = 0
148 self.dstFlag = 0
149 self.errorCount = 0
149 self.errorCount = 0
150
150
151 self.useLocalTime = useLocalTime
151 self.useLocalTime = useLocalTime
152
152
153 def read(self, fp):
153 def read(self, fp):
154
154
155 self.length = 0
155 self.length = 0
156 try:
156 try:
157 if hasattr(fp, 'read'):
157 if hasattr(fp, 'read'):
158 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
158 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
159 else:
159 else:
160 header = numpy.fromstring(fp, BASIC_STRUCTURE,1)
160 header = numpy.fromstring(fp, BASIC_STRUCTURE,1)
161 except Exception, e:
161 except Exception, e:
162 print "BasicHeader: "
162 print "BasicHeader: "
163 print e
163 print e
164 return 0
164 return 0
165
165
166 self.size = int(header['nSize'][0])
166 self.size = int(header['nSize'][0])
167 self.version = int(header['nVersion'][0])
167 self.version = int(header['nVersion'][0])
168 self.dataBlock = int(header['nDataBlockId'][0])
168 self.dataBlock = int(header['nDataBlockId'][0])
169 self.utc = int(header['nUtime'][0])
169 self.utc = int(header['nUtime'][0])
170 self.miliSecond = int(header['nMilsec'][0])
170 self.miliSecond = int(header['nMilsec'][0])
171 self.timeZone = int(header['nTimezone'][0])
171 self.timeZone = int(header['nTimezone'][0])
172 self.dstFlag = int(header['nDstflag'][0])
172 self.dstFlag = int(header['nDstflag'][0])
173 self.errorCount = int(header['nErrorCount'][0])
173 self.errorCount = int(header['nErrorCount'][0])
174
174
175 if self.size < 24:
175 if self.size < 24:
176 return 0
176 return 0
177
177
178 self.length = header.nbytes
178 self.length = header.nbytes
179 return 1
179 return 1
180
180
181 def write(self, fp):
181 def write(self, fp):
182
182
183 headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
183 headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
184 header = numpy.array(headerTuple, BASIC_STRUCTURE)
184 header = numpy.array(headerTuple, BASIC_STRUCTURE)
185 header.tofile(fp)
185 header.tofile(fp)
186
186
187 return 1
187 return 1
188
188
189 def get_ltc(self):
189 def get_ltc(self):
190
190
191 return self.utc - self.timeZone*60
191 return self.utc - self.timeZone*60
192
192
193 def set_ltc(self, value):
193 def set_ltc(self, value):
194
194
195 self.utc = value + self.timeZone*60
195 self.utc = value + self.timeZone*60
196
196
197 def get_datatime(self):
197 def get_datatime(self):
198
198
199 return datetime.datetime.utcfromtimestamp(self.ltc)
199 return datetime.datetime.utcfromtimestamp(self.ltc)
200
200
201 ltc = property(get_ltc, set_ltc)
201 ltc = property(get_ltc, set_ltc)
202 datatime = property(get_datatime)
202 datatime = property(get_datatime)
203
203
204 class SystemHeader(Header):
204 class SystemHeader(Header):
205
205
206 size = None
206 size = None
207 nSamples = None
207 nSamples = None
208 nProfiles = None
208 nProfiles = None
209 nChannels = None
209 nChannels = None
210 adcResolution = None
210 adcResolution = None
211 pciDioBusWidth = None
211 pciDioBusWidth = None
212 structure = SYSTEM_STRUCTURE
212 structure = SYSTEM_STRUCTURE
213
213
214 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
214 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
215
215
216 self.size = 24
216 self.size = 24
217 self.nSamples = nSamples
217 self.nSamples = nSamples
218 self.nProfiles = nProfiles
218 self.nProfiles = nProfiles
219 self.nChannels = nChannels
219 self.nChannels = nChannels
220 self.adcResolution = adcResolution
220 self.adcResolution = adcResolution
221 self.pciDioBusWidth = pciDioBusWidth
221 self.pciDioBusWidth = pciDioBusWidth
222
222
223 def read(self, fp):
223 def read(self, fp):
224 self.length = 0
224 self.length = 0
225 try:
225 try:
226 startFp = fp.tell()
226 startFp = fp.tell()
227 except Exception, e:
227 except Exception, e:
228 startFp = None
228 startFp = None
229 pass
229 pass
230
230
231 try:
231 try:
232 if hasattr(fp, 'read'):
232 if hasattr(fp, 'read'):
233 header = numpy.fromfile(fp, SYSTEM_STRUCTURE,1)
233 header = numpy.fromfile(fp, SYSTEM_STRUCTURE,1)
234 else:
234 else:
235 header = numpy.fromstring(fp, SYSTEM_STRUCTURE,1)
235 header = numpy.fromstring(fp, SYSTEM_STRUCTURE,1)
236 except Exception, e:
236 except Exception, e:
237 print "System Header: " + str(e)
237 print "System Header: " + str(e)
238 return 0
238 return 0
239
239
240 self.size = header['nSize'][0]
240 self.size = header['nSize'][0]
241 self.nSamples = header['nNumSamples'][0]
241 self.nSamples = header['nNumSamples'][0]
242 self.nProfiles = header['nNumProfiles'][0]
242 self.nProfiles = header['nNumProfiles'][0]
243 self.nChannels = header['nNumChannels'][0]
243 self.nChannels = header['nNumChannels'][0]
244 self.adcResolution = header['nADCResolution'][0]
244 self.adcResolution = header['nADCResolution'][0]
245 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
245 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
246
246
247
247
248 if startFp is not None:
248 if startFp is not None:
249 endFp = self.size + startFp
249 endFp = self.size + startFp
250
250
251 if fp.tell() > endFp:
251 if fp.tell() > endFp:
252 sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name)
252 sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name)
253 return 0
253 return 0
254
254
255 if fp.tell() < endFp:
255 if fp.tell() < endFp:
256 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
256 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
257 return 0
257 return 0
258
258
259 self.length = header.nbytes
259 self.length = header.nbytes
260 return 1
260 return 1
261
261
262 def write(self, fp):
262 def write(self, fp):
263
263
264 headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
264 headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
265 header = numpy.array(headerTuple,SYSTEM_STRUCTURE)
265 header = numpy.array(headerTuple,SYSTEM_STRUCTURE)
266 header.tofile(fp)
266 header.tofile(fp)
267
267
268 return 1
268 return 1
269
269
270 class RadarControllerHeader(Header):
270 class RadarControllerHeader(Header):
271
271
272 expType = None
272 expType = None
273 nTx = None
273 nTx = None
274 ipp = None
274 ipp = None
275 txA = None
275 txA = None
276 txB = None
276 txB = None
277 nWindows = None
277 nWindows = None
278 numTaus = None
278 numTaus = None
279 codeType = None
279 codeType = None
280 line6Function = None
280 line6Function = None
281 line5Function = None
281 line5Function = None
282 fClock = None
282 fClock = None
283 prePulseBefore = None
283 prePulseBefore = None
284 prePulseAfter = None
284 prePulseAfter = None
285 rangeIpp = None
285 rangeIpp = None
286 rangeTxA = None
286 rangeTxA = None
287 rangeTxB = None
287 rangeTxB = None
288 structure = RADAR_STRUCTURE
288 structure = RADAR_STRUCTURE
289 __size = None
289 __size = None
290
290
291 def __init__(self, expType=2, nTx=1,
291 def __init__(self, expType=2, nTx=1,
292 ipp=None, txA=0, txB=0,
292 ipp=None, txA=0, txB=0,
293 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
293 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
294 numTaus=0, line6Function=0, line5Function=0, fClock=None,
294 numTaus=0, line6Function=0, line5Function=0, fClock=None,
295 prePulseBefore=0, prePulseAfter=0,
295 prePulseBefore=0, prePulseAfter=0,
296 codeType=0, nCode=0, nBaud=0, code=None,
296 codeType=0, nCode=0, nBaud=0, code=None,
297 flip1=0, flip2=0):
297 flip1=0, flip2=0):
298
298
299 # self.size = 116
299 # self.size = 116
300 self.expType = expType
300 self.expType = expType
301 self.nTx = nTx
301 self.nTx = nTx
302 self.ipp = ipp
302 self.ipp = ipp
303 self.txA = txA
303 self.txA = txA
304 self.txB = txB
304 self.txB = txB
305 self.rangeIpp = ipp
305 self.rangeIpp = ipp
306 self.rangeTxA = txA
306 self.rangeTxA = txA
307 self.rangeTxB = txB
307 self.rangeTxB = txB
308
308
309 self.nWindows = nWindows
309 self.nWindows = nWindows
310 self.numTaus = numTaus
310 self.numTaus = numTaus
311 self.codeType = codeType
311 self.codeType = codeType
312 self.line6Function = line6Function
312 self.line6Function = line6Function
313 self.line5Function = line5Function
313 self.line5Function = line5Function
314 self.fClock = fClock
314 self.fClock = fClock
315 self.prePulseBefore = prePulseBefore
315 self.prePulseBefore = prePulseBefore
316 self.prePulseAfter = prePulseAfter
316 self.prePulseAfter = prePulseAfter
317
317
318 self.nHeights = nHeights
318 self.nHeights = nHeights
319 self.firstHeight = firstHeight
319 self.firstHeight = firstHeight
320 self.deltaHeight = deltaHeight
320 self.deltaHeight = deltaHeight
321 self.samplesWin = nHeights
321 self.samplesWin = nHeights
322
322
323 self.nCode = nCode
323 self.nCode = nCode
324 self.nBaud = nBaud
324 self.nBaud = nBaud
325 self.code = code
325 self.code = code
326 self.flip1 = flip1
326 self.flip1 = flip1
327 self.flip2 = flip2
327 self.flip2 = flip2
328
328
329 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
329 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
330 # self.dynamic = numpy.array([],numpy.dtype('byte'))
330 # self.dynamic = numpy.array([],numpy.dtype('byte'))
331
331
332 if self.fClock is None and self.deltaHeight is not None:
332 if self.fClock is None and self.deltaHeight is not None:
333 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
333 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
334
334
335 def read(self, fp):
335 def read(self, fp):
336 self.length = 0
336 self.length = 0
337 try:
337 try:
338 startFp = fp.tell()
338 startFp = fp.tell()
339 except Exception, e:
339 except Exception, e:
340 startFp = None
340 startFp = None
341 pass
341 pass
342
342
343 try:
343 try:
344 if hasattr(fp, 'read'):
344 if hasattr(fp, 'read'):
345 header = numpy.fromfile(fp, RADAR_STRUCTURE,1)
345 header = numpy.fromfile(fp, RADAR_STRUCTURE,1)
346 else:
346 else:
347 header = numpy.fromstring(fp, RADAR_STRUCTURE,1)
347 header = numpy.fromstring(fp, RADAR_STRUCTURE,1)
348 self.length += header.nbytes
348 self.length += header.nbytes
349 except Exception, e:
349 except Exception, e:
350 print "RadarControllerHeader: " + str(e)
350 print "RadarControllerHeader: " + str(e)
351 return 0
351 return 0
352
352
353 size = int(header['nSize'][0])
353 size = int(header['nSize'][0])
354 self.expType = int(header['nExpType'][0])
354 self.expType = int(header['nExpType'][0])
355 self.nTx = int(header['nNTx'][0])
355 self.nTx = int(header['nNTx'][0])
356 self.ipp = float(header['fIpp'][0])
356 self.ipp = float(header['fIpp'][0])
357 self.txA = float(header['fTxA'][0])
357 self.txA = float(header['fTxA'][0])
358 self.txB = float(header['fTxB'][0])
358 self.txB = float(header['fTxB'][0])
359 self.nWindows = int(header['nNumWindows'][0])
359 self.nWindows = int(header['nNumWindows'][0])
360 self.numTaus = int(header['nNumTaus'][0])
360 self.numTaus = int(header['nNumTaus'][0])
361 self.codeType = int(header['nCodeType'][0])
361 self.codeType = int(header['nCodeType'][0])
362 self.line6Function = int(header['nLine6Function'][0])
362 self.line6Function = int(header['nLine6Function'][0])
363 self.line5Function = int(header['nLine5Function'][0])
363 self.line5Function = int(header['nLine5Function'][0])
364 self.fClock = float(header['fClock'][0])
364 self.fClock = float(header['fClock'][0])
365 self.prePulseBefore = int(header['nPrePulseBefore'][0])
365 self.prePulseBefore = int(header['nPrePulseBefore'][0])
366 self.prePulseAfter = int(header['nPrePulseAfter'][0])
366 self.prePulseAfter = int(header['nPrePulseAfter'][0])
367 self.rangeIpp = header['sRangeIPP'][0]
367 self.rangeIpp = header['sRangeIPP'][0]
368 self.rangeTxA = header['sRangeTxA'][0]
368 self.rangeTxA = header['sRangeTxA'][0]
369 self.rangeTxB = header['sRangeTxB'][0]
369 self.rangeTxB = header['sRangeTxB'][0]
370
370
371 try:
371 try:
372 if hasattr(fp, 'read'):
372 if hasattr(fp, 'read'):
373 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
373 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
374 else:
374 else:
375 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
375 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
376 self.length += samplingWindow.nbytes
376 self.length += samplingWindow.nbytes
377 except Exception, e:
377 except Exception, e:
378 print "RadarControllerHeader: " + str(e)
378 print "RadarControllerHeader: " + str(e)
379 return 0
379 return 0
380 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
380 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
381 self.firstHeight = samplingWindow['h0']
381 self.firstHeight = samplingWindow['h0']
382 self.deltaHeight = samplingWindow['dh']
382 self.deltaHeight = samplingWindow['dh']
383 self.samplesWin = samplingWindow['nsa']
383 self.samplesWin = samplingWindow['nsa']
384
384
385
385
386
386
387 try:
387 try:
388 if hasattr(fp, 'read'):
388 if hasattr(fp, 'read'):
389 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
389 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
390 else:
390 else:
391 self.Taus = numpy.fromstring(fp[self.length:], '<f4', self.numTaus)
391 self.Taus = numpy.fromstring(fp[self.length:], '<f4', self.numTaus)
392 self.length += self.Taus.nbytes
392 self.length += self.Taus.nbytes
393 except Exception, e:
393 except Exception, e:
394 print "RadarControllerHeader: " + str(e)
394 print "RadarControllerHeader: " + str(e)
395 return 0
395 return 0
396
396
397
397
398
398
399 self.code_size = 0
399 self.code_size = 0
400 if self.codeType != 0:
400 if self.codeType != 0:
401
401
402 try:
402 try:
403 if hasattr(fp, 'read'):
403 if hasattr(fp, 'read'):
404 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
404 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
405 self.length += self.nCode.nbytes
405 self.length += self.nCode.nbytes
406 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
406 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
407 self.length += self.nBaud.nbytes
407 self.length += self.nBaud.nbytes
408 else:
408 else:
409 self.nCode = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
409 self.nCode = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
410 self.length += self.nCode.nbytes
410 self.length += self.nCode.nbytes
411 self.nBaud = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
411 self.nBaud = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
412 self.length += self.nBaud.nbytes
412 self.length += self.nBaud.nbytes
413 except Exception, e:
413 except Exception, e:
414 print "RadarControllerHeader: " + str(e)
414 print "RadarControllerHeader: " + str(e)
415 return 0
415 return 0
416 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
416 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
417
417
418 for ic in range(self.nCode):
418 for ic in range(self.nCode):
419 try:
419 try:
420 if hasattr(fp, 'read'):
420 if hasattr(fp, 'read'):
421 temp = numpy.fromfile(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
421 temp = numpy.fromfile(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
422 else:
422 else:
423 temp = numpy.fromstring(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
423 temp = numpy.fromstring(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
424 self.length += temp.nbytes
424 self.length += temp.nbytes
425 except Exception, e:
425 except Exception, e:
426 print "RadarControllerHeader: " + str(e)
426 print "RadarControllerHeader: " + str(e)
427 return 0
427 return 0
428
428
429 for ib in range(self.nBaud-1,-1,-1):
429 for ib in range(self.nBaud-1,-1,-1):
430 code[ic,ib] = temp[ib/32]%2
430 code[ic,ib] = temp[ib/32]%2
431 temp[ib/32] = temp[ib/32]/2
431 temp[ib/32] = temp[ib/32]/2
432
432
433 self.code = 2.0*code - 1.0
433 self.code = 2.0*code - 1.0
434 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
434 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
435
435
436 # if self.line5Function == RCfunction.FLIP:
436 # if self.line5Function == RCfunction.FLIP:
437 # self.flip1 = numpy.fromfile(fp,'<u4',1)
437 # self.flip1 = numpy.fromfile(fp,'<u4',1)
438 #
438 #
439 # if self.line6Function == RCfunction.FLIP:
439 # if self.line6Function == RCfunction.FLIP:
440 # self.flip2 = numpy.fromfile(fp,'<u4',1)
440 # self.flip2 = numpy.fromfile(fp,'<u4',1)
441 if startFp is not None:
441 if startFp is not None:
442 endFp = size + startFp
442 endFp = size + startFp
443
443
444 if fp.tell() != endFp:
444 if fp.tell() != endFp:
445 # fp.seek(endFp)
445 # fp.seek(endFp)
446 print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" %(fp.name, fp.tell()-startFp, size)
446 print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" %(fp.name, fp.tell()-startFp, size)
447 # return 0
447 # return 0
448
448
449 if fp.tell() > endFp:
449 if fp.tell() > endFp:
450 sys.stderr.write("Warning %s: Size value read from Radar Controller header is lower than it has to be\n" %fp.name)
450 sys.stderr.write("Warning %s: Size value read from Radar Controller header is lower than it has to be\n" %fp.name)
451 # return 0
451 # return 0
452
452
453 if fp.tell() < endFp:
453 if fp.tell() < endFp:
454 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
454 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
455
455
456
456
457 return 1
457 return 1
458
458
459 def write(self, fp):
459 def write(self, fp):
460
460
461 headerTuple = (self.size,
461 headerTuple = (self.size,
462 self.expType,
462 self.expType,
463 self.nTx,
463 self.nTx,
464 self.ipp,
464 self.ipp,
465 self.txA,
465 self.txA,
466 self.txB,
466 self.txB,
467 self.nWindows,
467 self.nWindows,
468 self.numTaus,
468 self.numTaus,
469 self.codeType,
469 self.codeType,
470 self.line6Function,
470 self.line6Function,
471 self.line5Function,
471 self.line5Function,
472 self.fClock,
472 self.fClock,
473 self.prePulseBefore,
473 self.prePulseBefore,
474 self.prePulseAfter,
474 self.prePulseAfter,
475 self.rangeIpp,
475 self.rangeIpp,
476 self.rangeTxA,
476 self.rangeTxA,
477 self.rangeTxB)
477 self.rangeTxB)
478
478
479 header = numpy.array(headerTuple,RADAR_STRUCTURE)
479 header = numpy.array(headerTuple,RADAR_STRUCTURE)
480 header.tofile(fp)
480 header.tofile(fp)
481
481
482 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
482 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
483 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
483 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
484 samplingWindow.tofile(fp)
484 samplingWindow.tofile(fp)
485
485
486 if self.numTaus > 0:
486 if self.numTaus > 0:
487 self.Taus.tofile(fp)
487 self.Taus.tofile(fp)
488
488
489 if self.codeType !=0:
489 if self.codeType !=0:
490 nCode = numpy.array(self.nCode, '<u4')
490 nCode = numpy.array(self.nCode, '<u4')
491 nCode.tofile(fp)
491 nCode.tofile(fp)
492 nBaud = numpy.array(self.nBaud, '<u4')
492 nBaud = numpy.array(self.nBaud, '<u4')
493 nBaud.tofile(fp)
493 nBaud.tofile(fp)
494 code1 = (self.code + 1.0)/2.
494 code1 = (self.code + 1.0)/2.
495
495
496 for ic in range(self.nCode):
496 for ic in range(self.nCode):
497 tempx = numpy.zeros(numpy.ceil(self.nBaud/32.))
497 tempx = numpy.zeros(int(numpy.ceil(self.nBaud/32.)))
498 start = 0
498 start = 0
499 end = 32
499 end = 32
500 for i in range(len(tempx)):
500 for i in range(len(tempx)):
501 code_selected = code1[ic,start:end]
501 code_selected = code1[ic,start:end]
502 for j in range(len(code_selected)-1,-1,-1):
502 for j in range(len(code_selected)-1,-1,-1):
503 if code_selected[j] == 1:
503 if code_selected[j] == 1:
504 tempx[i] = tempx[i] + 2**(len(code_selected)-1-j)
504 tempx[i] = tempx[i] + 2**(len(code_selected)-1-j)
505 start = start + 32
505 start = start + 32
506 end = end + 32
506 end = end + 32
507
507
508 tempx = tempx.astype('u4')
508 tempx = tempx.astype('u4')
509 tempx.tofile(fp)
509 tempx.tofile(fp)
510
510
511 # if self.line5Function == RCfunction.FLIP:
511 # if self.line5Function == RCfunction.FLIP:
512 # self.flip1.tofile(fp)
512 # self.flip1.tofile(fp)
513 #
513 #
514 # if self.line6Function == RCfunction.FLIP:
514 # if self.line6Function == RCfunction.FLIP:
515 # self.flip2.tofile(fp)
515 # self.flip2.tofile(fp)
516
516
517 return 1
517 return 1
518
518
519 def get_ippSeconds(self):
519 def get_ippSeconds(self):
520 '''
520 '''
521 '''
521 '''
522 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
522 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
523
523
524 return ippSeconds
524 return ippSeconds
525
525
526 def set_ippSeconds(self, ippSeconds):
526 def set_ippSeconds(self, ippSeconds):
527 '''
527 '''
528 '''
528 '''
529
529
530 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000)
530 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000)
531
531
532 return
532 return
533
533
534 def get_size(self):
534 def get_size(self):
535
535
536 self.__size = 116 + 12*self.nWindows + 4*self.numTaus
536 self.__size = 116 + 12*self.nWindows + 4*self.numTaus
537
537
538 if self.codeType != 0:
538 if self.codeType != 0:
539 self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
539 self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
540
540
541 return self.__size
541 return self.__size
542
542
543 def set_size(self, value):
543 def set_size(self, value):
544
544
545 raise IOError, "size is a property and it cannot be set, just read"
545 raise IOError, "size is a property and it cannot be set, just read"
546
546
547 return
547 return
548
548
549 ippSeconds = property(get_ippSeconds, set_ippSeconds)
549 ippSeconds = property(get_ippSeconds, set_ippSeconds)
550 size = property(get_size, set_size)
550 size = property(get_size, set_size)
551
551
552 class ProcessingHeader(Header):
552 class ProcessingHeader(Header):
553
553
554 # size = None
554 # size = None
555 dtype = None
555 dtype = None
556 blockSize = None
556 blockSize = None
557 profilesPerBlock = None
557 profilesPerBlock = None
558 dataBlocksPerFile = None
558 dataBlocksPerFile = None
559 nWindows = None
559 nWindows = None
560 processFlags = None
560 processFlags = None
561 nCohInt = None
561 nCohInt = None
562 nIncohInt = None
562 nIncohInt = None
563 totalSpectra = None
563 totalSpectra = None
564 structure = PROCESSING_STRUCTURE
564 structure = PROCESSING_STRUCTURE
565 flag_dc = None
565 flag_dc = None
566 flag_cspc = None
566 flag_cspc = None
567
567
568 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0,processFlags=0, nCohInt=0,
568 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0,processFlags=0, nCohInt=0,
569 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
569 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
570 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
570 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
571 ):
571 ):
572
572
573 # self.size = 0
573 # self.size = 0
574 self.dtype = dtype
574 self.dtype = dtype
575 self.blockSize = blockSize
575 self.blockSize = blockSize
576 self.profilesPerBlock = 0
576 self.profilesPerBlock = 0
577 self.dataBlocksPerFile = 0
577 self.dataBlocksPerFile = 0
578 self.nWindows = 0
578 self.nWindows = 0
579 self.processFlags = 0
579 self.processFlags = 0
580 self.nCohInt = 0
580 self.nCohInt = 0
581 self.nIncohInt = 0
581 self.nIncohInt = 0
582 self.totalSpectra = 0
582 self.totalSpectra = 0
583
583
584 self.nHeights = 0
584 self.nHeights = 0
585 self.firstHeight = 0
585 self.firstHeight = 0
586 self.deltaHeight = 0
586 self.deltaHeight = 0
587 self.samplesWin = 0
587 self.samplesWin = 0
588 self.spectraComb = 0
588 self.spectraComb = 0
589 self.nCode = None
589 self.nCode = None
590 self.code = None
590 self.code = None
591 self.nBaud = None
591 self.nBaud = None
592
592
593 self.shif_fft = False
593 self.shif_fft = False
594 self.flag_dc = False
594 self.flag_dc = False
595 self.flag_cspc = False
595 self.flag_cspc = False
596 self.flag_decode = False
596 self.flag_decode = False
597 self.flag_deflip = False
597 self.flag_deflip = False
598 self.length = 0
598 self.length = 0
599
599
600 def read(self, fp):
600 def read(self, fp):
601 self.length = 0
601 self.length = 0
602 try:
602 try:
603 startFp = fp.tell()
603 startFp = fp.tell()
604 except Exception, e:
604 except Exception, e:
605 startFp = None
605 startFp = None
606 pass
606 pass
607
607
608 try:
608 try:
609 if hasattr(fp, 'read'):
609 if hasattr(fp, 'read'):
610 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
610 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
611 else:
611 else:
612 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
612 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
613 self.length += header.nbytes
613 self.length += header.nbytes
614 except Exception, e:
614 except Exception, e:
615 print "ProcessingHeader: " + str(e)
615 print "ProcessingHeader: " + str(e)
616 return 0
616 return 0
617
617
618 size = int(header['nSize'][0])
618 size = int(header['nSize'][0])
619 self.dtype = int(header['nDataType'][0])
619 self.dtype = int(header['nDataType'][0])
620 self.blockSize = int(header['nSizeOfDataBlock'][0])
620 self.blockSize = int(header['nSizeOfDataBlock'][0])
621 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
621 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
622 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
622 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
623 self.nWindows = int(header['nNumWindows'][0])
623 self.nWindows = int(header['nNumWindows'][0])
624 self.processFlags = header['nProcessFlags']
624 self.processFlags = header['nProcessFlags']
625 self.nCohInt = int(header['nCoherentIntegrations'][0])
625 self.nCohInt = int(header['nCoherentIntegrations'][0])
626 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
626 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
627 self.totalSpectra = int(header['nTotalSpectra'][0])
627 self.totalSpectra = int(header['nTotalSpectra'][0])
628
628
629 try:
629 try:
630 if hasattr(fp, 'read'):
630 if hasattr(fp, 'read'):
631 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
631 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
632 else:
632 else:
633 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
633 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
634 self.length += samplingWindow.nbytes
634 self.length += samplingWindow.nbytes
635 except Exception, e:
635 except Exception, e:
636 print "ProcessingHeader: " + str(e)
636 print "ProcessingHeader: " + str(e)
637 return 0
637 return 0
638
638
639 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
639 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
640 self.firstHeight = float(samplingWindow['h0'][0])
640 self.firstHeight = float(samplingWindow['h0'][0])
641 self.deltaHeight = float(samplingWindow['dh'][0])
641 self.deltaHeight = float(samplingWindow['dh'][0])
642 self.samplesWin = samplingWindow['nsa'][0]
642 self.samplesWin = samplingWindow['nsa'][0]
643
643
644
644
645 try:
645 try:
646 if hasattr(fp, 'read'):
646 if hasattr(fp, 'read'):
647 self.spectraComb = numpy.fromfile(fp, 'u1', 2*self.totalSpectra)
647 self.spectraComb = numpy.fromfile(fp, 'u1', 2*self.totalSpectra)
648 else:
648 else:
649 self.spectraComb = numpy.fromstring(fp[self.length:], 'u1', 2*self.totalSpectra)
649 self.spectraComb = numpy.fromstring(fp[self.length:], 'u1', 2*self.totalSpectra)
650 self.length += self.spectraComb.nbytes
650 self.length += self.spectraComb.nbytes
651 except Exception, e:
651 except Exception, e:
652 print "ProcessingHeader: " + str(e)
652 print "ProcessingHeader: " + str(e)
653 return 0
653 return 0
654
654
655 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
655 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
656 self.nCode = int(numpy.fromfile(fp,'<u4',1))
656 self.nCode = int(numpy.fromfile(fp,'<u4',1))
657 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
657 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
658 self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud)
658 self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud)
659
659
660 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
660 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
661 exp_name_len = int(numpy.fromfile(fp,'<u4',1))
661 exp_name_len = int(numpy.fromfile(fp,'<u4',1))
662 exp_name = numpy.fromfile(fp,'u1',exp_name_len+1)
662 exp_name = numpy.fromfile(fp,'u1',exp_name_len+1)
663
663
664 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
664 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
665 self.shif_fft = True
665 self.shif_fft = True
666 else:
666 else:
667 self.shif_fft = False
667 self.shif_fft = False
668
668
669 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
669 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
670 self.flag_dc = True
670 self.flag_dc = True
671 else:
671 else:
672 self.flag_dc = False
672 self.flag_dc = False
673
673
674 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
674 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
675 self.flag_decode = True
675 self.flag_decode = True
676 else:
676 else:
677 self.flag_decode = False
677 self.flag_decode = False
678
678
679 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
679 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
680 self.flag_deflip = True
680 self.flag_deflip = True
681 else:
681 else:
682 self.flag_deflip = False
682 self.flag_deflip = False
683
683
684 nChannels = 0
684 nChannels = 0
685 nPairs = 0
685 nPairs = 0
686 pairList = []
686 pairList = []
687
687
688 for i in range( 0, self.totalSpectra*2, 2 ):
688 for i in range( 0, self.totalSpectra*2, 2 ):
689 if self.spectraComb[i] == self.spectraComb[i+1]:
689 if self.spectraComb[i] == self.spectraComb[i+1]:
690 nChannels = nChannels + 1 #par de canales iguales
690 nChannels = nChannels + 1 #par de canales iguales
691 else:
691 else:
692 nPairs = nPairs + 1 #par de canales diferentes
692 nPairs = nPairs + 1 #par de canales diferentes
693 pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
693 pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
694
694
695 self.flag_cspc = False
695 self.flag_cspc = False
696 if nPairs > 0:
696 if nPairs > 0:
697 self.flag_cspc = True
697 self.flag_cspc = True
698
698
699
699
700
700
701 if startFp is not None:
701 if startFp is not None:
702 endFp = size + startFp
702 endFp = size + startFp
703 if fp.tell() > endFp:
703 if fp.tell() > endFp:
704 sys.stderr.write("Warning: Processing header size is lower than it has to be")
704 sys.stderr.write("Warning: Processing header size is lower than it has to be")
705 return 0
705 return 0
706
706
707 if fp.tell() < endFp:
707 if fp.tell() < endFp:
708 sys.stderr.write("Warning: Processing header size is greater than it is considered")
708 sys.stderr.write("Warning: Processing header size is greater than it is considered")
709
709
710 return 1
710 return 1
711
711
712 def write(self, fp):
712 def write(self, fp):
713 #Clear DEFINE_PROCESS_CODE
713 #Clear DEFINE_PROCESS_CODE
714 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
714 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
715
715
716 headerTuple = (self.size,
716 headerTuple = (self.size,
717 self.dtype,
717 self.dtype,
718 self.blockSize,
718 self.blockSize,
719 self.profilesPerBlock,
719 self.profilesPerBlock,
720 self.dataBlocksPerFile,
720 self.dataBlocksPerFile,
721 self.nWindows,
721 self.nWindows,
722 self.processFlags,
722 self.processFlags,
723 self.nCohInt,
723 self.nCohInt,
724 self.nIncohInt,
724 self.nIncohInt,
725 self.totalSpectra)
725 self.totalSpectra)
726
726
727 header = numpy.array(headerTuple,PROCESSING_STRUCTURE)
727 header = numpy.array(headerTuple,PROCESSING_STRUCTURE)
728 header.tofile(fp)
728 header.tofile(fp)
729
729
730 if self.nWindows != 0:
730 if self.nWindows != 0:
731 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
731 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
732 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
732 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
733 samplingWindow.tofile(fp)
733 samplingWindow.tofile(fp)
734
734
735 if self.totalSpectra != 0:
735 if self.totalSpectra != 0:
736 # spectraComb = numpy.array([],numpy.dtype('u1'))
736 # spectraComb = numpy.array([],numpy.dtype('u1'))
737 spectraComb = self.spectraComb
737 spectraComb = self.spectraComb
738 spectraComb.tofile(fp)
738 spectraComb.tofile(fp)
739
739
740 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
740 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
741 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
741 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
742 # nCode.tofile(fp)
742 # nCode.tofile(fp)
743 #
743 #
744 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
744 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
745 # nBaud.tofile(fp)
745 # nBaud.tofile(fp)
746 #
746 #
747 # code = self.code.reshape(self.nCode*self.nBaud)
747 # code = self.code.reshape(self.nCode*self.nBaud)
748 # code = code.astype(numpy.dtype('<f4'))
748 # code = code.astype(numpy.dtype('<f4'))
749 # code.tofile(fp)
749 # code.tofile(fp)
750
750
751 return 1
751 return 1
752
752
753 def get_size(self):
753 def get_size(self):
754
754
755 self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra
755 self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra
756
756
757 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
757 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
758 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
758 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
759 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
759 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
760
760
761 return self.__size
761 return self.__size
762
762
763 def set_size(self, value):
763 def set_size(self, value):
764
764
765 raise IOError, "size is a property and it cannot be set, just read"
765 raise IOError, "size is a property and it cannot be set, just read"
766
766
767 return
767 return
768
768
769 size = property(get_size, set_size)
769 size = property(get_size, set_size)
770
770
771 class RCfunction:
771 class RCfunction:
772 NONE=0
772 NONE=0
773 FLIP=1
773 FLIP=1
774 CODE=2
774 CODE=2
775 SAMPLING=3
775 SAMPLING=3
776 LIN6DIV256=4
776 LIN6DIV256=4
777 SYNCHRO=5
777 SYNCHRO=5
778
778
779 class nCodeType:
779 class nCodeType:
780 NONE=0
780 NONE=0
781 USERDEFINE=1
781 USERDEFINE=1
782 BARKER2=2
782 BARKER2=2
783 BARKER3=3
783 BARKER3=3
784 BARKER4=4
784 BARKER4=4
785 BARKER5=5
785 BARKER5=5
786 BARKER7=6
786 BARKER7=6
787 BARKER11=7
787 BARKER11=7
788 BARKER13=8
788 BARKER13=8
789 AC128=9
789 AC128=9
790 COMPLEMENTARYCODE2=10
790 COMPLEMENTARYCODE2=10
791 COMPLEMENTARYCODE4=11
791 COMPLEMENTARYCODE4=11
792 COMPLEMENTARYCODE8=12
792 COMPLEMENTARYCODE8=12
793 COMPLEMENTARYCODE16=13
793 COMPLEMENTARYCODE16=13
794 COMPLEMENTARYCODE32=14
794 COMPLEMENTARYCODE32=14
795 COMPLEMENTARYCODE64=15
795 COMPLEMENTARYCODE64=15
796 COMPLEMENTARYCODE128=16
796 COMPLEMENTARYCODE128=16
797 CODE_BINARY28=17
797 CODE_BINARY28=17
798
798
799 class PROCFLAG:
799 class PROCFLAG:
800
800
801 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
801 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
802 DECODE_DATA = numpy.uint32(0x00000002)
802 DECODE_DATA = numpy.uint32(0x00000002)
803 SPECTRA_CALC = numpy.uint32(0x00000004)
803 SPECTRA_CALC = numpy.uint32(0x00000004)
804 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
804 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
805 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
805 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
806 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
806 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
807
807
808 DATATYPE_CHAR = numpy.uint32(0x00000040)
808 DATATYPE_CHAR = numpy.uint32(0x00000040)
809 DATATYPE_SHORT = numpy.uint32(0x00000080)
809 DATATYPE_SHORT = numpy.uint32(0x00000080)
810 DATATYPE_LONG = numpy.uint32(0x00000100)
810 DATATYPE_LONG = numpy.uint32(0x00000100)
811 DATATYPE_INT64 = numpy.uint32(0x00000200)
811 DATATYPE_INT64 = numpy.uint32(0x00000200)
812 DATATYPE_FLOAT = numpy.uint32(0x00000400)
812 DATATYPE_FLOAT = numpy.uint32(0x00000400)
813 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
813 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
814
814
815 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
815 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
816 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
816 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
817 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
817 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
818
818
819 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
819 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
820 DEFLIP_DATA = numpy.uint32(0x00010000)
820 DEFLIP_DATA = numpy.uint32(0x00010000)
821 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
821 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
822
822
823 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
823 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
824 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
824 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
825 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
825 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
826 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
826 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
827 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
827 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
828
828
829 EXP_NAME_ESP = numpy.uint32(0x00200000)
829 EXP_NAME_ESP = numpy.uint32(0x00200000)
830 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
830 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
831
831
832 OPERATION_MASK = numpy.uint32(0x0000003F)
832 OPERATION_MASK = numpy.uint32(0x0000003F)
833 DATATYPE_MASK = numpy.uint32(0x00000FC0)
833 DATATYPE_MASK = numpy.uint32(0x00000FC0)
834 DATAARRANGE_MASK = numpy.uint32(0x00007000)
834 DATAARRANGE_MASK = numpy.uint32(0x00007000)
835 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
835 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
836
836
837 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
837 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
838 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
838 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
839 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
839 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
840 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
840 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
841 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
841 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
842 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
842 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
843
843
844 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
844 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
845
845
846 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
846 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
847 PROCFLAG.DATATYPE_SHORT,
847 PROCFLAG.DATATYPE_SHORT,
848 PROCFLAG.DATATYPE_LONG,
848 PROCFLAG.DATATYPE_LONG,
849 PROCFLAG.DATATYPE_INT64,
849 PROCFLAG.DATATYPE_INT64,
850 PROCFLAG.DATATYPE_FLOAT,
850 PROCFLAG.DATATYPE_FLOAT,
851 PROCFLAG.DATATYPE_DOUBLE]
851 PROCFLAG.DATATYPE_DOUBLE]
852
852
853 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
853 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
854
854
855 def get_dtype_index(numpy_dtype):
855 def get_dtype_index(numpy_dtype):
856
856
857 index = None
857 index = None
858
858
859 for i in range(len(NUMPY_DTYPE_LIST)):
859 for i in range(len(NUMPY_DTYPE_LIST)):
860 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
860 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
861 index = i
861 index = i
862 break
862 break
863
863
864 return index
864 return index
865
865
866 def get_numpy_dtype(index):
866 def get_numpy_dtype(index):
867
867
868 return NUMPY_DTYPE_LIST[index]
868 return NUMPY_DTYPE_LIST[index]
869
869
870 def get_procflag_dtype(index):
870 def get_procflag_dtype(index):
871
871
872 return PROCFLAG_DTYPE_LIST[index]
872 return PROCFLAG_DTYPE_LIST[index]
873
873
874 def get_dtype_width(index):
874 def get_dtype_width(index):
875
875
876 return DTYPE_WIDTH[index] No newline at end of file
876 return DTYPE_WIDTH[index]
Show file before | Show file after | Hide comments
@@ -1,1833 +1,1824
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time
13 import time
14 import datetime
14 import datetime
15 import traceback
15 import traceback
16 import zmq
16 import zmq
17
17
18 try:
18 try:
19 from gevent import sleep
19 from gevent import sleep
20 except:
20 except:
21 from time import sleep
21 from time import sleep
22
22
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 from schainpy.utils import log
25
26
26 LOCALTIME = True
27 LOCALTIME = True
27
28
28
29
29 def isNumber(cad):
30 def isNumber(cad):
30 """
31 """
31 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
32 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
32
33
33 Excepciones:
34 Excepciones:
34 Si un determinado string no puede ser convertido a numero
35 Si un determinado string no puede ser convertido a numero
35 Input:
36 Input:
36 str, string al cual se le analiza para determinar si convertible a un numero o no
37 str, string al cual se le analiza para determinar si convertible a un numero o no
37
38
38 Return:
39 Return:
39 True : si el string es uno numerico
40 True : si el string es uno numerico
40 False : no es un string numerico
41 False : no es un string numerico
41 """
42 """
42 try:
43 try:
43 float(cad)
44 float(cad)
44 return True
45 return True
45 except:
46 except:
46 return False
47 return False
47
48
48
49
49 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
50 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
50 """
51 """
51 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
52 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
52
53
53 Inputs:
54 Inputs:
54 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
55 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
55
56
56 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
57 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
57 segundos contados desde 01/01/1970.
58 segundos contados desde 01/01/1970.
58 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
59 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
59 segundos contados desde 01/01/1970.
60 segundos contados desde 01/01/1970.
60
61
61 Return:
62 Return:
62 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
63 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
63 fecha especificado, de lo contrario retorna False.
64 fecha especificado, de lo contrario retorna False.
64
65
65 Excepciones:
66 Excepciones:
66 Si el archivo no existe o no puede ser abierto
67 Si el archivo no existe o no puede ser abierto
67 Si la cabecera no puede ser leida.
68 Si la cabecera no puede ser leida.
68
69
69 """
70 """
70 basicHeaderObj = BasicHeader(LOCALTIME)
71 basicHeaderObj = BasicHeader(LOCALTIME)
71
72
72 try:
73 try:
73 fp = open(filename, 'rb')
74 fp = open(filename, 'rb')
74 except IOError:
75 except IOError:
75 print "The file %s can't be opened" % (filename)
76 print "The file %s can't be opened" % (filename)
76 return 0
77 return 0
77
78
78 sts = basicHeaderObj.read(fp)
79 sts = basicHeaderObj.read(fp)
79 fp.close()
80 fp.close()
80
81
81 if not(sts):
82 if not(sts):
82 print "Skipping the file %s because it has not a valid header" % (filename)
83 print "Skipping the file %s because it has not a valid header" % (filename)
83 return 0
84 return 0
84
85
85 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
86 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
86 return 0
87 return 0
87
88
88 return 1
89 return 1
89
90
90
91
91 def isTimeInRange(thisTime, startTime, endTime):
92 def isTimeInRange(thisTime, startTime, endTime):
92 if endTime >= startTime:
93 if endTime >= startTime:
93 if (thisTime < startTime) or (thisTime > endTime):
94 if (thisTime < startTime) or (thisTime > endTime):
94 return 0
95 return 0
95 return 1
96 return 1
96 else:
97 else:
97 if (thisTime < startTime) and (thisTime > endTime):
98 if (thisTime < startTime) and (thisTime > endTime):
98 return 0
99 return 0
99 return 1
100 return 1
100
101
101
102
102 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
103 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
103 """
104 """
104 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
105 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
105
106
106 Inputs:
107 Inputs:
107 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
108 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
108
109
109 startDate : fecha inicial del rango seleccionado en formato datetime.date
110 startDate : fecha inicial del rango seleccionado en formato datetime.date
110
111
111 endDate : fecha final del rango seleccionado en formato datetime.date
112 endDate : fecha final del rango seleccionado en formato datetime.date
112
113
113 startTime : tiempo inicial del rango seleccionado en formato datetime.time
114 startTime : tiempo inicial del rango seleccionado en formato datetime.time
114
115
115 endTime : tiempo final del rango seleccionado en formato datetime.time
116 endTime : tiempo final del rango seleccionado en formato datetime.time
116
117
117 Return:
118 Return:
118 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
119 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
119 fecha especificado, de lo contrario retorna False.
120 fecha especificado, de lo contrario retorna False.
120
121
121 Excepciones:
122 Excepciones:
122 Si el archivo no existe o no puede ser abierto
123 Si el archivo no existe o no puede ser abierto
123 Si la cabecera no puede ser leida.
124 Si la cabecera no puede ser leida.
124
125
125 """
126 """
126
127
127 try:
128 try:
128 fp = open(filename, 'rb')
129 fp = open(filename, 'rb')
129 except IOError:
130 except IOError:
130 print "The file %s can't be opened" % (filename)
131 print "The file %s can't be opened" % (filename)
131 return None
132 return None
132
133
133 firstBasicHeaderObj = BasicHeader(LOCALTIME)
134 firstBasicHeaderObj = BasicHeader(LOCALTIME)
134 systemHeaderObj = SystemHeader()
135 systemHeaderObj = SystemHeader()
135 radarControllerHeaderObj = RadarControllerHeader()
136 radarControllerHeaderObj = RadarControllerHeader()
136 processingHeaderObj = ProcessingHeader()
137 processingHeaderObj = ProcessingHeader()
137
138
138 lastBasicHeaderObj = BasicHeader(LOCALTIME)
139 lastBasicHeaderObj = BasicHeader(LOCALTIME)
139
140
140 sts = firstBasicHeaderObj.read(fp)
141 sts = firstBasicHeaderObj.read(fp)
141
142
142 if not(sts):
143 if not(sts):
143 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
144 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
144 return None
145 return None
145
146
146 if not systemHeaderObj.read(fp):
147 if not systemHeaderObj.read(fp):
147 return None
148 return None
148
149
149 if not radarControllerHeaderObj.read(fp):
150 if not radarControllerHeaderObj.read(fp):
150 return None
151 return None
151
152
152 if not processingHeaderObj.read(fp):
153 if not processingHeaderObj.read(fp):
153 return None
154 return None
154
155
155 filesize = os.path.getsize(filename)
156 filesize = os.path.getsize(filename)
156
157
157 offset = processingHeaderObj.blockSize + 24 # header size
158 offset = processingHeaderObj.blockSize + 24 # header size
158
159
159 if filesize <= offset:
160 if filesize <= offset:
160 print "[Reading] %s: This file has not enough data" % filename
161 print "[Reading] %s: This file has not enough data" % filename
161 return None
162 return None
162
163
163 fp.seek(-offset, 2)
164 fp.seek(-offset, 2)
164
165
165 sts = lastBasicHeaderObj.read(fp)
166 sts = lastBasicHeaderObj.read(fp)
166
167
167 fp.close()
168 fp.close()
168
169
169 thisDatetime = lastBasicHeaderObj.datatime
170 thisDatetime = lastBasicHeaderObj.datatime
170 thisTime_last_block = thisDatetime.time()
171 thisTime_last_block = thisDatetime.time()
171
172
172 thisDatetime = firstBasicHeaderObj.datatime
173 thisDatetime = firstBasicHeaderObj.datatime
173 thisDate = thisDatetime.date()
174 thisDate = thisDatetime.date()
174 thisTime_first_block = thisDatetime.time()
175 thisTime_first_block = thisDatetime.time()
175
176
176 # General case
177 # General case
177 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
178 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
178 #-----------o----------------------------o-----------
179 #-----------o----------------------------o-----------
179 # startTime endTime
180 # startTime endTime
180
181
181 if endTime >= startTime:
182 if endTime >= startTime:
182 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
183 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
183 return None
184 return None
184
185
185 return thisDatetime
186 return thisDatetime
186
187
187 # If endTime < startTime then endTime belongs to the next day
188 # If endTime < startTime then endTime belongs to the next day
188
189
189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 #<<<<<<<<<<<o o>>>>>>>>>>>
190 #-----------o----------------------------o-----------
191 #-----------o----------------------------o-----------
191 # endTime startTime
192 # endTime startTime
192
193
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 return None
195 return None
195
196
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 return None
198 return None
198
199
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 return None
201 return None
201
202
202 return thisDatetime
203 return thisDatetime
203
204
204
205
205 def isFolderInDateRange(folder, startDate=None, endDate=None):
206 def isFolderInDateRange(folder, startDate=None, endDate=None):
206 """
207 """
207 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
208 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
208
209
209 Inputs:
210 Inputs:
210 folder : nombre completo del directorio.
211 folder : nombre completo del directorio.
211 Su formato deberia ser "/path_root/?YYYYDDD"
212 Su formato deberia ser "/path_root/?YYYYDDD"
212
213
213 siendo:
214 siendo:
214 YYYY : Anio (ejemplo 2015)
215 YYYY : Anio (ejemplo 2015)
215 DDD : Dia del anio (ejemplo 305)
216 DDD : Dia del anio (ejemplo 305)
216
217
217 startDate : fecha inicial del rango seleccionado en formato datetime.date
218 startDate : fecha inicial del rango seleccionado en formato datetime.date
218
219
219 endDate : fecha final del rango seleccionado en formato datetime.date
220 endDate : fecha final del rango seleccionado en formato datetime.date
220
221
221 Return:
222 Return:
222 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
223 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
223 fecha especificado, de lo contrario retorna False.
224 fecha especificado, de lo contrario retorna False.
224 Excepciones:
225 Excepciones:
225 Si el directorio no tiene el formato adecuado
226 Si el directorio no tiene el formato adecuado
226 """
227 """
227
228
228 basename = os.path.basename(folder)
229 basename = os.path.basename(folder)
229
230
230 if not isRadarFolder(basename):
231 if not isRadarFolder(basename):
231 print "The folder %s has not the rigth format" % folder
232 print "The folder %s has not the rigth format" % folder
232 return 0
233 return 0
233
234
234 if startDate and endDate:
235 if startDate and endDate:
235 thisDate = getDateFromRadarFolder(basename)
236 thisDate = getDateFromRadarFolder(basename)
236
237
237 if thisDate < startDate:
238 if thisDate < startDate:
238 return 0
239 return 0
239
240
240 if thisDate > endDate:
241 if thisDate > endDate:
241 return 0
242 return 0
242
243
243 return 1
244 return 1
244
245
245
246
246 def isFileInDateRange(filename, startDate=None, endDate=None):
247 def isFileInDateRange(filename, startDate=None, endDate=None):
247 """
248 """
248 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
249 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
249
250
250 Inputs:
251 Inputs:
251 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
252 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
252
253
253 Su formato deberia ser "?YYYYDDDsss"
254 Su formato deberia ser "?YYYYDDDsss"
254
255
255 siendo:
256 siendo:
256 YYYY : Anio (ejemplo 2015)
257 YYYY : Anio (ejemplo 2015)
257 DDD : Dia del anio (ejemplo 305)
258 DDD : Dia del anio (ejemplo 305)
258 sss : set
259 sss : set
259
260
260 startDate : fecha inicial del rango seleccionado en formato datetime.date
261 startDate : fecha inicial del rango seleccionado en formato datetime.date
261
262
262 endDate : fecha final del rango seleccionado en formato datetime.date
263 endDate : fecha final del rango seleccionado en formato datetime.date
263
264
264 Return:
265 Return:
265 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
266 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
266 fecha especificado, de lo contrario retorna False.
267 fecha especificado, de lo contrario retorna False.
267 Excepciones:
268 Excepciones:
268 Si el archivo no tiene el formato adecuado
269 Si el archivo no tiene el formato adecuado
269 """
270 """
270
271
271 basename = os.path.basename(filename)
272 basename = os.path.basename(filename)
272
273
273 if not isRadarFile(basename):
274 if not isRadarFile(basename):
274 print "The filename %s has not the rigth format" % filename
275 print "The filename %s has not the rigth format" % filename
275 return 0
276 return 0
276
277
277 if startDate and endDate:
278 if startDate and endDate:
278 thisDate = getDateFromRadarFile(basename)
279 thisDate = getDateFromRadarFile(basename)
279
280
280 if thisDate < startDate:
281 if thisDate < startDate:
281 return 0
282 return 0
282
283
283 if thisDate > endDate:
284 if thisDate > endDate:
284 return 0
285 return 0
285
286
286 return 1
287 return 1
287
288
288
289
289 def getFileFromSet(path, ext, set):
290 def getFileFromSet(path, ext, set):
290 validFilelist = []
291 validFilelist = []
291 fileList = os.listdir(path)
292 fileList = os.listdir(path)
292
293
293 # 0 1234 567 89A BCDE
294 # 0 1234 567 89A BCDE
294 # H YYYY DDD SSS .ext
295 # H YYYY DDD SSS .ext
295
296
296 for thisFile in fileList:
297 for thisFile in fileList:
297 try:
298 try:
298 year = int(thisFile[1:5])
299 year = int(thisFile[1:5])
299 doy = int(thisFile[5:8])
300 doy = int(thisFile[5:8])
300 except:
301 except:
301 continue
302 continue
302
303
303 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
304 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
304 continue
305 continue
305
306
306 validFilelist.append(thisFile)
307 validFilelist.append(thisFile)
307
308
308 myfile = fnmatch.filter(
309 myfile = fnmatch.filter(
309 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
310 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
310
311
311 if len(myfile) != 0:
312 if len(myfile) != 0:
312 return myfile[0]
313 return myfile[0]
313 else:
314 else:
314 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
315 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
315 print 'the filename %s does not exist' % filename
316 print 'the filename %s does not exist' % filename
316 print '...going to the last file: '
317 print '...going to the last file: '
317
318
318 if validFilelist:
319 if validFilelist:
319 validFilelist = sorted(validFilelist, key=str.lower)
320 validFilelist = sorted(validFilelist, key=str.lower)
320 return validFilelist[-1]
321 return validFilelist[-1]
321
322
322 return None
323 return None
323
324
324
325
325 def getlastFileFromPath(path, ext):
326 def getlastFileFromPath(path, ext):
326 """
327 """
327 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
328 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
328 al final de la depuracion devuelve el ultimo file de la lista que quedo.
329 al final de la depuracion devuelve el ultimo file de la lista que quedo.
329
330
330 Input:
331 Input:
331 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
332 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
332 ext : extension de los files contenidos en una carpeta
333 ext : extension de los files contenidos en una carpeta
333
334
334 Return:
335 Return:
335 El ultimo file de una determinada carpeta, no se considera el path.
336 El ultimo file de una determinada carpeta, no se considera el path.
336 """
337 """
337 validFilelist = []
338 validFilelist = []
338 fileList = os.listdir(path)
339 fileList = os.listdir(path)
339
340
340 # 0 1234 567 89A BCDE
341 # 0 1234 567 89A BCDE
341 # H YYYY DDD SSS .ext
342 # H YYYY DDD SSS .ext
342
343
343 for thisFile in fileList:
344 for thisFile in fileList:
344
345
345 year = thisFile[1:5]
346 year = thisFile[1:5]
346 if not isNumber(year):
347 if not isNumber(year):
347 continue
348 continue
348
349
349 doy = thisFile[5:8]
350 doy = thisFile[5:8]
350 if not isNumber(doy):
351 if not isNumber(doy):
351 continue
352 continue
352
353
353 year = int(year)
354 year = int(year)
354 doy = int(doy)
355 doy = int(doy)
355
356
356 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
357 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
357 continue
358 continue
358
359
359 validFilelist.append(thisFile)
360 validFilelist.append(thisFile)
360
361
361 if validFilelist:
362 if validFilelist:
362 validFilelist = sorted(validFilelist, key=str.lower)
363 validFilelist = sorted(validFilelist, key=str.lower)
363 return validFilelist[-1]
364 return validFilelist[-1]
364
365
365 return None
366 return None
366
367
367
368
368 def checkForRealPath(path, foldercounter, year, doy, set, ext):
369 def checkForRealPath(path, foldercounter, year, doy, set, ext):
369 """
370 """
370 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
371 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
371 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
372 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
372 el path exacto de un determinado file.
373 el path exacto de un determinado file.
373
374
374 Example :
375 Example :
375 nombre correcto del file es .../.../D2009307/P2009307367.ext
376 nombre correcto del file es .../.../D2009307/P2009307367.ext
376
377
377 Entonces la funcion prueba con las siguientes combinaciones
378 Entonces la funcion prueba con las siguientes combinaciones
378 .../.../y2009307367.ext
379 .../.../y2009307367.ext
379 .../.../Y2009307367.ext
380 .../.../Y2009307367.ext
380 .../.../x2009307/y2009307367.ext
381 .../.../x2009307/y2009307367.ext
381 .../.../x2009307/Y2009307367.ext
382 .../.../x2009307/Y2009307367.ext
382 .../.../X2009307/y2009307367.ext
383 .../.../X2009307/y2009307367.ext
383 .../.../X2009307/Y2009307367.ext
384 .../.../X2009307/Y2009307367.ext
384 siendo para este caso, la ultima combinacion de letras, identica al file buscado
385 siendo para este caso, la ultima combinacion de letras, identica al file buscado
385
386
386 Return:
387 Return:
387 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
388 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
388 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
389 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
389 para el filename
390 para el filename
390 """
391 """
391 fullfilename = None
392 fullfilename = None
392 find_flag = False
393 find_flag = False
393 filename = None
394 filename = None
394
395
395 prefixDirList = [None, 'd', 'D']
396 prefixDirList = [None, 'd', 'D']
396 if ext.lower() == ".r": # voltage
397 if ext.lower() == ".r": # voltage
397 prefixFileList = ['d', 'D']
398 prefixFileList = ['d', 'D']
398 elif ext.lower() == ".pdata": # spectra
399 elif ext.lower() == ".pdata": # spectra
399 prefixFileList = ['p', 'P']
400 prefixFileList = ['p', 'P']
400 else:
401 else:
401 return None, filename
402 return None, filename
402
403
403 # barrido por las combinaciones posibles
404 # barrido por las combinaciones posibles
404 for prefixDir in prefixDirList:
405 for prefixDir in prefixDirList:
405 thispath = path
406 thispath = path
406 if prefixDir != None:
407 if prefixDir != None:
407 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
408 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
408 if foldercounter == 0:
409 if foldercounter == 0:
409 thispath = os.path.join(path, "%s%04d%03d" %
410 thispath = os.path.join(path, "%s%04d%03d" %
410 (prefixDir, year, doy))
411 (prefixDir, year, doy))
411 else:
412 else:
412 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
413 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
413 prefixDir, year, doy, foldercounter))
414 prefixDir, year, doy, foldercounter))
414 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
415 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
415 # formo el nombre del file xYYYYDDDSSS.ext
416 # formo el nombre del file xYYYYDDDSSS.ext
416 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
417 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
417 fullfilename = os.path.join(
418 fullfilename = os.path.join(
418 thispath, filename) # formo el path completo
419 thispath, filename) # formo el path completo
419
420
420 if os.path.exists(fullfilename): # verifico que exista
421 if os.path.exists(fullfilename): # verifico que exista
421 find_flag = True
422 find_flag = True
422 break
423 break
423 if find_flag:
424 if find_flag:
424 break
425 break
425
426
426 if not(find_flag):
427 if not(find_flag):
427 return None, filename
428 return None, filename
428
429
429 return fullfilename, filename
430 return fullfilename, filename
430
431
431
432
432 def isRadarFolder(folder):
433 def isRadarFolder(folder):
433 try:
434 try:
434 year = int(folder[1:5])
435 year = int(folder[1:5])
435 doy = int(folder[5:8])
436 doy = int(folder[5:8])
436 except:
437 except:
437 return 0
438 return 0
438
439
439 return 1
440 return 1
440
441
441
442
442 def isRadarFile(file):
443 def isRadarFile(file):
443 try:
444 try:
444 year = int(file[1:5])
445 year = int(file[1:5])
445 doy = int(file[5:8])
446 doy = int(file[5:8])
446 set = int(file[8:11])
447 set = int(file[8:11])
447 except:
448 except:
448 return 0
449 return 0
449
450
450 return 1
451 return 1
451
452
452
453
453 def getDateFromRadarFile(file):
454 def getDateFromRadarFile(file):
454 try:
455 try:
455 year = int(file[1:5])
456 year = int(file[1:5])
456 doy = int(file[5:8])
457 doy = int(file[5:8])
457 set = int(file[8:11])
458 set = int(file[8:11])
458 except:
459 except:
459 return None
460 return None
460
461
461 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
462 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
462 return thisDate
463 return thisDate
463
464
464
465
465 def getDateFromRadarFolder(folder):
466 def getDateFromRadarFolder(folder):
466 try:
467 try:
467 year = int(folder[1:5])
468 year = int(folder[1:5])
468 doy = int(folder[5:8])
469 doy = int(folder[5:8])
469 except:
470 except:
470 return None
471 return None
471
472
472 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
473 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
473 return thisDate
474 return thisDate
474
475
475
476
476 class JRODataIO:
477 class JRODataIO:
477
478
478 c = 3E8
479 c = 3E8
479
480
480 isConfig = False
481 isConfig = False
481
482
482 basicHeaderObj = None
483 basicHeaderObj = None
483
484
484 systemHeaderObj = None
485 systemHeaderObj = None
485
486
486 radarControllerHeaderObj = None
487 radarControllerHeaderObj = None
487
488
488 processingHeaderObj = None
489 processingHeaderObj = None
489
490
490 dtype = None
491 dtype = None
491
492
492 pathList = []
493 pathList = []
493
494
494 filenameList = []
495 filenameList = []
495
496
496 filename = None
497 filename = None
497
498
498 ext = None
499 ext = None
499
500
500 flagIsNewFile = 1
501 flagIsNewFile = 1
501
502
502 flagDiscontinuousBlock = 0
503 flagDiscontinuousBlock = 0
503
504
504 flagIsNewBlock = 0
505 flagIsNewBlock = 0
505
506
506 fp = None
507 fp = None
507
508
508 firstHeaderSize = 0
509 firstHeaderSize = 0
509
510
510 basicHeaderSize = 24
511 basicHeaderSize = 24
511
512
512 versionFile = 1103
513 versionFile = 1103
513
514
514 fileSize = None
515 fileSize = None
515
516
516 # ippSeconds = None
517 # ippSeconds = None
517
518
518 fileSizeByHeader = None
519 fileSizeByHeader = None
519
520
520 fileIndex = None
521 fileIndex = None
521
522
522 profileIndex = None
523 profileIndex = None
523
524
524 blockIndex = None
525 blockIndex = None
525
526
526 nTotalBlocks = None
527 nTotalBlocks = None
527
528
528 maxTimeStep = 30
529 maxTimeStep = 30
529
530
530 lastUTTime = None
531 lastUTTime = None
531
532
532 datablock = None
533 datablock = None
533
534
534 dataOut = None
535 dataOut = None
535
536
536 blocksize = None
537 blocksize = None
537
538
538 getByBlock = False
539 getByBlock = False
539
540
540 def __init__(self):
541 def __init__(self):
541
542
542 raise NotImplementedError
543 raise NotImplementedError
543
544
544 def run(self):
545 def run(self):
545
546
546 raise NotImplementedError
547 raise NotImplementedError
547
548
548 def getDtypeWidth(self):
549 def getDtypeWidth(self):
549
550
550 dtype_index = get_dtype_index(self.dtype)
551 dtype_index = get_dtype_index(self.dtype)
551 dtype_width = get_dtype_width(dtype_index)
552 dtype_width = get_dtype_width(dtype_index)
552
553
553 return dtype_width
554 return dtype_width
554
555
555 def getAllowedArgs(self):
556 def getAllowedArgs(self):
556 if hasattr(self, '__attrs__'):
557 if hasattr(self, '__attrs__'):
557 return self.__attrs__
558 return self.__attrs__
558 else:
559 else:
559 return inspect.getargspec(self.run).args
560 return inspect.getargspec(self.run).args
560
561
561
562
562 class JRODataReader(JRODataIO):
563 class JRODataReader(JRODataIO):
563
564
564 online = 0
565 online = 0
565
566
566 realtime = 0
567 realtime = 0
567
568
568 nReadBlocks = 0
569 nReadBlocks = 0
569
570
570 delay = 10 # number of seconds waiting a new file
571 delay = 10 # number of seconds waiting a new file
571
572
572 nTries = 3 # quantity tries
573 nTries = 3 # quantity tries
573
574
574 nFiles = 3 # number of files for searching
575 nFiles = 3 # number of files for searching
575
576
576 path = None
577 path = None
577
578
578 foldercounter = 0
579 foldercounter = 0
579
580
580 flagNoMoreFiles = 0
581 flagNoMoreFiles = 0
581
582
582 datetimeList = []
583 datetimeList = []
583
584
584 __isFirstTimeOnline = 1
585 __isFirstTimeOnline = 1
585
586
586 __printInfo = True
587 __printInfo = True
587
588
588 profileIndex = None
589 profileIndex = None
589
590
590 nTxs = 1
591 nTxs = 1
591
592
592 txIndex = None
593 txIndex = None
593
594
594 # Added--------------------
595 # Added--------------------
595
596
596 selBlocksize = None
597 selBlocksize = None
597
598
598 selBlocktime = None
599 selBlocktime = None
599
600
600 def __init__(self):
601 def __init__(self):
601 """
602 """
602 This class is used to find data files
603 This class is used to find data files
603
604
604 Example:
605 Example:
605 reader = JRODataReader()
606 reader = JRODataReader()
606 fileList = reader.findDataFiles()
607 fileList = reader.findDataFiles()
607
608
608 """
609 """
609 pass
610 pass
610
611
611 def createObjByDefault(self):
612 def createObjByDefault(self):
612 """
613 """
613
614
614 """
615 """
615 raise NotImplementedError
616 raise NotImplementedError
616
617
617 def getBlockDimension(self):
618 def getBlockDimension(self):
618
619
619 raise NotImplementedError
620 raise NotImplementedError
620
621
621 def searchFilesOffLine(self,
622 def searchFilesOffLine(self,
622 path,
623 path,
623 startDate=None,
624 startDate=None,
624 endDate=None,
625 endDate=None,
625 startTime=datetime.time(0, 0, 0),
626 startTime=datetime.time(0, 0, 0),
626 endTime=datetime.time(23, 59, 59),
627 endTime=datetime.time(23, 59, 59),
627 set=None,
628 set=None,
628 expLabel='',
629 expLabel='',
629 ext='.r',
630 ext='.r',
630 cursor=None,
631 cursor=None,
631 skip=None,
632 skip=None,
632 walk=True):
633 walk=True):
633
634
634 self.filenameList = []
635 self.filenameList = []
635 self.datetimeList = []
636 self.datetimeList = []
636
637
637 pathList = []
638 pathList = []
638
639
639 dateList, pathList = self.findDatafiles(
640 dateList, pathList = self.findDatafiles(
640 path, startDate, endDate, expLabel, ext, walk, include_path=True)
641 path, startDate, endDate, expLabel, ext, walk, include_path=True)
641
642
642 if dateList == []:
643 if dateList == []:
643 return [], []
644 return [], []
644
645
645 if len(dateList) > 1:
646 if len(dateList) > 1:
646 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
647 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
647 else:
648 else:
648 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
649 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
649
650
650 filenameList = []
651 filenameList = []
651 datetimeList = []
652 datetimeList = []
652
653
653 for thisPath in pathList:
654 for thisPath in pathList:
654
655
655 fileList = glob.glob1(thisPath, "*%s" % ext)
656 fileList = glob.glob1(thisPath, "*%s" % ext)
656 fileList.sort()
657 fileList.sort()
657
658
658 skippedFileList = []
659 for file in fileList:
659
660 if cursor is not None and skip is not None:
661
662 if skip == 0:
663 skippedFileList = []
664 else:
665 skippedFileList = fileList[cursor *
666 skip: cursor * skip + skip]
667
668 else:
669 skippedFileList = fileList
670
671 for file in skippedFileList:
672
660
673 filename = os.path.join(thisPath, file)
661 filename = os.path.join(thisPath, file)
674
662
675 if not isFileInDateRange(filename, startDate, endDate):
663 if not isFileInDateRange(filename, startDate, endDate):
676 continue
664 continue
677
665
678 thisDatetime = isFileInTimeRange(
666 thisDatetime = isFileInTimeRange(
679 filename, startDate, endDate, startTime, endTime)
667 filename, startDate, endDate, startTime, endTime)
680
668
681 if not(thisDatetime):
669 if not(thisDatetime):
682 continue
670 continue
683
671
684 filenameList.append(filename)
672 filenameList.append(filename)
685 datetimeList.append(thisDatetime)
673 datetimeList.append(thisDatetime)
686
674
675 if cursor is not None and skip is not None:
676 filenameList = filenameList[cursor * skip:cursor * skip + skip]
677 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
678
687 if not(filenameList):
679 if not(filenameList):
688 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
680 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
689 return [], []
681 return [], []
690
682
691 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
683 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
692 print
693
684
694 # for i in range(len(filenameList)):
685 # for i in range(len(filenameList)):
695 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
686 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
696
687
697 self.filenameList = filenameList
688 self.filenameList = filenameList
698 self.datetimeList = datetimeList
689 self.datetimeList = datetimeList
699
690
700 return pathList, filenameList
691 return pathList, filenameList
701
692
702 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
693 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
703 """
694 """
704 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
695 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
705 devuelve el archivo encontrado ademas de otros datos.
696 devuelve el archivo encontrado ademas de otros datos.
706
697
707 Input:
698 Input:
708 path : carpeta donde estan contenidos los files que contiene data
699 path : carpeta donde estan contenidos los files que contiene data
709
700
710 expLabel : Nombre del subexperimento (subfolder)
701 expLabel : Nombre del subexperimento (subfolder)
711
702
712 ext : extension de los files
703 ext : extension de los files
713
704
714 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
705 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
715
706
716 Return:
707 Return:
717 directory : eL directorio donde esta el file encontrado
708 directory : eL directorio donde esta el file encontrado
718 filename : el ultimo file de una determinada carpeta
709 filename : el ultimo file de una determinada carpeta
719 year : el anho
710 year : el anho
720 doy : el numero de dia del anho
711 doy : el numero de dia del anho
721 set : el set del archivo
712 set : el set del archivo
722
713
723
714
724 """
715 """
725 if not os.path.isdir(path):
716 if not os.path.isdir(path):
726 return None, None, None, None, None, None
717 return None, None, None, None, None, None
727
718
728 dirList = []
719 dirList = []
729
720
730 if not walk:
721 if not walk:
731 fullpath = path
722 fullpath = path
732 foldercounter = 0
723 foldercounter = 0
733 else:
724 else:
734 # Filtra solo los directorios
725 # Filtra solo los directorios
735 for thisPath in os.listdir(path):
726 for thisPath in os.listdir(path):
736 if not os.path.isdir(os.path.join(path, thisPath)):
727 if not os.path.isdir(os.path.join(path, thisPath)):
737 continue
728 continue
738 if not isRadarFolder(thisPath):
729 if not isRadarFolder(thisPath):
739 continue
730 continue
740
731
741 dirList.append(thisPath)
732 dirList.append(thisPath)
742
733
743 if not(dirList):
734 if not(dirList):
744 return None, None, None, None, None, None
735 return None, None, None, None, None, None
745
736
746 dirList = sorted(dirList, key=str.lower)
737 dirList = sorted(dirList, key=str.lower)
747
738
748 doypath = dirList[-1]
739 doypath = dirList[-1]
749 foldercounter = int(doypath.split('_')[1]) if len(
740 foldercounter = int(doypath.split('_')[1]) if len(
750 doypath.split('_')) > 1 else 0
741 doypath.split('_')) > 1 else 0
751 fullpath = os.path.join(path, doypath, expLabel)
742 fullpath = os.path.join(path, doypath, expLabel)
752
743
753 print "[Reading] %s folder was found: " % (fullpath)
744 print "[Reading] %s folder was found: " % (fullpath)
754
745
755 if set == None:
746 if set == None:
756 filename = getlastFileFromPath(fullpath, ext)
747 filename = getlastFileFromPath(fullpath, ext)
757 else:
748 else:
758 filename = getFileFromSet(fullpath, ext, set)
749 filename = getFileFromSet(fullpath, ext, set)
759
750
760 if not(filename):
751 if not(filename):
761 return None, None, None, None, None, None
752 return None, None, None, None, None, None
762
753
763 print "[Reading] %s file was found" % (filename)
754 print "[Reading] %s file was found" % (filename)
764
755
765 if not(self.__verifyFile(os.path.join(fullpath, filename))):
756 if not(self.__verifyFile(os.path.join(fullpath, filename))):
766 return None, None, None, None, None, None
757 return None, None, None, None, None, None
767
758
768 year = int(filename[1:5])
759 year = int(filename[1:5])
769 doy = int(filename[5:8])
760 doy = int(filename[5:8])
770 set = int(filename[8:11])
761 set = int(filename[8:11])
771
762
772 return fullpath, foldercounter, filename, year, doy, set
763 return fullpath, foldercounter, filename, year, doy, set
773
764
774 def __setNextFileOffline(self):
765 def __setNextFileOffline(self):
775
766
776 idFile = self.fileIndex
767 idFile = self.fileIndex
777
768
778 while (True):
769 while (True):
779 idFile += 1
770 idFile += 1
780 if not(idFile < len(self.filenameList)):
771 if not(idFile < len(self.filenameList)):
781 self.flagNoMoreFiles = 1
772 self.flagNoMoreFiles = 1
782 # print "[Reading] No more Files"
773 # print "[Reading] No more Files"
783 return 0
774 return 0
784
775
785 filename = self.filenameList[idFile]
776 filename = self.filenameList[idFile]
786
777
787 if not(self.__verifyFile(filename)):
778 if not(self.__verifyFile(filename)):
788 continue
779 continue
789
780
790 fileSize = os.path.getsize(filename)
781 fileSize = os.path.getsize(filename)
791 fp = open(filename, 'rb')
782 fp = open(filename, 'rb')
792 break
783 break
793
784
794 self.flagIsNewFile = 1
785 self.flagIsNewFile = 1
795 self.fileIndex = idFile
786 self.fileIndex = idFile
796 self.filename = filename
787 self.filename = filename
797 self.fileSize = fileSize
788 self.fileSize = fileSize
798 self.fp = fp
789 self.fp = fp
799
790
800 # print "[Reading] Setting the file: %s"%self.filename
791 # print "[Reading] Setting the file: %s"%self.filename
801
792
802 return 1
793 return 1
803
794
804 def __setNextFileOnline(self):
795 def __setNextFileOnline(self):
805 """
796 """
806 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
797 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
807 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
798 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
808 siguientes.
799 siguientes.
809
800
810 Affected:
801 Affected:
811 self.flagIsNewFile
802 self.flagIsNewFile
812 self.filename
803 self.filename
813 self.fileSize
804 self.fileSize
814 self.fp
805 self.fp
815 self.set
806 self.set
816 self.flagNoMoreFiles
807 self.flagNoMoreFiles
817
808
818 Return:
809 Return:
819 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
810 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
820 1 : si el file fue abierto con exito y esta listo a ser leido
811 1 : si el file fue abierto con exito y esta listo a ser leido
821
812
822 Excepciones:
813 Excepciones:
823 Si un determinado file no puede ser abierto
814 Si un determinado file no puede ser abierto
824 """
815 """
825 nFiles = 0
816 nFiles = 0
826 fileOk_flag = False
817 fileOk_flag = False
827 firstTime_flag = True
818 firstTime_flag = True
828
819
829 self.set += 1
820 self.set += 1
830
821
831 if self.set > 999:
822 if self.set > 999:
832 self.set = 0
823 self.set = 0
833 self.foldercounter += 1
824 self.foldercounter += 1
834
825
835 # busca el 1er file disponible
826 # busca el 1er file disponible
836 fullfilename, filename = checkForRealPath(
827 fullfilename, filename = checkForRealPath(
837 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
828 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
838 if fullfilename:
829 if fullfilename:
839 if self.__verifyFile(fullfilename, False):
830 if self.__verifyFile(fullfilename, False):
840 fileOk_flag = True
831 fileOk_flag = True
841
832
842 # si no encuentra un file entonces espera y vuelve a buscar
833 # si no encuentra un file entonces espera y vuelve a buscar
843 if not(fileOk_flag):
834 if not(fileOk_flag):
844 # busco en los siguientes self.nFiles+1 files posibles
835 # busco en los siguientes self.nFiles+1 files posibles
845 for nFiles in range(self.nFiles + 1):
836 for nFiles in range(self.nFiles + 1):
846
837
847 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
838 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
848 tries = self.nTries
839 tries = self.nTries
849 else:
840 else:
850 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
841 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
851
842
852 for nTries in range(tries):
843 for nTries in range(tries):
853 if firstTime_flag:
844 if firstTime_flag:
854 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
845 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
855 sleep(self.delay)
846 sleep(self.delay)
856 else:
847 else:
857 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
848 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
858
849
859 fullfilename, filename = checkForRealPath(
850 fullfilename, filename = checkForRealPath(
860 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
851 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
861 if fullfilename:
852 if fullfilename:
862 if self.__verifyFile(fullfilename):
853 if self.__verifyFile(fullfilename):
863 fileOk_flag = True
854 fileOk_flag = True
864 break
855 break
865
856
866 if fileOk_flag:
857 if fileOk_flag:
867 break
858 break
868
859
869 firstTime_flag = False
860 firstTime_flag = False
870
861
871 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
862 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
872 self.set += 1
863 self.set += 1
873
864
874 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
865 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
875 if nFiles == (self.nFiles - 1):
866 if nFiles == (self.nFiles - 1):
876 self.set = 0
867 self.set = 0
877 self.doy += 1
868 self.doy += 1
878 self.foldercounter = 0
869 self.foldercounter = 0
879
870
880 if fileOk_flag:
871 if fileOk_flag:
881 self.fileSize = os.path.getsize(fullfilename)
872 self.fileSize = os.path.getsize(fullfilename)
882 self.filename = fullfilename
873 self.filename = fullfilename
883 self.flagIsNewFile = 1
874 self.flagIsNewFile = 1
884 if self.fp != None:
875 if self.fp != None:
885 self.fp.close()
876 self.fp.close()
886 self.fp = open(fullfilename, 'rb')
877 self.fp = open(fullfilename, 'rb')
887 self.flagNoMoreFiles = 0
878 self.flagNoMoreFiles = 0
888 # print '[Reading] Setting the file: %s' % fullfilename
879 # print '[Reading] Setting the file: %s' % fullfilename
889 else:
880 else:
890 self.fileSize = 0
881 self.fileSize = 0
891 self.filename = None
882 self.filename = None
892 self.flagIsNewFile = 0
883 self.flagIsNewFile = 0
893 self.fp = None
884 self.fp = None
894 self.flagNoMoreFiles = 1
885 self.flagNoMoreFiles = 1
895 # print '[Reading] No more files to read'
886 # print '[Reading] No more files to read'
896
887
897 return fileOk_flag
888 return fileOk_flag
898
889
899 def setNextFile(self):
890 def setNextFile(self):
900 if self.fp != None:
891 if self.fp != None:
901 self.fp.close()
892 self.fp.close()
902
893
903 if self.online:
894 if self.online:
904 newFile = self.__setNextFileOnline()
895 newFile = self.__setNextFileOnline()
905 else:
896 else:
906 newFile = self.__setNextFileOffline()
897 newFile = self.__setNextFileOffline()
907
898
908 if not(newFile):
899 if not(newFile):
909 print '[Reading] No more files to read'
900 print '[Reading] No more files to read'
910 return 0
901 return 0
911
902
912 if self.verbose:
903 if self.verbose:
913 print '[Reading] Setting the file: %s' % self.filename
904 print '[Reading] Setting the file: %s' % self.filename
914
905
915 self.__readFirstHeader()
906 self.__readFirstHeader()
916 self.nReadBlocks = 0
907 self.nReadBlocks = 0
917 return 1
908 return 1
918
909
919 def __waitNewBlock(self):
910 def __waitNewBlock(self):
920 """
911 """
921 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
912 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
922
913
923 Si el modo de lectura es OffLine siempre retorn 0
914 Si el modo de lectura es OffLine siempre retorn 0
924 """
915 """
925 if not self.online:
916 if not self.online:
926 return 0
917 return 0
927
918
928 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
919 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
929 return 0
920 return 0
930
921
931 currentPointer = self.fp.tell()
922 currentPointer = self.fp.tell()
932
923
933 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
924 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
934
925
935 for nTries in range(self.nTries):
926 for nTries in range(self.nTries):
936
927
937 self.fp.close()
928 self.fp.close()
938 self.fp = open(self.filename, 'rb')
929 self.fp = open(self.filename, 'rb')
939 self.fp.seek(currentPointer)
930 self.fp.seek(currentPointer)
940
931
941 self.fileSize = os.path.getsize(self.filename)
932 self.fileSize = os.path.getsize(self.filename)
942 currentSize = self.fileSize - currentPointer
933 currentSize = self.fileSize - currentPointer
943
934
944 if (currentSize >= neededSize):
935 if (currentSize >= neededSize):
945 self.basicHeaderObj.read(self.fp)
936 self.basicHeaderObj.read(self.fp)
946 return 1
937 return 1
947
938
948 if self.fileSize == self.fileSizeByHeader:
939 if self.fileSize == self.fileSizeByHeader:
949 # self.flagEoF = True
940 # self.flagEoF = True
950 return 0
941 return 0
951
942
952 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
943 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
953 sleep(self.delay)
944 sleep(self.delay)
954
945
955 return 0
946 return 0
956
947
957 def waitDataBlock(self, pointer_location):
948 def waitDataBlock(self, pointer_location):
958
949
959 currentPointer = pointer_location
950 currentPointer = pointer_location
960
951
961 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
952 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
962
953
963 for nTries in range(self.nTries):
954 for nTries in range(self.nTries):
964 self.fp.close()
955 self.fp.close()
965 self.fp = open(self.filename, 'rb')
956 self.fp = open(self.filename, 'rb')
966 self.fp.seek(currentPointer)
957 self.fp.seek(currentPointer)
967
958
968 self.fileSize = os.path.getsize(self.filename)
959 self.fileSize = os.path.getsize(self.filename)
969 currentSize = self.fileSize - currentPointer
960 currentSize = self.fileSize - currentPointer
970
961
971 if (currentSize >= neededSize):
962 if (currentSize >= neededSize):
972 return 1
963 return 1
973
964
974 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
965 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
975 sleep(self.delay)
966 sleep(self.delay)
976
967
977 return 0
968 return 0
978
969
979 def __jumpToLastBlock(self):
970 def __jumpToLastBlock(self):
980
971
981 if not(self.__isFirstTimeOnline):
972 if not(self.__isFirstTimeOnline):
982 return
973 return
983
974
984 csize = self.fileSize - self.fp.tell()
975 csize = self.fileSize - self.fp.tell()
985 blocksize = self.processingHeaderObj.blockSize
976 blocksize = self.processingHeaderObj.blockSize
986
977
987 # salta el primer bloque de datos
978 # salta el primer bloque de datos
988 if csize > self.processingHeaderObj.blockSize:
979 if csize > self.processingHeaderObj.blockSize:
989 self.fp.seek(self.fp.tell() + blocksize)
980 self.fp.seek(self.fp.tell() + blocksize)
990 else:
981 else:
991 return
982 return
992
983
993 csize = self.fileSize - self.fp.tell()
984 csize = self.fileSize - self.fp.tell()
994 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
985 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 while True:
986 while True:
996
987
997 if self.fp.tell() < self.fileSize:
988 if self.fp.tell() < self.fileSize:
998 self.fp.seek(self.fp.tell() + neededsize)
989 self.fp.seek(self.fp.tell() + neededsize)
999 else:
990 else:
1000 self.fp.seek(self.fp.tell() - neededsize)
991 self.fp.seek(self.fp.tell() - neededsize)
1001 break
992 break
1002
993
1003 # csize = self.fileSize - self.fp.tell()
994 # csize = self.fileSize - self.fp.tell()
1004 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1005 # factor = int(csize/neededsize)
996 # factor = int(csize/neededsize)
1006 # if factor > 0:
997 # if factor > 0:
1007 # self.fp.seek(self.fp.tell() + factor*neededsize)
998 # self.fp.seek(self.fp.tell() + factor*neededsize)
1008
999
1009 self.flagIsNewFile = 0
1000 self.flagIsNewFile = 0
1010 self.__isFirstTimeOnline = 0
1001 self.__isFirstTimeOnline = 0
1011
1002
1012 def __setNewBlock(self):
1003 def __setNewBlock(self):
1013 # if self.server is None:
1004 # if self.server is None:
1014 if self.fp == None:
1005 if self.fp == None:
1015 return 0
1006 return 0
1016
1007
1017 # if self.online:
1008 # if self.online:
1018 # self.__jumpToLastBlock()
1009 # self.__jumpToLastBlock()
1019
1010
1020 if self.flagIsNewFile:
1011 if self.flagIsNewFile:
1021 self.lastUTTime = self.basicHeaderObj.utc
1012 self.lastUTTime = self.basicHeaderObj.utc
1022 return 1
1013 return 1
1023
1014
1024 if self.realtime:
1015 if self.realtime:
1025 self.flagDiscontinuousBlock = 1
1016 self.flagDiscontinuousBlock = 1
1026 if not(self.setNextFile()):
1017 if not(self.setNextFile()):
1027 return 0
1018 return 0
1028 else:
1019 else:
1029 return 1
1020 return 1
1030 # if self.server is None:
1021 # if self.server is None:
1031 currentSize = self.fileSize - self.fp.tell()
1022 currentSize = self.fileSize - self.fp.tell()
1032 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1023 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1033 if (currentSize >= neededSize):
1024 if (currentSize >= neededSize):
1034 self.basicHeaderObj.read(self.fp)
1025 self.basicHeaderObj.read(self.fp)
1035 self.lastUTTime = self.basicHeaderObj.utc
1026 self.lastUTTime = self.basicHeaderObj.utc
1036 return 1
1027 return 1
1037 # else:
1028 # else:
1038 # self.basicHeaderObj.read(self.zHeader)
1029 # self.basicHeaderObj.read(self.zHeader)
1039 # self.lastUTTime = self.basicHeaderObj.utc
1030 # self.lastUTTime = self.basicHeaderObj.utc
1040 # return 1
1031 # return 1
1041 if self.__waitNewBlock():
1032 if self.__waitNewBlock():
1042 self.lastUTTime = self.basicHeaderObj.utc
1033 self.lastUTTime = self.basicHeaderObj.utc
1043 return 1
1034 return 1
1044 # if self.server is None:
1035 # if self.server is None:
1045 if not(self.setNextFile()):
1036 if not(self.setNextFile()):
1046 return 0
1037 return 0
1047
1038
1048 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1039 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1049 self.lastUTTime = self.basicHeaderObj.utc
1040 self.lastUTTime = self.basicHeaderObj.utc
1050
1041
1051 self.flagDiscontinuousBlock = 0
1042 self.flagDiscontinuousBlock = 0
1052
1043
1053 if deltaTime > self.maxTimeStep:
1044 if deltaTime > self.maxTimeStep:
1054 self.flagDiscontinuousBlock = 1
1045 self.flagDiscontinuousBlock = 1
1055
1046
1056 return 1
1047 return 1
1057
1048
1058 def readNextBlock(self):
1049 def readNextBlock(self):
1059
1050
1060 # Skip block out of startTime and endTime
1051 # Skip block out of startTime and endTime
1061 while True:
1052 while True:
1062 if not(self.__setNewBlock()):
1053 if not(self.__setNewBlock()):
1063 return 0
1054 return 0
1064
1055
1065 if not(self.readBlock()):
1056 if not(self.readBlock()):
1066 return 0
1057 return 0
1067
1058
1068 self.getBasicHeader()
1059 self.getBasicHeader()
1069 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1060 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1070 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1061 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1071 self.processingHeaderObj.dataBlocksPerFile,
1062 self.processingHeaderObj.dataBlocksPerFile,
1072 self.dataOut.datatime.ctime())
1063 self.dataOut.datatime.ctime())
1073 continue
1064 continue
1074
1065
1075 break
1066 break
1076
1067
1077 if self.verbose:
1068 if self.verbose:
1078 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1069 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1079 self.processingHeaderObj.dataBlocksPerFile,
1070 self.processingHeaderObj.dataBlocksPerFile,
1080 self.dataOut.datatime.ctime())
1071 self.dataOut.datatime.ctime())
1081 return 1
1072 return 1
1082
1073
1083 def __readFirstHeader(self):
1074 def __readFirstHeader(self):
1084
1075
1085 self.basicHeaderObj.read(self.fp)
1076 self.basicHeaderObj.read(self.fp)
1086 self.systemHeaderObj.read(self.fp)
1077 self.systemHeaderObj.read(self.fp)
1087 self.radarControllerHeaderObj.read(self.fp)
1078 self.radarControllerHeaderObj.read(self.fp)
1088 self.processingHeaderObj.read(self.fp)
1079 self.processingHeaderObj.read(self.fp)
1089
1080
1090 self.firstHeaderSize = self.basicHeaderObj.size
1081 self.firstHeaderSize = self.basicHeaderObj.size
1091
1082
1092 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1083 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1093 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1084 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1094 if datatype == 0:
1085 if datatype == 0:
1095 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1086 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1096 elif datatype == 1:
1087 elif datatype == 1:
1097 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1088 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1098 elif datatype == 2:
1089 elif datatype == 2:
1099 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1090 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1100 elif datatype == 3:
1091 elif datatype == 3:
1101 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1092 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1102 elif datatype == 4:
1093 elif datatype == 4:
1103 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1094 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1104 elif datatype == 5:
1095 elif datatype == 5:
1105 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1096 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1106 else:
1097 else:
1107 raise ValueError, 'Data type was not defined'
1098 raise ValueError, 'Data type was not defined'
1108
1099
1109 self.dtype = datatype_str
1100 self.dtype = datatype_str
1110 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1101 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1111 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1102 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1112 self.firstHeaderSize + self.basicHeaderSize * \
1103 self.firstHeaderSize + self.basicHeaderSize * \
1113 (self.processingHeaderObj.dataBlocksPerFile - 1)
1104 (self.processingHeaderObj.dataBlocksPerFile - 1)
1114 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1105 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1115 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1106 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1116 self.getBlockDimension()
1107 self.getBlockDimension()
1117
1108
1118 def __verifyFile(self, filename, msgFlag=True):
1109 def __verifyFile(self, filename, msgFlag=True):
1119
1110
1120 msg = None
1111 msg = None
1121
1112
1122 try:
1113 try:
1123 fp = open(filename, 'rb')
1114 fp = open(filename, 'rb')
1124 except IOError:
1115 except IOError:
1125
1116
1126 if msgFlag:
1117 if msgFlag:
1127 print "[Reading] File %s can't be opened" % (filename)
1118 print "[Reading] File %s can't be opened" % (filename)
1128
1119
1129 return False
1120 return False
1130
1121
1131 currentPosition = fp.tell()
1122 currentPosition = fp.tell()
1132 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1123 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1133
1124
1134 if neededSize == 0:
1125 if neededSize == 0:
1135 basicHeaderObj = BasicHeader(LOCALTIME)
1126 basicHeaderObj = BasicHeader(LOCALTIME)
1136 systemHeaderObj = SystemHeader()
1127 systemHeaderObj = SystemHeader()
1137 radarControllerHeaderObj = RadarControllerHeader()
1128 radarControllerHeaderObj = RadarControllerHeader()
1138 processingHeaderObj = ProcessingHeader()
1129 processingHeaderObj = ProcessingHeader()
1139
1130
1140 if not(basicHeaderObj.read(fp)):
1131 if not(basicHeaderObj.read(fp)):
1141 fp.close()
1132 fp.close()
1142 return False
1133 return False
1143
1134
1144 if not(systemHeaderObj.read(fp)):
1135 if not(systemHeaderObj.read(fp)):
1145 fp.close()
1136 fp.close()
1146 return False
1137 return False
1147
1138
1148 if not(radarControllerHeaderObj.read(fp)):
1139 if not(radarControllerHeaderObj.read(fp)):
1149 fp.close()
1140 fp.close()
1150 return False
1141 return False
1151
1142
1152 if not(processingHeaderObj.read(fp)):
1143 if not(processingHeaderObj.read(fp)):
1153 fp.close()
1144 fp.close()
1154 return False
1145 return False
1155
1146
1156 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1147 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1157 else:
1148 else:
1158 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1149 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1159
1150
1160 fp.close()
1151 fp.close()
1161
1152
1162 fileSize = os.path.getsize(filename)
1153 fileSize = os.path.getsize(filename)
1163 currentSize = fileSize - currentPosition
1154 currentSize = fileSize - currentPosition
1164
1155
1165 if currentSize < neededSize:
1156 if currentSize < neededSize:
1166 if msgFlag and (msg != None):
1157 if msgFlag and (msg != None):
1167 print msg
1158 print msg
1168 return False
1159 return False
1169
1160
1170 return True
1161 return True
1171
1162
1172 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1163 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1173
1164
1174 path_empty = True
1165 path_empty = True
1175
1166
1176 dateList = []
1167 dateList = []
1177 pathList = []
1168 pathList = []
1178
1169
1179 multi_path = path.split(',')
1170 multi_path = path.split(',')
1180
1171
1181 if not walk:
1172 if not walk:
1182
1173
1183 for single_path in multi_path:
1174 for single_path in multi_path:
1184
1175
1185 if not os.path.isdir(single_path):
1176 if not os.path.isdir(single_path):
1186 continue
1177 continue
1187
1178
1188 fileList = glob.glob1(single_path, "*" + ext)
1179 fileList = glob.glob1(single_path, "*" + ext)
1189
1180
1190 if not fileList:
1181 if not fileList:
1191 continue
1182 continue
1192
1183
1193 path_empty = False
1184 path_empty = False
1194
1185
1195 fileList.sort()
1186 fileList.sort()
1196
1187
1197 for thisFile in fileList:
1188 for thisFile in fileList:
1198
1189
1199 if not os.path.isfile(os.path.join(single_path, thisFile)):
1190 if not os.path.isfile(os.path.join(single_path, thisFile)):
1200 continue
1191 continue
1201
1192
1202 if not isRadarFile(thisFile):
1193 if not isRadarFile(thisFile):
1203 continue
1194 continue
1204
1195
1205 if not isFileInDateRange(thisFile, startDate, endDate):
1196 if not isFileInDateRange(thisFile, startDate, endDate):
1206 continue
1197 continue
1207
1198
1208 thisDate = getDateFromRadarFile(thisFile)
1199 thisDate = getDateFromRadarFile(thisFile)
1209
1200
1210 if thisDate in dateList:
1201 if thisDate in dateList:
1211 continue
1202 continue
1212
1203
1213 dateList.append(thisDate)
1204 dateList.append(thisDate)
1214 pathList.append(single_path)
1205 pathList.append(single_path)
1215
1206
1216 else:
1207 else:
1217 for single_path in multi_path:
1208 for single_path in multi_path:
1218
1209
1219 if not os.path.isdir(single_path):
1210 if not os.path.isdir(single_path):
1220 continue
1211 continue
1221
1212
1222 dirList = []
1213 dirList = []
1223
1214
1224 for thisPath in os.listdir(single_path):
1215 for thisPath in os.listdir(single_path):
1225
1216
1226 if not os.path.isdir(os.path.join(single_path, thisPath)):
1217 if not os.path.isdir(os.path.join(single_path, thisPath)):
1227 continue
1218 continue
1228
1219
1229 if not isRadarFolder(thisPath):
1220 if not isRadarFolder(thisPath):
1230 continue
1221 continue
1231
1222
1232 if not isFolderInDateRange(thisPath, startDate, endDate):
1223 if not isFolderInDateRange(thisPath, startDate, endDate):
1233 continue
1224 continue
1234
1225
1235 dirList.append(thisPath)
1226 dirList.append(thisPath)
1236
1227
1237 if not dirList:
1228 if not dirList:
1238 continue
1229 continue
1239
1230
1240 dirList.sort()
1231 dirList.sort()
1241
1232
1242 for thisDir in dirList:
1233 for thisDir in dirList:
1243
1234
1244 datapath = os.path.join(single_path, thisDir, expLabel)
1235 datapath = os.path.join(single_path, thisDir, expLabel)
1245 fileList = glob.glob1(datapath, "*" + ext)
1236 fileList = glob.glob1(datapath, "*" + ext)
1246
1237
1247 if not fileList:
1238 if not fileList:
1248 continue
1239 continue
1249
1240
1250 path_empty = False
1241 path_empty = False
1251
1242
1252 thisDate = getDateFromRadarFolder(thisDir)
1243 thisDate = getDateFromRadarFolder(thisDir)
1253
1244
1254 pathList.append(datapath)
1245 pathList.append(datapath)
1255 dateList.append(thisDate)
1246 dateList.append(thisDate)
1256
1247
1257 dateList.sort()
1248 dateList.sort()
1258
1249
1259 if walk:
1250 if walk:
1260 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1251 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1261 else:
1252 else:
1262 pattern_path = multi_path[0]
1253 pattern_path = multi_path[0]
1263
1254
1264 if path_empty:
1255 if path_empty:
1265 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1256 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1266 else:
1257 else:
1267 if not dateList:
1258 if not dateList:
1268 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1259 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1269
1260
1270 if include_path:
1261 if include_path:
1271 return dateList, pathList
1262 return dateList, pathList
1272
1263
1273 return dateList
1264 return dateList
1274
1265
1275 def setup(self,
1266 def setup(self,
1276 path=None,
1267 path=None,
1277 startDate=None,
1268 startDate=None,
1278 endDate=None,
1269 endDate=None,
1279 startTime=datetime.time(0, 0, 0),
1270 startTime=datetime.time(0, 0, 0),
1280 endTime=datetime.time(23, 59, 59),
1271 endTime=datetime.time(23, 59, 59),
1281 set=None,
1272 set=None,
1282 expLabel="",
1273 expLabel="",
1283 ext=None,
1274 ext=None,
1284 online=False,
1275 online=False,
1285 delay=60,
1276 delay=60,
1286 walk=True,
1277 walk=True,
1287 getblock=False,
1278 getblock=False,
1288 nTxs=1,
1279 nTxs=1,
1289 realtime=False,
1280 realtime=False,
1290 blocksize=None,
1281 blocksize=None,
1291 blocktime=None,
1282 blocktime=None,
1292 skip=None,
1283 skip=None,
1293 cursor=None,
1284 cursor=None,
1294 warnings=True,
1285 warnings=True,
1295 verbose=True,
1286 verbose=True,
1296 server=None,
1287 server=None,
1297 format=None,
1288 format=None,
1298 oneDDict=None,
1289 oneDDict=None,
1299 twoDDict=None,
1290 twoDDict=None,
1300 ind2DList=None):
1291 ind2DList=None):
1301 if server is not None:
1292 if server is not None:
1302 if 'tcp://' in server:
1293 if 'tcp://' in server:
1303 address = server
1294 address = server
1304 else:
1295 else:
1305 address = 'ipc:///tmp/%s' % server
1296 address = 'ipc:///tmp/%s' % server
1306 self.server = address
1297 self.server = address
1307 self.context = zmq.Context()
1298 self.context = zmq.Context()
1308 self.receiver = self.context.socket(zmq.PULL)
1299 self.receiver = self.context.socket(zmq.PULL)
1309 self.receiver.connect(self.server)
1300 self.receiver.connect(self.server)
1310 time.sleep(0.5)
1301 time.sleep(0.5)
1311 print '[Starting] ReceiverData from {}'.format(self.server)
1302 print '[Starting] ReceiverData from {}'.format(self.server)
1312 else:
1303 else:
1313 self.server = None
1304 self.server = None
1314 if path == None:
1305 if path == None:
1315 raise ValueError, "[Reading] The path is not valid"
1306 raise ValueError, "[Reading] The path is not valid"
1316
1307
1317 if ext == None:
1308 if ext == None:
1318 ext = self.ext
1309 ext = self.ext
1319
1310
1320 if online:
1311 if online:
1321 print "[Reading] Searching files in online mode..."
1312 print "[Reading] Searching files in online mode..."
1322
1313
1323 for nTries in range(self.nTries):
1314 for nTries in range(self.nTries):
1324 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1315 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1325 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1316 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1326
1317
1327 if fullpath:
1318 if fullpath:
1328 break
1319 break
1329
1320
1330 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1321 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1331 sleep(self.delay)
1322 sleep(self.delay)
1332
1323
1333 if not(fullpath):
1324 if not(fullpath):
1334 print "[Reading] There 'isn't any valid file in %s" % path
1325 print "[Reading] There 'isn't any valid file in %s" % path
1335 return
1326 return
1336
1327
1337 self.year = year
1328 self.year = year
1338 self.doy = doy
1329 self.doy = doy
1339 self.set = set - 1
1330 self.set = set - 1
1340 self.path = path
1331 self.path = path
1341 self.foldercounter = foldercounter
1332 self.foldercounter = foldercounter
1342 last_set = None
1333 last_set = None
1343 else:
1334 else:
1344 print "[Reading] Searching files in offline mode ..."
1335 print "[Reading] Searching files in offline mode ..."
1345 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1336 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1346 startTime=startTime, endTime=endTime,
1337 startTime=startTime, endTime=endTime,
1347 set=set, expLabel=expLabel, ext=ext,
1338 set=set, expLabel=expLabel, ext=ext,
1348 walk=walk, cursor=cursor,
1339 walk=walk, cursor=cursor,
1349 skip=skip)
1340 skip=skip)
1350
1341
1351 if not(pathList):
1342 if not(pathList):
1352 self.fileIndex = -1
1343 self.fileIndex = -1
1353 self.pathList = []
1344 self.pathList = []
1354 self.filenameList = []
1345 self.filenameList = []
1355 return
1346 return
1356
1347
1357 self.fileIndex = -1
1348 self.fileIndex = -1
1358 self.pathList = pathList
1349 self.pathList = pathList
1359 self.filenameList = filenameList
1350 self.filenameList = filenameList
1360 file_name = os.path.basename(filenameList[-1])
1351 file_name = os.path.basename(filenameList[-1])
1361 basename, ext = os.path.splitext(file_name)
1352 basename, ext = os.path.splitext(file_name)
1362 last_set = int(basename[-3:])
1353 last_set = int(basename[-3:])
1363
1354
1364 self.online = online
1355 self.online = online
1365 self.realtime = realtime
1356 self.realtime = realtime
1366 self.delay = delay
1357 self.delay = delay
1367 ext = ext.lower()
1358 ext = ext.lower()
1368 self.ext = ext
1359 self.ext = ext
1369 self.getByBlock = getblock
1360 self.getByBlock = getblock
1370 self.nTxs = nTxs
1361 self.nTxs = nTxs
1371 self.startTime = startTime
1362 self.startTime = startTime
1372 self.endTime = endTime
1363 self.endTime = endTime
1373 self.endDate = endDate
1364 self.endDate = endDate
1374 self.startDate = startDate
1365 self.startDate = startDate
1375 # Added-----------------
1366 # Added-----------------
1376 self.selBlocksize = blocksize
1367 self.selBlocksize = blocksize
1377 self.selBlocktime = blocktime
1368 self.selBlocktime = blocktime
1378
1369
1379 # Verbose-----------
1370 # Verbose-----------
1380 self.verbose = verbose
1371 self.verbose = verbose
1381 self.warnings = warnings
1372 self.warnings = warnings
1382
1373
1383 if not(self.setNextFile()):
1374 if not(self.setNextFile()):
1384 if (startDate != None) and (endDate != None):
1375 if (startDate != None) and (endDate != None):
1385 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1376 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1386 elif startDate != None:
1377 elif startDate != None:
1387 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1378 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1388 else:
1379 else:
1389 print "[Reading] No files"
1380 print "[Reading] No files"
1390
1381
1391 self.fileIndex = -1
1382 self.fileIndex = -1
1392 self.pathList = []
1383 self.pathList = []
1393 self.filenameList = []
1384 self.filenameList = []
1394 return
1385 return
1395
1386
1396 # self.getBasicHeader()
1387 # self.getBasicHeader()
1397
1388
1398 if last_set != None:
1389 if last_set != None:
1399 self.dataOut.last_block = last_set * \
1390 self.dataOut.last_block = last_set * \
1400 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1391 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1401 return
1392 return
1402
1393
1403 def getBasicHeader(self):
1394 def getBasicHeader(self):
1404
1395
1405 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1396 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1406 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1397 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1407
1398
1408 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1399 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1409
1400
1410 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1401 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1411
1402
1412 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1403 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1413
1404
1414 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1405 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1415
1406
1416 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1407 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1417
1408
1418 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1409 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1419
1410
1420 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1411 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1421
1412
1422 def getFirstHeader(self):
1413 def getFirstHeader(self):
1423
1414
1424 raise NotImplementedError
1415 raise NotImplementedError
1425
1416
1426 def getData(self):
1417 def getData(self):
1427
1418
1428 raise NotImplementedError
1419 raise NotImplementedError
1429
1420
1430 def hasNotDataInBuffer(self):
1421 def hasNotDataInBuffer(self):
1431
1422
1432 raise NotImplementedError
1423 raise NotImplementedError
1433
1424
1434 def readBlock(self):
1425 def readBlock(self):
1435
1426
1436 raise NotImplementedError
1427 raise NotImplementedError
1437
1428
1438 def isEndProcess(self):
1429 def isEndProcess(self):
1439
1430
1440 return self.flagNoMoreFiles
1431 return self.flagNoMoreFiles
1441
1432
1442 def printReadBlocks(self):
1433 def printReadBlocks(self):
1443
1434
1444 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1435 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1445
1436
1446 def printTotalBlocks(self):
1437 def printTotalBlocks(self):
1447
1438
1448 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1439 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1449
1440
1450 def printNumberOfBlock(self):
1441 def printNumberOfBlock(self):
1451 'SPAM!'
1442 'SPAM!'
1452
1443
1453 # if self.flagIsNewBlock:
1444 # if self.flagIsNewBlock:
1454 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1445 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1455 # self.processingHeaderObj.dataBlocksPerFile,
1446 # self.processingHeaderObj.dataBlocksPerFile,
1456 # self.dataOut.datatime.ctime())
1447 # self.dataOut.datatime.ctime())
1457
1448
1458 def printInfo(self):
1449 def printInfo(self):
1459
1450
1460 if self.__printInfo == False:
1451 if self.__printInfo == False:
1461 return
1452 return
1462
1453
1463 self.basicHeaderObj.printInfo()
1454 self.basicHeaderObj.printInfo()
1464 self.systemHeaderObj.printInfo()
1455 self.systemHeaderObj.printInfo()
1465 self.radarControllerHeaderObj.printInfo()
1456 self.radarControllerHeaderObj.printInfo()
1466 self.processingHeaderObj.printInfo()
1457 self.processingHeaderObj.printInfo()
1467
1458
1468 self.__printInfo = False
1459 self.__printInfo = False
1469
1460
1470 def run(self,
1461 def run(self,
1471 path=None,
1462 path=None,
1472 startDate=None,
1463 startDate=None,
1473 endDate=None,
1464 endDate=None,
1474 startTime=datetime.time(0, 0, 0),
1465 startTime=datetime.time(0, 0, 0),
1475 endTime=datetime.time(23, 59, 59),
1466 endTime=datetime.time(23, 59, 59),
1476 set=None,
1467 set=None,
1477 expLabel="",
1468 expLabel="",
1478 ext=None,
1469 ext=None,
1479 online=False,
1470 online=False,
1480 delay=60,
1471 delay=60,
1481 walk=True,
1472 walk=True,
1482 getblock=False,
1473 getblock=False,
1483 nTxs=1,
1474 nTxs=1,
1484 realtime=False,
1475 realtime=False,
1485 blocksize=None,
1476 blocksize=None,
1486 blocktime=None,
1477 blocktime=None,
1487 skip=None,
1478 skip=None,
1488 cursor=None,
1479 cursor=None,
1489 warnings=True,
1480 warnings=True,
1490 server=None,
1481 server=None,
1491 verbose=True,
1482 verbose=True,
1492 format=None,
1483 format=None,
1493 oneDDict=None,
1484 oneDDict=None,
1494 twoDDict=None,
1485 twoDDict=None,
1495 ind2DList=None, **kwargs):
1486 ind2DList=None, **kwargs):
1496
1487
1497 if not(self.isConfig):
1488 if not(self.isConfig):
1498 self.setup(path=path,
1489 self.setup(path=path,
1499 startDate=startDate,
1490 startDate=startDate,
1500 endDate=endDate,
1491 endDate=endDate,
1501 startTime=startTime,
1492 startTime=startTime,
1502 endTime=endTime,
1493 endTime=endTime,
1503 set=set,
1494 set=set,
1504 expLabel=expLabel,
1495 expLabel=expLabel,
1505 ext=ext,
1496 ext=ext,
1506 online=online,
1497 online=online,
1507 delay=delay,
1498 delay=delay,
1508 walk=walk,
1499 walk=walk,
1509 getblock=getblock,
1500 getblock=getblock,
1510 nTxs=nTxs,
1501 nTxs=nTxs,
1511 realtime=realtime,
1502 realtime=realtime,
1512 blocksize=blocksize,
1503 blocksize=blocksize,
1513 blocktime=blocktime,
1504 blocktime=blocktime,
1514 skip=skip,
1505 skip=skip,
1515 cursor=cursor,
1506 cursor=cursor,
1516 warnings=warnings,
1507 warnings=warnings,
1517 server=server,
1508 server=server,
1518 verbose=verbose,
1509 verbose=verbose,
1519 format=format,
1510 format=format,
1520 oneDDict=oneDDict,
1511 oneDDict=oneDDict,
1521 twoDDict=twoDDict,
1512 twoDDict=twoDDict,
1522 ind2DList=ind2DList)
1513 ind2DList=ind2DList)
1523 self.isConfig = True
1514 self.isConfig = True
1524 if server is None:
1515 if server is None:
1525 self.getData()
1516 self.getData()
1526 else:
1517 else:
1527 self.getFromServer()
1518 self.getFromServer()
1528
1519
1529
1520
1530 class JRODataWriter(JRODataIO):
1521 class JRODataWriter(JRODataIO):
1531
1522
1532 """
1523 """
1533 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1524 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1534 de los datos siempre se realiza por bloques.
1525 de los datos siempre se realiza por bloques.
1535 """
1526 """
1536
1527
1537 blockIndex = 0
1528 blockIndex = 0
1538
1529
1539 path = None
1530 path = None
1540
1531
1541 setFile = None
1532 setFile = None
1542
1533
1543 profilesPerBlock = None
1534 profilesPerBlock = None
1544
1535
1545 blocksPerFile = None
1536 blocksPerFile = None
1546
1537
1547 nWriteBlocks = 0
1538 nWriteBlocks = 0
1548
1539
1549 fileDate = None
1540 fileDate = None
1550
1541
1551 def __init__(self, dataOut=None):
1542 def __init__(self, dataOut=None):
1552 raise NotImplementedError
1543 raise NotImplementedError
1553
1544
1554 def hasAllDataInBuffer(self):
1545 def hasAllDataInBuffer(self):
1555 raise NotImplementedError
1546 raise NotImplementedError
1556
1547
1557 def setBlockDimension(self):
1548 def setBlockDimension(self):
1558 raise NotImplementedError
1549 raise NotImplementedError
1559
1550
1560 def writeBlock(self):
1551 def writeBlock(self):
1561 raise NotImplementedError
1552 raise NotImplementedError
1562
1553
1563 def putData(self):
1554 def putData(self):
1564 raise NotImplementedError
1555 raise NotImplementedError
1565
1556
1566 def getProcessFlags(self):
1557 def getProcessFlags(self):
1567
1558
1568 processFlags = 0
1559 processFlags = 0
1569
1560
1570 dtype_index = get_dtype_index(self.dtype)
1561 dtype_index = get_dtype_index(self.dtype)
1571 procflag_dtype = get_procflag_dtype(dtype_index)
1562 procflag_dtype = get_procflag_dtype(dtype_index)
1572
1563
1573 processFlags += procflag_dtype
1564 processFlags += procflag_dtype
1574
1565
1575 if self.dataOut.flagDecodeData:
1566 if self.dataOut.flagDecodeData:
1576 processFlags += PROCFLAG.DECODE_DATA
1567 processFlags += PROCFLAG.DECODE_DATA
1577
1568
1578 if self.dataOut.flagDeflipData:
1569 if self.dataOut.flagDeflipData:
1579 processFlags += PROCFLAG.DEFLIP_DATA
1570 processFlags += PROCFLAG.DEFLIP_DATA
1580
1571
1581 if self.dataOut.code is not None:
1572 if self.dataOut.code is not None:
1582 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1573 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1583
1574
1584 if self.dataOut.nCohInt > 1:
1575 if self.dataOut.nCohInt > 1:
1585 processFlags += PROCFLAG.COHERENT_INTEGRATION
1576 processFlags += PROCFLAG.COHERENT_INTEGRATION
1586
1577
1587 if self.dataOut.type == "Spectra":
1578 if self.dataOut.type == "Spectra":
1588 if self.dataOut.nIncohInt > 1:
1579 if self.dataOut.nIncohInt > 1:
1589 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1580 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1590
1581
1591 if self.dataOut.data_dc is not None:
1582 if self.dataOut.data_dc is not None:
1592 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1583 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1593
1584
1594 if self.dataOut.flagShiftFFT:
1585 if self.dataOut.flagShiftFFT:
1595 processFlags += PROCFLAG.SHIFT_FFT_DATA
1586 processFlags += PROCFLAG.SHIFT_FFT_DATA
1596
1587
1597 return processFlags
1588 return processFlags
1598
1589
1599 def setBasicHeader(self):
1590 def setBasicHeader(self):
1600
1591
1601 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1592 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1602 self.basicHeaderObj.version = self.versionFile
1593 self.basicHeaderObj.version = self.versionFile
1603 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1594 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1604
1595
1605 utc = numpy.floor(self.dataOut.utctime)
1596 utc = numpy.floor(self.dataOut.utctime)
1606 milisecond = (self.dataOut.utctime - utc) * 1000.0
1597 milisecond = (self.dataOut.utctime - utc) * 1000.0
1607
1598
1608 self.basicHeaderObj.utc = utc
1599 self.basicHeaderObj.utc = utc
1609 self.basicHeaderObj.miliSecond = milisecond
1600 self.basicHeaderObj.miliSecond = milisecond
1610 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1601 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1611 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1602 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1612 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1603 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1613
1604
1614 def setFirstHeader(self):
1605 def setFirstHeader(self):
1615 """
1606 """
1616 Obtiene una copia del First Header
1607 Obtiene una copia del First Header
1617
1608
1618 Affected:
1609 Affected:
1619
1610
1620 self.basicHeaderObj
1611 self.basicHeaderObj
1621 self.systemHeaderObj
1612 self.systemHeaderObj
1622 self.radarControllerHeaderObj
1613 self.radarControllerHeaderObj
1623 self.processingHeaderObj self.
1614 self.processingHeaderObj self.
1624
1615
1625 Return:
1616 Return:
1626 None
1617 None
1627 """
1618 """
1628
1619
1629 raise NotImplementedError
1620 raise NotImplementedError
1630
1621
1631 def __writeFirstHeader(self):
1622 def __writeFirstHeader(self):
1632 """
1623 """
1633 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1624 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1634
1625
1635 Affected:
1626 Affected:
1636 __dataType
1627 __dataType
1637
1628
1638 Return:
1629 Return:
1639 None
1630 None
1640 """
1631 """
1641
1632
1642 # CALCULAR PARAMETROS
1633 # CALCULAR PARAMETROS
1643
1634
1644 sizeLongHeader = self.systemHeaderObj.size + \
1635 sizeLongHeader = self.systemHeaderObj.size + \
1645 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1636 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1646 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1637 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1647
1638
1648 self.basicHeaderObj.write(self.fp)
1639 self.basicHeaderObj.write(self.fp)
1649 self.systemHeaderObj.write(self.fp)
1640 self.systemHeaderObj.write(self.fp)
1650 self.radarControllerHeaderObj.write(self.fp)
1641 self.radarControllerHeaderObj.write(self.fp)
1651 self.processingHeaderObj.write(self.fp)
1642 self.processingHeaderObj.write(self.fp)
1652
1643
1653 def __setNewBlock(self):
1644 def __setNewBlock(self):
1654 """
1645 """
1655 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1646 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1656
1647
1657 Return:
1648 Return:
1658 0 : si no pudo escribir nada
1649 0 : si no pudo escribir nada
1659 1 : Si escribio el Basic el First Header
1650 1 : Si escribio el Basic el First Header
1660 """
1651 """
1661 if self.fp == None:
1652 if self.fp == None:
1662 self.setNextFile()
1653 self.setNextFile()
1663
1654
1664 if self.flagIsNewFile:
1655 if self.flagIsNewFile:
1665 return 1
1656 return 1
1666
1657
1667 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1658 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1668 self.basicHeaderObj.write(self.fp)
1659 self.basicHeaderObj.write(self.fp)
1669 return 1
1660 return 1
1670
1661
1671 if not(self.setNextFile()):
1662 if not(self.setNextFile()):
1672 return 0
1663 return 0
1673
1664
1674 return 1
1665 return 1
1675
1666
1676 def writeNextBlock(self):
1667 def writeNextBlock(self):
1677 """
1668 """
1678 Selecciona el bloque siguiente de datos y los escribe en un file
1669 Selecciona el bloque siguiente de datos y los escribe en un file
1679
1670
1680 Return:
1671 Return:
1681 0 : Si no hizo pudo escribir el bloque de datos
1672 0 : Si no hizo pudo escribir el bloque de datos
1682 1 : Si no pudo escribir el bloque de datos
1673 1 : Si no pudo escribir el bloque de datos
1683 """
1674 """
1684 if not(self.__setNewBlock()):
1675 if not(self.__setNewBlock()):
1685 return 0
1676 return 0
1686
1677
1687 self.writeBlock()
1678 self.writeBlock()
1688
1679
1689 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1680 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1690 self.processingHeaderObj.dataBlocksPerFile)
1681 self.processingHeaderObj.dataBlocksPerFile)
1691
1682
1692 return 1
1683 return 1
1693
1684
1694 def setNextFile(self):
1685 def setNextFile(self):
1695 """
1686 """
1696 Determina el siguiente file que sera escrito
1687 Determina el siguiente file que sera escrito
1697
1688
1698 Affected:
1689 Affected:
1699 self.filename
1690 self.filename
1700 self.subfolder
1691 self.subfolder
1701 self.fp
1692 self.fp
1702 self.setFile
1693 self.setFile
1703 self.flagIsNewFile
1694 self.flagIsNewFile
1704
1695
1705 Return:
1696 Return:
1706 0 : Si el archivo no puede ser escrito
1697 0 : Si el archivo no puede ser escrito
1707 1 : Si el archivo esta listo para ser escrito
1698 1 : Si el archivo esta listo para ser escrito
1708 """
1699 """
1709 ext = self.ext
1700 ext = self.ext
1710 path = self.path
1701 path = self.path
1711
1702
1712 if self.fp != None:
1703 if self.fp != None:
1713 self.fp.close()
1704 self.fp.close()
1714
1705
1715 timeTuple = time.localtime(self.dataOut.utctime)
1706 timeTuple = time.localtime(self.dataOut.utctime)
1716 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1707 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1717
1708
1718 fullpath = os.path.join(path, subfolder)
1709 fullpath = os.path.join(path, subfolder)
1719 setFile = self.setFile
1710 setFile = self.setFile
1720
1711
1721 if not(os.path.exists(fullpath)):
1712 if not(os.path.exists(fullpath)):
1722 os.mkdir(fullpath)
1713 os.mkdir(fullpath)
1723 setFile = -1 # inicializo mi contador de seteo
1714 setFile = -1 # inicializo mi contador de seteo
1724 else:
1715 else:
1725 filesList = os.listdir(fullpath)
1716 filesList = os.listdir(fullpath)
1726 if len(filesList) > 0:
1717 if len(filesList) > 0:
1727 filesList = sorted(filesList, key=str.lower)
1718 filesList = sorted(filesList, key=str.lower)
1728 filen = filesList[-1]
1719 filen = filesList[-1]
1729 # el filename debera tener el siguiente formato
1720 # el filename debera tener el siguiente formato
1730 # 0 1234 567 89A BCDE (hex)
1721 # 0 1234 567 89A BCDE (hex)
1731 # x YYYY DDD SSS .ext
1722 # x YYYY DDD SSS .ext
1732 if isNumber(filen[8:11]):
1723 if isNumber(filen[8:11]):
1733 # inicializo mi contador de seteo al seteo del ultimo file
1724 # inicializo mi contador de seteo al seteo del ultimo file
1734 setFile = int(filen[8:11])
1725 setFile = int(filen[8:11])
1735 else:
1726 else:
1736 setFile = -1
1727 setFile = -1
1737 else:
1728 else:
1738 setFile = -1 # inicializo mi contador de seteo
1729 setFile = -1 # inicializo mi contador de seteo
1739
1730
1740 setFile += 1
1731 setFile += 1
1741
1732
1742 # If this is a new day it resets some values
1733 # If this is a new day it resets some values
1743 if self.dataOut.datatime.date() > self.fileDate:
1734 if self.dataOut.datatime.date() > self.fileDate:
1744 setFile = 0
1735 setFile = 0
1745 self.nTotalBlocks = 0
1736 self.nTotalBlocks = 0
1746
1737
1747 filen = '%s%4.4d%3.3d%3.3d%s' % (
1738 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1748 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1739 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1749
1740
1750 filename = os.path.join(path, subfolder, filen)
1741 filename = os.path.join(path, subfolder, filen)
1751
1742
1752 fp = open(filename, 'wb')
1743 fp = open(filename, 'wb')
1753
1744
1754 self.blockIndex = 0
1745 self.blockIndex = 0
1755
1746
1756 # guardando atributos
1747 # guardando atributos
1757 self.filename = filename
1748 self.filename = filename
1758 self.subfolder = subfolder
1749 self.subfolder = subfolder
1759 self.fp = fp
1750 self.fp = fp
1760 self.setFile = setFile
1751 self.setFile = setFile
1761 self.flagIsNewFile = 1
1752 self.flagIsNewFile = 1
1762 self.fileDate = self.dataOut.datatime.date()
1753 self.fileDate = self.dataOut.datatime.date()
1763
1754
1764 self.setFirstHeader()
1755 self.setFirstHeader()
1765
1756
1766 print '[Writing] Opening file: %s' % self.filename
1757 print '[Writing] Opening file: %s' % self.filename
1767
1758
1768 self.__writeFirstHeader()
1759 self.__writeFirstHeader()
1769
1760
1770 return 1
1761 return 1
1771
1762
1772 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1763 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1773 """
1764 """
1774 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1765 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1775
1766
1776 Inputs:
1767 Inputs:
1777 path : directory where data will be saved
1768 path : directory where data will be saved
1778 profilesPerBlock : number of profiles per block
1769 profilesPerBlock : number of profiles per block
1779 set : initial file set
1770 set : initial file set
1780 datatype : An integer number that defines data type:
1771 datatype : An integer number that defines data type:
1781 0 : int8 (1 byte)
1772 0 : int8 (1 byte)
1782 1 : int16 (2 bytes)
1773 1 : int16 (2 bytes)
1783 2 : int32 (4 bytes)
1774 2 : int32 (4 bytes)
1784 3 : int64 (8 bytes)
1775 3 : int64 (8 bytes)
1785 4 : float32 (4 bytes)
1776 4 : float32 (4 bytes)
1786 5 : double64 (8 bytes)
1777 5 : double64 (8 bytes)
1787
1778
1788 Return:
1779 Return:
1789 0 : Si no realizo un buen seteo
1780 0 : Si no realizo un buen seteo
1790 1 : Si realizo un buen seteo
1781 1 : Si realizo un buen seteo
1791 """
1782 """
1792
1783
1793 if ext == None:
1784 if ext == None:
1794 ext = self.ext
1785 ext = self.ext
1795
1786
1796 self.ext = ext.lower()
1787 self.ext = ext.lower()
1797
1788
1798 self.path = path
1789 self.path = path
1799
1790
1800 if set is None:
1791 if set is None:
1801 self.setFile = -1
1792 self.setFile = -1
1802 else:
1793 else:
1803 self.setFile = set - 1
1794 self.setFile = set - 1
1804
1795
1805 self.blocksPerFile = blocksPerFile
1796 self.blocksPerFile = blocksPerFile
1806
1797
1807 self.profilesPerBlock = profilesPerBlock
1798 self.profilesPerBlock = profilesPerBlock
1808
1799
1809 self.dataOut = dataOut
1800 self.dataOut = dataOut
1810 self.fileDate = self.dataOut.datatime.date()
1801 self.fileDate = self.dataOut.datatime.date()
1811 # By default
1802 # By default
1812 self.dtype = self.dataOut.dtype
1803 self.dtype = self.dataOut.dtype
1813
1804
1814 if datatype is not None:
1805 if datatype is not None:
1815 self.dtype = get_numpy_dtype(datatype)
1806 self.dtype = get_numpy_dtype(datatype)
1816
1807
1817 if not(self.setNextFile()):
1808 if not(self.setNextFile()):
1818 print "[Writing] There isn't a next file"
1809 print "[Writing] There isn't a next file"
1819 return 0
1810 return 0
1820
1811
1821 self.setBlockDimension()
1812 self.setBlockDimension()
1822
1813
1823 return 1
1814 return 1
1824
1815
1825 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1816 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1826
1817
1827 if not(self.isConfig):
1818 if not(self.isConfig):
1828
1819
1829 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1820 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1830 set=set, ext=ext, datatype=datatype, **kwargs)
1821 set=set, ext=ext, datatype=datatype, **kwargs)
1831 self.isConfig = True
1822 self.isConfig = True
1832
1823
1833 self.putData()
1824 self.putData()
Show file before | Show file after | Hide comments
@@ -1,69 +1,69
1 '''
1 '''
2 Created on Jul 16, 2014
2 Created on Jul 16, 2014
3
3
4 @author: Miguel Urco
4 @author: Miguel Urco
5 '''
5 '''
6
6
7 import os
7 import os
8 from setuptools import setup, Extension
8 from setuptools import setup, Extension
9 from setuptools.command.build_ext import build_ext as _build_ext
9 from setuptools.command.build_ext import build_ext as _build_ext
10 from schainpy import __version__
10 from schainpy import __version__
11
11
12 class build_ext(_build_ext):
12 class build_ext(_build_ext):
13 def finalize_options(self):
13 def finalize_options(self):
14 _build_ext.finalize_options(self)
14 _build_ext.finalize_options(self)
15 # Prevent numpy from thinking it is still in its setup process:
15 # Prevent numpy from thinking it is still in its setup process:
16 __builtins__.__NUMPY_SETUP__ = False
16 __builtins__.__NUMPY_SETUP__ = False
17 import numpy
17 import numpy
18 self.include_dirs.append(numpy.get_include())
18 self.include_dirs.append(numpy.get_include())
19
19
20 setup(name="schainpy",
20 setup(name="schainpy",
21 version=__version__,
21 version=__version__,
22 description="Python tools to read, write and process Jicamarca data",
22 description="Python tools to read, write and process Jicamarca data",
23 author="Miguel Urco",
23 author="Miguel Urco",
24 author_email="miguel.urco@jro.igp.gob.pe",
24 author_email="miguel.urco@jro.igp.gob.pe",
25 url="http://jro.igp.gob.pe",
25 url="http://jro.igp.gob.pe",
26 packages={'schainpy',
26 packages={'schainpy',
27 'schainpy.model',
27 'schainpy.model',
28 'schainpy.model.data',
28 'schainpy.model.data',
29 'schainpy.model.graphics',
29 'schainpy.model.graphics',
30 'schainpy.model.io',
30 'schainpy.model.io',
31 'schainpy.model.proc',
31 'schainpy.model.proc',
32 'schainpy.model.serializer',
32 'schainpy.model.serializer',
33 'schainpy.model.utils',
33 'schainpy.model.utils',
34 'schainpy.utils',
34 'schainpy.utils',
35 'schainpy.gui',
35 'schainpy.gui',
36 'schainpy.gui.figures',
36 'schainpy.gui.figures',
37 'schainpy.gui.viewcontroller',
37 'schainpy.gui.viewcontroller',
38 'schainpy.gui.viewer',
38 'schainpy.gui.viewer',
39 'schainpy.gui.viewer.windows'},
39 'schainpy.gui.viewer.windows'},
40 ext_package='schainpy',
40 ext_package='schainpy',
41 py_modules=[''],
41 py_modules=[''],
42 package_data={'': ['schain.conf.template'],
42 package_data={'': ['schain.conf.template'],
43 'schainpy.gui.figures': ['*.png', '*.jpg'],
43 'schainpy.gui.figures': ['*.png', '*.jpg'],
44 },
44 },
45 include_package_data=False,
45 include_package_data=False,
46 scripts=['schainpy/gui/schainGUI'],
46 scripts=['schainpy/gui/schainGUI'],
47 ext_modules=[
47 ext_modules=[
48 Extension("cSchain", ["schainpy/model/proc/extensions.c"]
48 Extension("cSchain", ["schainpy/model/proc/extensions.c"]
49 )],
49 )],
50 entry_points={
50 entry_points={
51 'console_scripts': [
51 'console_scripts': [
52 'schain = schaincli.cli:main',
52 'schain = schaincli.cli:main',
53 ],
53 ],
54 },
54 },
55 cmdclass={'build_ext': build_ext},
55 cmdclass={'build_ext': build_ext},
56 setup_requires=["numpy >= 1.11.2"],
56 setup_requires=["numpy >= 1.11.2"],
57 install_requires=[
57 install_requires=[
58 "scipy >= 0.14.0",
58 "scipy >= 0.14.0",
59 "h5py >= 2.2.1",
59 "h5py >= 2.2.1",
60 "matplotlib >= 1.4.2",
60 "matplotlib >= 2.0.0",
61 "pyfits >= 3.4",
61 "pyfits >= 3.4",
62 "paramiko >= 2.1.2",
62 "paramiko >= 2.1.2",
63 "paho-mqtt >= 1.2",
63 "paho-mqtt >= 1.2",
64 "zmq",
64 "zmq",
65 "fuzzywuzzy",
65 "fuzzywuzzy",
66 "click",
66 "click",
67 "python-Levenshtein"
67 "python-Levenshtein"
68 ],
68 ],
69 )
69 )
General Comments 0
You need to be logged in to leave comments. Login now