##// END OF EJS Templates
schain
RSS
-USRP Reader was added to Signal Chain GUI...
Miguel Valdez -
r589:341989823444
parent child
Show More

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -0,0 +1,87
1 """
2 Classes to save parameters from Windows.
3
4 -Project window
5 -Voltage window
6 -Spectra window
7 -SpectraHeis window
8 -Correlation window
9
10 """
11
12 class ProjectParms():
13
14 parmsOk = False
15 project_name = None
16 datatype = None
17 ext = None
18 dpath = None
19 startDate = None
20 endDate = None
21 startTime = None
22 endTime = None
23 online = None
24 delay = None
25 walk = None
26 expLabel = None
27 set = None
28 ippKm = None
29
30 def __init__(self):
31
32 self.parmsOk = True
33 self.expLabel = ''
34 self.set = None
35 self.ippKm = None
36 self.walk = None
37 self.delay = None
38
39 def getDatatypeIndex(self):
40
41 indexDatatype = None
42
43 if self.datatype.lower() == 'voltage':
44 indexDatatype = 0
45 if self.datatype.lower() == 'spectra':
46 indexDatatype = 1
47 if self.datatype.lower() == 'fits':
48 indexDatatype = 2
49 if self.datatype.lower() == 'usrp':
50 indexDatatype = 3
51
52 return indexDatatype
53
54 def getExt(self):
55
56 ext = None
57
58 if self.datatype.lower() == 'voltage':
59 ext = '.r'
60 if self.datatype.lower() == 'spectra':
61 ext = '.pdata'
62 if self.datatype.lower() == 'fits':
63 ext = '.fits'
64 if self.datatype.lower() == 'usrp':
65 ext = '.hdf5'
66
67 return ext
68
69 def set(self, project_name, datatype, ext, dpath, online,
70 startDate=None, endDate=None, startTime=None, endTime=None,
71 delay=None, walk=None, set=None, ippKm=None, parmsOk=True):
72
73 project_name = project_name
74 datatype = datatype
75 ext = ext
76 dpath = dpath
77 startDate = startDate
78 endDate = endDate
79 startTime = startTime
80 endTime = endTime
81 online = online
82 delay = delay
83 walk = walk
84 set = set
85 ippKm = ippKm
86
87 self.parmsOk = parmsOk No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,211
1 # -*- coding: utf-8 -*-
2 """
3 This module contains every model class to create, modify and show a property tree on a GUI.
4 """
5
6 from PyQt4 import QtCore
7 import itertools
8
9 HORIZONTAL_HEADERS = ("Property","Value " )
10
11 HORIZONTAL = ("RAMA :",)
12
13 class PropertyBuffer():
14
15 def __init__(self):
16
17 self.clear()
18
19 def clear(self):
20
21 self.headerList = []
22 self.parmList = []
23 self.valueList = []
24
25 def append(self, header, parm, value):
26
27 self.headerList.append(header)
28 self.parmList.append(parm)
29 self.valueList.append(value)
30
31 return
32
33 def get(self):
34
35 return self.headerList, self.parmList, self.valueList
36
37 def getPropertyModel(self):
38
39 propertiesModel = TreeModel()
40 propertiesModel.showProperties(self.headerList, self.parmList, self.valueList)
41
42 return propertiesModel
43
44
45 class TreeModel(QtCore.QAbstractItemModel):
46 '''
47 a model to display a few names, ordered by encabezado
48
49 '''
50 def __init__(self ,parent=None):
51 super(TreeModel, self).__init__(parent)
52 self.people = []
53
54 def initProjectView(self):
55 """
56 Reemplazo del mΓ©todo showtree
57 """
58 HORIZONTAL_HEADERS = ("Property","Value " )
59 HORIZONTAL = ("RAMA :",)
60 self.rootItem = TreeItem(None, "ALL", None)
61 self.parents = {0 : self.rootItem}
62 self.__setupModelData()
63
64 def initPUVoltageView(self):
65 HORIZONTAL_HEADERS = ("Operation"," Parameter Value " )
66 HORIZONTAL = ("RAMA :",)
67 self.rootItem = TreeItem(None, "ALL", None)
68 self.parents = {0 : self.rootItem}
69 self.__setupModelData()
70
71 def showProperties(self,headerList, parmList, valueList):
72 """
73 set2Obje
74 """
75 for header, parameter, value in itertools.izip(headerList, parmList, valueList):
76 person = person_class(header, parameter, value)
77 self.people.append(person)
78
79 self.rootItem = TreeItem(None, "ALL", None)
80 self.parents = {0 : self.rootItem}
81 self.__setupModelData()
82
83 def columnCount(self, parent=None):
84 if parent and parent.isValid():
85 return parent.internalPointer().columnCount()
86 else:
87 return len(HORIZONTAL_HEADERS)
88
89 def data(self, index, role):
90 if not index.isValid():
91 return QtCore.QVariant()
92
93 item = index.internalPointer()
94 if role == QtCore.Qt.DisplayRole:
95 return item.data(index.column())
96 if role == QtCore.Qt.UserRole:
97 if item:
98 return item.person
99
100 return QtCore.QVariant()
101
102 def index(self, row, column, parent):
103 if not self.hasIndex(row, column, parent):
104 return QtCore.QModelIndex()
105
106 if not parent.isValid():
107 parentItem = self.rootItem
108 else:
109 parentItem = parent.internalPointer()
110
111 childItem = parentItem.child(row)
112 if childItem:
113 return self.createIndex(row, column, childItem)
114 else:
115 return QtCore.QModelIndex()
116
117 def parent(self, index):
118 if not index.isValid():
119 return QtCore.QModelIndex()
120
121 childItem = index.internalPointer()
122 if not childItem:
123 return QtCore.QModelIndex()
124
125 parentItem = childItem.parent()
126
127 if parentItem == self.rootItem:
128 return QtCore.QModelIndex()
129
130 return self.createIndex(parentItem.row(), 0, parentItem)
131
132 def rowCount(self, parent=QtCore.QModelIndex()):
133 if parent.column() > 0:
134 return 0
135 if not parent.isValid():
136 p_Item = self.rootItem
137 else:
138 p_Item = parent.internalPointer()
139 return p_Item.childCount()
140
141 def __setupModelData(self):
142 for person in self.people:
143 if person.value:
144 encabezado = person.header
145
146 if not self.parents.has_key(encabezado):
147 newparent = TreeItem(None, encabezado, self.rootItem)
148 self.rootItem.appendChild(newparent)
149
150 self.parents[encabezado] = newparent
151
152 parentItem = self.parents[encabezado]
153 newItem = TreeItem(person, "", parentItem)
154 parentItem.appendChild(newItem)
155
156 class person_class(object):
157 '''
158 a trivial custom data object
159 '''
160 def __init__(self, header, parameter, value):
161 self.header = header
162 self.parameter = parameter
163 self.value = value
164
165 def __repr__(self):
166 return "PERSON - %s %s"% (self.parameter, self.header)
167
168 class TreeItem(object):
169 '''
170 a python object used to return row/column data, and keep note of
171 it's parents and/or children
172 '''
173 def __init__(self, person, header, parentItem):
174 self.person = person
175 self.parentItem = parentItem
176 self.header = header
177 self.childItems = []
178
179 def appendChild(self, item):
180 self.childItems.append(item)
181
182 def child(self, row):
183 return self.childItems[row]
184
185 def childCount(self):
186 return len(self.childItems)
187
188 def columnCount(self):
189 return 2
190
191 def data(self, column):
192 if self.person == None:
193 if column == 0:
194 return QtCore.QVariant(self.header)
195 if column == 1:
196 return QtCore.QVariant("")
197 else:
198 if column == 0:
199 return QtCore.QVariant(self.person.parameter)
200 if column == 1:
201 return QtCore.QVariant(self.person.value)
202 return QtCore.QVariant()
203
204 def parent(self):
205 return self.parentItem
206
207 def row(self):
208 if self.parentItem:
209 return self.parentItem.childItems.index(self)
210 return 0
211 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1061 +1,1088
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5 from xml.etree.ElementTree import Element, SubElement
6 6 from xml.etree import ElementTree as ET
7 7 from xml.dom import minidom
8 8
9 9 #import datetime
10 10 from model import *
11 11
12 12 try:
13 13 from gevent import sleep
14 14 except:
15 15 from time import sleep
16 16
17 17 import ast
18 18
19 19 def prettify(elem):
20 20 """Return a pretty-printed XML string for the Element.
21 21 """
22 22 rough_string = ET.tostring(elem, 'utf-8')
23 23 reparsed = minidom.parseString(rough_string)
24 24 return reparsed.toprettyxml(indent=" ")
25 25
26 26 class ParameterConf():
27 27
28 28 id = None
29 29 name = None
30 30 value = None
31 31 format = None
32 32
33 33 __formated_value = None
34 34
35 35 ELEMENTNAME = 'Parameter'
36 36
37 37 def __init__(self):
38 38
39 39 self.format = 'str'
40 40
41 41 def getElementName(self):
42 42
43 43 return self.ELEMENTNAME
44 44
45 45 def getValue(self):
46 46
47 47 if self.__formated_value != None:
48 48
49 49 return self.__formated_value
50 50
51 51 value = self.value
52 52
53 53 if self.format == 'bool':
54 54 value = int(value)
55 55
56 56 if self.format == 'list':
57 57 strList = value.split(',')
58 58
59 59 self.__formated_value = strList
60 60
61 61 return self.__formated_value
62 62
63 63 if self.format == 'intlist':
64 64 """
65 65 Example:
66 66 value = (0,1,2)
67 67 """
68 68 value = value.replace('(', '')
69 69 value = value.replace(')', '')
70 70
71 71 value = value.replace('[', '')
72 72 value = value.replace(']', '')
73 73
74 74 strList = value.split(',')
75 75 intList = [int(x) for x in strList]
76 76
77 77 self.__formated_value = intList
78 78
79 79 return self.__formated_value
80 80
81 81 if self.format == 'floatlist':
82 82 """
83 83 Example:
84 84 value = (0.5, 1.4, 2.7)
85 85 """
86 86
87 87 value = value.replace('(', '')
88 88 value = value.replace(')', '')
89 89
90 90 value = value.replace('[', '')
91 91 value = value.replace(']', '')
92 92
93 93 strList = value.split(',')
94 94 floatList = [float(x) for x in strList]
95 95
96 96 self.__formated_value = floatList
97 97
98 98 return self.__formated_value
99 99
100 100 if self.format == 'date':
101 101 strList = value.split('/')
102 102 intList = [int(x) for x in strList]
103 103 date = datetime.date(intList[0], intList[1], intList[2])
104 104
105 105 self.__formated_value = date
106 106
107 107 return self.__formated_value
108 108
109 109 if self.format == 'time':
110 110 strList = value.split(':')
111 111 intList = [int(x) for x in strList]
112 112 time = datetime.time(intList[0], intList[1], intList[2])
113 113
114 114 self.__formated_value = time
115 115
116 116 return self.__formated_value
117 117
118 118 if self.format == 'pairslist':
119 119 """
120 120 Example:
121 121 value = (0,1),(1,2)
122 122 """
123 123
124 124 value = value.replace('(', '')
125 125 value = value.replace(')', '')
126 126
127 127 value = value.replace('[', '')
128 128 value = value.replace(']', '')
129 129
130 130 strList = value.split(',')
131 131 intList = [int(item) for item in strList]
132 132 pairList = []
133 133 for i in range(len(intList)/2):
134 134 pairList.append((intList[i*2], intList[i*2 + 1]))
135 135
136 136 self.__formated_value = pairList
137 137
138 138 return self.__formated_value
139 139
140 140 if self.format == 'multilist':
141 141 """
142 142 Example:
143 143 value = (0,1,2),(3,4,5)
144 144 """
145 145 multiList = ast.literal_eval(value)
146 146
147 147 self.__formated_value = multiList
148 148
149 149 return self.__formated_value
150 150
151 151 format_func = eval(self.format)
152 152
153 153 self.__formated_value = format_func(value)
154 154
155 155 return self.__formated_value
156 156
157 157 def setup(self, id, name, value, format='str'):
158 158
159 159 self.id = id
160 160 self.name = name
161 161 self.value = str(value)
162 162 self.format = str.lower(format)
163 163
164 164 def update(self, name, value, format='str'):
165 165
166 166 self.name = name
167 167 self.value = str(value)
168 168 self.format = format
169 169
170 170 def makeXml(self, opElement):
171 171
172 172 parmElement = SubElement(opElement, self.ELEMENTNAME)
173 173 parmElement.set('id', str(self.id))
174 174 parmElement.set('name', self.name)
175 175 parmElement.set('value', self.value)
176 176 parmElement.set('format', self.format)
177 177
178 178 def readXml(self, parmElement):
179 179
180 180 self.id = parmElement.get('id')
181 181 self.name = parmElement.get('name')
182 182 self.value = parmElement.get('value')
183 183 self.format = str.lower(parmElement.get('format'))
184 184
185 185 #Compatible with old signal chain version
186 186 if self.format == 'int' and self.name == 'idfigure':
187 187 self.name = 'id'
188 188
189 189 def printattr(self):
190 190
191 191 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
192 192
193 193 class OperationConf():
194 194
195 195 id = None
196 196 name = None
197 197 priority = None
198 198 type = None
199 199
200 200 parmConfObjList = []
201 201
202 202 ELEMENTNAME = 'Operation'
203 203
204 204 def __init__(self):
205 205
206 self.id = 0
206 self.id = '0'
207 207 self.name = None
208 208 self.priority = None
209 209 self.type = 'self'
210 210
211 211
212 212 def __getNewId(self):
213 213
214 214 return int(self.id)*10 + len(self.parmConfObjList) + 1
215 215
216 216 def getElementName(self):
217 217
218 218 return self.ELEMENTNAME
219 219
220 220 def getParameterObjList(self):
221 221
222 222 return self.parmConfObjList
223 223
224 224 def getParameterObj(self, parameterName):
225 225
226 226 for parmConfObj in self.parmConfObjList:
227 227
228 228 if parmConfObj.name != parameterName:
229 229 continue
230 230
231 231 return parmConfObj
232 232
233 233 return None
234 234
235 235 def getParameterObjfromValue(self,parameterValue):
236 236 for parmConfObj in self.parmConfObjList:
237 237
238 238 if parmConfObj.getValue() != parameterValue:
239 239 continue
240 240
241 241 return parmConfObj.getValue()
242 242
243 243 return None
244 244
245 245 def getParameterValue(self, parameterName):
246 246
247 247 parameterObj = self.getParameterObj(parameterName)
248 248 value = parameterObj.getValue()
249 249
250 250 return value
251 251
252 252 def setup(self, id, name, priority, type):
253 253
254 254 self.id = id
255 255 self.name = name
256 256 self.type = type
257 257 self.priority = priority
258 258
259 259 self.parmConfObjList = []
260 260
261 261 def removeParameters(self):
262 262
263 263 for obj in self.parmConfObjList:
264 264 del obj
265 265
266 266 self.parmConfObjList = []
267 267
268 268 def addParameter(self, name, value, format='str'):
269 269
270 270 id = self.__getNewId()
271 271
272 272 parmConfObj = ParameterConf()
273 273 parmConfObj.setup(id, name, value, format)
274 274
275 275 self.parmConfObjList.append(parmConfObj)
276 276
277 277 return parmConfObj
278 278
279 279 def changeParameter(self, name, value, format='str'):
280 280
281 281 parmConfObj = self.getParameterObj(name)
282 282 parmConfObj.update(name, value, format)
283 283
284 284 return parmConfObj
285 285
286 286 def makeXml(self, upElement):
287 287
288 288 opElement = SubElement(upElement, self.ELEMENTNAME)
289 289 opElement.set('id', str(self.id))
290 290 opElement.set('name', self.name)
291 291 opElement.set('type', self.type)
292 292 opElement.set('priority', str(self.priority))
293 293
294 294 for parmConfObj in self.parmConfObjList:
295 295 parmConfObj.makeXml(opElement)
296 296
297 297 def readXml(self, opElement):
298 298
299 299 self.id = opElement.get('id')
300 300 self.name = opElement.get('name')
301 301 self.type = opElement.get('type')
302 302 self.priority = opElement.get('priority')
303 303
304 304 #Compatible with old signal chain version
305 305 #Use of 'run' method instead 'init'
306 306 if self.type == 'self' and self.name == 'init':
307 307 self.name = 'run'
308 308
309 309 self.parmConfObjList = []
310 310
311 311 parmElementList = opElement.getiterator(ParameterConf().getElementName())
312 312
313 313 for parmElement in parmElementList:
314 314 parmConfObj = ParameterConf()
315 315 parmConfObj.readXml(parmElement)
316 316
317 317 #Compatible with old signal chain version
318 318 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
319 319 if self.type != 'self' and self.name == 'Plot':
320 320 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
321 321 self.name = parmConfObj.value
322 322 continue
323 323
324 324 self.parmConfObjList.append(parmConfObj)
325 325
326 326 def printattr(self):
327 327
328 328 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
329 329 self.id,
330 330 self.name,
331 331 self.type,
332 332 self.priority)
333 333
334 334 for parmConfObj in self.parmConfObjList:
335 335 parmConfObj.printattr()
336 336
337 337 def createObject(self):
338 338
339 339 if self.type == 'self':
340 340 raise ValueError, "This operation type cannot be created"
341 341
342 342 if self.type == 'external' or self.type == 'other':
343 343 className = eval(self.name)
344 344 opObj = className()
345 345
346 346 return opObj
347 347
348 348 class ProcUnitConf():
349 349
350 350 id = None
351 351 name = None
352 352 datatype = None
353 353 inputId = None
354 354 parentId = None
355 355
356 356 opConfObjList = []
357 357
358 358 procUnitObj = None
359 359 opObjList = []
360 360
361 361 ELEMENTNAME = 'ProcUnit'
362 362
363 363 def __init__(self):
364 364
365 365 self.id = None
366 366 self.datatype = None
367 367 self.name = None
368 368 self.inputId = None
369 369
370 370 self.opConfObjList = []
371 371
372 372 self.procUnitObj = None
373 373 self.opObjDict = {}
374 374
375 375 def __getPriority(self):
376 376
377 377 return len(self.opConfObjList)+1
378 378
379 379 def __getNewId(self):
380 380
381 381 return int(self.id)*10 + len(self.opConfObjList) + 1
382 382
383 383 def getElementName(self):
384 384
385 385 return self.ELEMENTNAME
386 386
387 387 def getId(self):
388 388
389 return str(self.id)
389 return self.id
390 390
391 391 def getInputId(self):
392 392
393 return str(self.inputId)
393 return self.inputId
394 394
395 395 def getOperationObjList(self):
396 396
397 397 return self.opConfObjList
398 398
399 399 def getOperationObj(self, name=None):
400 400
401 401 for opConfObj in self.opConfObjList:
402 402
403 403 if opConfObj.name != name:
404 404 continue
405 405
406 406 return opConfObj
407 407
408 408 return None
409 409
410 410 def getOpObjfromParamValue(self,value=None):
411 411
412 412 for opConfObj in self.opConfObjList:
413 413 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
414 414 continue
415 415 return opConfObj
416 416 return None
417 417
418 418 def getProcUnitObj(self):
419 419
420 420 return self.procUnitObj
421 421
422 422 def setup(self, id, name, datatype, inputId, parentId=None):
423 423
424 424 self.id = id
425 425 self.name = name
426 426 self.datatype = datatype
427 427 self.inputId = inputId
428 428 self.parentId = parentId
429 429
430 430 self.opConfObjList = []
431 431
432 432 self.addOperation(name='run', optype='self')
433 433
434 434 def removeOperations(self):
435 435
436 436 for obj in self.opConfObjList:
437 437 del obj
438 438
439 439 self.opConfObjList = []
440 440 self.addOperation(name='run')
441 441
442 442 def addParameter(self, **kwargs):
443 443 '''
444 444 Add parameters to "run" operation
445 445 '''
446 446 opObj = self.opConfObjList[0]
447 447
448 448 opObj.addParameter(**kwargs)
449 449
450 450 return opObj
451 451
452 452 def addOperation(self, name, optype='self'):
453 453
454 454 id = self.__getNewId()
455 455 priority = self.__getPriority()
456 456
457 457 opConfObj = OperationConf()
458 458 opConfObj.setup(id, name=name, priority=priority, type=optype)
459 459
460 460 self.opConfObjList.append(opConfObj)
461 461
462 462 return opConfObj
463 463
464 464 def makeXml(self, procUnitElement):
465 465
466 466 upElement = SubElement(procUnitElement, self.ELEMENTNAME)
467 467 upElement.set('id', str(self.id))
468 468 upElement.set('name', self.name)
469 469 upElement.set('datatype', self.datatype)
470 470 upElement.set('inputId', str(self.inputId))
471 471
472 472 for opConfObj in self.opConfObjList:
473 473 opConfObj.makeXml(upElement)
474 474
475 475 def readXml(self, upElement):
476 476
477 477 self.id = upElement.get('id')
478 478 self.name = upElement.get('name')
479 479 self.datatype = upElement.get('datatype')
480 480 self.inputId = upElement.get('inputId')
481 481
482 if self.inputId == 'None':
483 self.inputId = '0'
484
482 485 self.opConfObjList = []
483 486
484 487 opElementList = upElement.getiterator(OperationConf().getElementName())
485 488
486 489 for opElement in opElementList:
487 490 opConfObj = OperationConf()
488 491 opConfObj.readXml(opElement)
489 492 self.opConfObjList.append(opConfObj)
490 493
491 494 def printattr(self):
492 495
493 496 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
494 497 self.id,
495 498 self.name,
496 499 self.datatype,
497 500 self.inputId)
498 501
499 502 for opConfObj in self.opConfObjList:
500 503 opConfObj.printattr()
501 504
502 505 def createObjects(self):
503 506
504 507 className = eval(self.name)
505 508 procUnitObj = className()
506 509
507 510 for opConfObj in self.opConfObjList:
508 511
509 512 if opConfObj.type == 'self':
510 513 continue
511 514
512 515 opObj = opConfObj.createObject()
513 516
514 517 self.opObjDict[opConfObj.id] = opObj
515 518 procUnitObj.addOperation(opObj, opConfObj.id)
516 519
517 520 self.procUnitObj = procUnitObj
518 521
519 522 return procUnitObj
520 523
521 524 def run(self):
522 525
523 526 finalSts = False
524 527
525 528 for opConfObj in self.opConfObjList:
526 529
527 530 kwargs = {}
528 531 for parmConfObj in opConfObj.getParameterObjList():
529 532 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
530 533 continue
531 534
532 535 kwargs[parmConfObj.name] = parmConfObj.getValue()
533 536
534 537 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
535 538 sts = self.procUnitObj.call(opType = opConfObj.type,
536 539 opName = opConfObj.name,
537 540 opId = opConfObj.id,
538 541 **kwargs)
539 542 finalSts = finalSts or sts
540 543
541 544 return finalSts
542 545
543 546 def close(self):
544 547
545 548 for opConfObj in self.opConfObjList:
546 549 if opConfObj.type == 'self':
547 550 continue
548 551
549 552 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
550 553 opObj.close()
551 554
552 555 self.procUnitObj.close()
553 556
554 557 return
555 558
556 559 class ReadUnitConf(ProcUnitConf):
557 560
558 561 path = None
559 562 startDate = None
560 563 endDate = None
561 564 startTime = None
562 565 endTime = None
563 566
564 567 ELEMENTNAME = 'ReadUnit'
565 568
566 569 def __init__(self):
567 570
568 571 self.id = None
569 572 self.datatype = None
570 573 self.name = None
571 self.inputId = 0
574 self.inputId = None
575
576 self.parentId = None
572 577
573 578 self.opConfObjList = []
574 579 self.opObjList = []
575 580
576 581 def getElementName(self):
577 582
578 583 return self.ELEMENTNAME
579 584
580 585 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
581 586
582 587 self.id = id
583 588 self.name = name
584 589 self.datatype = datatype
585 590
586 591 self.path = path
587 592 self.startDate = startDate
588 593 self.endDate = endDate
589 594 self.startTime = startTime
590 595 self.endTime = endTime
591 596
597 self.inputId = '0'
598 self.parentId = parentId
599
592 600 self.addRunOperation(**kwargs)
593 601
594 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, **kwargs):
602 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
603
604 if name==None:
605 if 'Reader' in datatype:
606 name = datatype
607 else:
608 name = '%sReader' %(datatype)
609
610 if datatype==None:
611 datatype = name.replace('Reader','')
595 612
596 613 self.datatype = datatype
614 self.name = name
597 615 self.path = path
598 616 self.startDate = startDate
599 617 self.endDate = endDate
600 618 self.startTime = startTime
601 619 self.endTime = endTime
602 620
621 self.inputId = None
622 self.parentId = parentId
623
603 624 self.updateRunOperation(**kwargs)
604 625
605 626 def addRunOperation(self, **kwargs):
606 627
607 628 opObj = self.addOperation(name = 'run', optype = 'self')
608 629
609 630 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
610 631 opObj.addParameter(name='path' , value=self.path, format='str')
611 632 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
612 633 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
613 634 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
614 635 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
615 636
616 637 for key, value in kwargs.items():
617 638 opObj.addParameter(name=key, value=value, format=type(value).__name__)
618 639
619 640 return opObj
620 641
621 642 def updateRunOperation(self, **kwargs):
622 643
623 644 opObj = self.getOperationObj(name = 'run')
624 645 opObj.removeParameters()
625 646
626 647 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
627 648 opObj.addParameter(name='path' , value=self.path, format='str')
628 649 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
629 650 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
630 651 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
631 652 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
632 653
633 654 for key, value in kwargs.items():
634 655 opObj.addParameter(name=key, value=value, format=type(value).__name__)
635 656
636 657 return opObj
637 658
638 659 class Project():
639 660
640 661 id = None
641 662 name = None
642 663 description = None
643 664 # readUnitConfObjList = None
644 665 procUnitConfObjDict = None
645 666
646 667 ELEMENTNAME = 'Project'
647 668
648 669 def __init__(self, control=None, dataq=None):
649 670
650 671 self.id = None
651 672 self.name = None
652 673 self.description = None
653 674
654 675 self.procUnitConfObjDict = {}
655 676
656 677 #global data_q
657 678 #data_q = dataq
658 679
659 680 if control==None:
660 681 control = {}
661 682 control['stop'] = False
662 683 control['pause'] = False
663 684
664 685 self.control = control
665 686
666 687 def __getNewId(self):
667 688
668 689 id = int(self.id)*10 + len(self.procUnitConfObjDict) + 1
669 690
670 691 return str(id)
671 692
672 693 def getElementName(self):
673 694
674 695 return self.ELEMENTNAME
675 696
676 697 def getId(self):
677 698
678 699 return self.id
679 700
680 701 def setup(self, id, name, description):
681 702
682 703 self.id = id
683 704 self.name = name
684 705 self.description = description
685 706
686 707 def update(self, name, description):
687 708
688 709 self.name = name
689 710 self.description = description
690 711
691 712 def addReadUnit(self, datatype=None, name=None, **kwargs):
692 713
693 714 #Compatible with old signal chain version
694 715 if datatype==None and name==None:
695 716 raise ValueError, "datatype or name should be defined"
696 717
697 718 if name==None:
698 719 if 'Reader' in datatype:
699 720 name = datatype
700 721 else:
701 722 name = '%sReader' %(datatype)
702 723
703 724 if datatype==None:
704 725 datatype = name.replace('Reader','')
705 726
706 id = self.__getNewId()
727 idReadUnit = self.__getNewId()
707 728
708 729 readUnitConfObj = ReadUnitConf()
709 readUnitConfObj.setup(id, name, datatype, parentId=self.id, **kwargs)
730 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
710 731
711 732 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
712 733
713 734 return readUnitConfObj
714 735
715 def addProcUnit(self, inputId=0, datatype=None, name=None):
736 def addProcUnit(self, inputId='0', datatype=None, name=None):
716 737
717 738 #Compatible with old signal chain version
718 739 if datatype==None and name==None:
719 740 raise ValueError, "datatype or name should be defined"
720 741
721 742 if name==None:
722 743 if 'Proc' in datatype:
723 744 name = datatype
724 745 else:
725 746 name = '%sProc' %(datatype)
726 747
727 748 if datatype==None:
728 749 datatype = name.replace('Proc','')
729 750
730 id = self.__getNewId()
751 idProcUnit = self.__getNewId()
731 752
732 753 procUnitConfObj = ProcUnitConf()
733 procUnitConfObj.setup(id, name, datatype, inputId, parentId=self.id)
754 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
734 755
735 756 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
736 757
737 758 return procUnitConfObj
738 759
739 760 def removeProcUnit(self, id):
740 761
741 762 if id in self.procUnitConfObjDict.keys():
742 763 self.procUnitConfObjDict.pop(id)
743 764
744 765 def getReadUnitId(self):
745 766
746 767 readUnitConfObj = self.getReadUnitObj()
747 768
748 769 return readUnitConfObj.id
749 770
750 771 def getReadUnitObj(self):
751 772
752 773 for obj in self.procUnitConfObjDict.values():
753 774 if obj.getElementName() == "ReadUnit":
754 775 return obj
755 776
756 777 return None
757 778
758 779 def getProcUnitObj(self, id):
759 780
760 781 return self.procUnitConfObjDict[id]
761 782
762 783 def getProcUnitObjByName(self, name):
763 784
764 785 for obj in self.procUnitConfObjDict.values():
765 786 if obj.name == name:
766 787 return obj
767 788
768 789 return None
769 790
770 791 def makeXml(self):
771 792
772 793 projectElement = Element('Project')
773 794 projectElement.set('id', str(self.id))
774 795 projectElement.set('name', self.name)
775 796 projectElement.set('description', self.description)
776 797
777 798 # for readUnitConfObj in self.readUnitConfObjList:
778 799 # readUnitConfObj.makeXml(projectElement)
779 800
780 801 for procUnitConfObj in self.procUnitConfObjDict.values():
781 802 procUnitConfObj.makeXml(projectElement)
782 803
783 804 self.projectElement = projectElement
784 805
785 806 def writeXml(self, filename):
786 807
787 808 self.makeXml()
788 809
789 810 #print prettify(self.projectElement)
790 811
791 812 ElementTree(self.projectElement).write(filename, method='xml')
792 813
793 814 def readXml(self, filename):
794 815
795 816 #tree = ET.parse(filename)
796 817 self.projectElement = None
797 818 # self.readUnitConfObjList = []
798 819 self.procUnitConfObjDict = {}
799 820
800 821 self.projectElement = ElementTree().parse(filename)
801 822
802 823 self.project = self.projectElement.tag
803 824
804 825 self.id = self.projectElement.get('id')
805 826 self.name = self.projectElement.get('name')
806 827 self.description = self.projectElement.get('description')
807 828
808 829 readUnitElementList = self.projectElement.getiterator(ReadUnitConf().getElementName())
809 830
810 831 for readUnitElement in readUnitElementList:
811 832 readUnitConfObj = ReadUnitConf()
812 833 readUnitConfObj.readXml(readUnitElement)
813 834
835 if readUnitConfObj.parentId == None:
836 readUnitConfObj.parentId = self.id
837
814 838 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
815 839
816 840 procUnitElementList = self.projectElement.getiterator(ProcUnitConf().getElementName())
817 841
818 842 for procUnitElement in procUnitElementList:
819 843 procUnitConfObj = ProcUnitConf()
820 844 procUnitConfObj.readXml(procUnitElement)
821 845
846 if procUnitConfObj.parentId == None:
847 procUnitConfObj.parentId = self.id
848
822 849 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
823 850
824 851 def printattr(self):
825 852
826 853 print "Project[%s]: name = %s, description = %s" %(self.id,
827 854 self.name,
828 855 self.description)
829 856
830 857 # for readUnitConfObj in self.readUnitConfObjList:
831 858 # readUnitConfObj.printattr()
832 859
833 860 for procUnitConfObj in self.procUnitConfObjDict.values():
834 861 procUnitConfObj.printattr()
835 862
836 863 def createObjects(self):
837 864
838 865 # for readUnitConfObj in self.readUnitConfObjList:
839 866 # readUnitConfObj.createObjects()
840 867
841 868 for procUnitConfObj in self.procUnitConfObjDict.values():
842 869 procUnitConfObj.createObjects()
843 870
844 871 def __connect(self, objIN, thisObj):
845 872
846 873 thisObj.setInput(objIN.getOutputObj())
847 874
848 875 def connectObjects(self):
849 876
850 877 for thisPUConfObj in self.procUnitConfObjDict.values():
851 878
852 879 inputId = thisPUConfObj.getInputId()
853 880
854 881 if int(inputId) == 0:
855 882 continue
856 883
857 884 #Get input object
858 885 puConfINObj = self.procUnitConfObjDict[inputId]
859 886 puObjIN = puConfINObj.getProcUnitObj()
860 887
861 888 #Get current object
862 889 thisPUObj = thisPUConfObj.getProcUnitObj()
863 890
864 891 self.__connect(puObjIN, thisPUObj)
865 892
866 893 def run(self):
867 894
868 895 # for readUnitConfObj in self.readUnitConfObjList:
869 896 # readUnitConfObj.run()
870 897 print
871 898 print "*"*40
872 899 print " Starting SIGNAL CHAIN PROCESSING "
873 900 print "*"*40
874 901 print
875 902
876 903 keyList = self.procUnitConfObjDict.keys()
877 904 keyList.sort()
878 905
879 906 while(True):
880 907
881 908 finalSts = False
882 909
883 910 for procKey in keyList:
884 911 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
885 912
886 913 procUnitConfObj = self.procUnitConfObjDict[procKey]
887 914 sts = procUnitConfObj.run()
888 915 finalSts = finalSts or sts
889 916
890 917 #If every process unit finished so end process
891 918 if not(finalSts):
892 919 print "Every process unit have finished"
893 920 break
894 921
895 922 if self.control['pause']:
896 923 print "Process suspended"
897 924
898 925 while True:
899 926 sleep(0.1)
900 927
901 928 if not self.control['pause']:
902 929 break
903 930
904 931 if self.control['stop']:
905 932 break
906 933 print "Process reinitialized"
907 934
908 935 if self.control['stop']:
909 936 print "Process stopped"
910 937 break
911 938
912 939 #Closing every process
913 940 for procKey in keyList:
914 941 procUnitConfObj = self.procUnitConfObjDict[procKey]
915 942 procUnitConfObj.close()
916 943
917 944 print "Process finished"
918 945
919 946 def start(self, filename):
920 947
921 948 self.writeXml(filename)
922 949 self.readXml(filename)
923 950
924 951 self.createObjects()
925 952 self.connectObjects()
926 953 self.run()
927 954
928 955 if __name__ == '__main__':
929 956
930 957 desc = "Segundo Test"
931 958 filename = "schain.xml"
932 959
933 960 controllerObj = Project()
934 961
935 962 controllerObj.setup(id = '191', name='test01', description=desc)
936 963
937 964 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
938 965 path='data/rawdata/',
939 966 startDate='2011/01/01',
940 967 endDate='2012/12/31',
941 968 startTime='00:00:00',
942 969 endTime='23:59:59',
943 970 online=1,
944 971 walk=1)
945 972
946 973 # opObj00 = readUnitConfObj.addOperation(name='printInfo')
947 974
948 975 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
949 976
950 977 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
951 978 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
952 979
953 980 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
954 981 opObj10.addParameter(name='minHei', value='90', format='float')
955 982 opObj10.addParameter(name='maxHei', value='180', format='float')
956 983
957 984 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
958 985 opObj12.addParameter(name='n', value='10', format='int')
959 986
960 987 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
961 988 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
962 989 # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
963 990
964 991
965 992 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
966 993 opObj11.addParameter(name='idfigure', value='1', format='int')
967 994 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
968 995 opObj11.addParameter(name='zmin', value='40', format='int')
969 996 opObj11.addParameter(name='zmax', value='90', format='int')
970 997 opObj11.addParameter(name='showprofile', value='1', format='int')
971 998
972 999 # opObj11 = procUnitConfObj1.addOperation(name='CrossSpectraPlot', optype='external')
973 1000 # opObj11.addParameter(name='idfigure', value='2', format='int')
974 1001 # opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
975 1002 # opObj11.addParameter(name='zmin', value='40', format='int')
976 1003 # opObj11.addParameter(name='zmax', value='90', format='int')
977 1004
978 1005
979 1006 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
980 1007 #
981 1008 # opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='external')
982 1009 # opObj12.addParameter(name='n', value='2', format='int')
983 1010 # opObj12.addParameter(name='overlapping', value='1', format='int')
984 1011 #
985 1012 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
986 1013 # procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
987 1014 #
988 1015 # opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='external')
989 1016 # opObj11.addParameter(name='idfigure', value='2', format='int')
990 1017 # opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
991 1018 # opObj11.addParameter(name='zmin', value='40', format='int')
992 1019 # opObj11.addParameter(name='zmax', value='90', format='int')
993 1020 # opObj11.addParameter(name='showprofile', value='1', format='int')
994 1021
995 1022 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
996 1023 # opObj11.addParameter(name='idfigure', value='10', format='int')
997 1024 # opObj11.addParameter(name='wintitle', value='RTI', format='str')
998 1025 ## opObj11.addParameter(name='xmin', value='21', format='float')
999 1026 ## opObj11.addParameter(name='xmax', value='22', format='float')
1000 1027 # opObj11.addParameter(name='zmin', value='40', format='int')
1001 1028 # opObj11.addParameter(name='zmax', value='90', format='int')
1002 1029 # opObj11.addParameter(name='showprofile', value='1', format='int')
1003 1030 # opObj11.addParameter(name='timerange', value=str(60), format='int')
1004 1031
1005 1032 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
1006 1033 # opObj10.addParameter(name='channelList', value='0,2,4,6', format='intlist')
1007 1034 #
1008 1035 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
1009 1036 # opObj12.addParameter(name='n', value='2', format='int')
1010 1037 #
1011 1038 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
1012 1039 # opObj11.addParameter(name='idfigure', value='2', format='int')
1013 1040 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
1014 1041 # opObj11.addParameter(name='zmin', value='70', format='int')
1015 1042 # opObj11.addParameter(name='zmax', value='90', format='int')
1016 1043 #
1017 1044 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
1018 1045 # opObj10.addParameter(name='channelList', value='2,6', format='intlist')
1019 1046 #
1020 1047 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
1021 1048 # opObj12.addParameter(name='n', value='2', format='int')
1022 1049 #
1023 1050 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
1024 1051 # opObj11.addParameter(name='idfigure', value='3', format='int')
1025 1052 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
1026 1053 # opObj11.addParameter(name='zmin', value='70', format='int')
1027 1054 # opObj11.addParameter(name='zmax', value='90', format='int')
1028 1055
1029 1056
1030 1057 # opObj12 = procUnitConfObj1.addOperation(name='decoder')
1031 1058 # opObj12.addParameter(name='ncode', value='2', format='int')
1032 1059 # opObj12.addParameter(name='nbauds', value='8', format='int')
1033 1060 # opObj12.addParameter(name='code0', value='001110011', format='int')
1034 1061 # opObj12.addParameter(name='code1', value='001110011', format='int')
1035 1062
1036 1063
1037 1064
1038 1065 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj1.getId())
1039 1066 #
1040 1067 # opObj21 = procUnitConfObj2.addOperation(name='IncohInt', optype='external')
1041 1068 # opObj21.addParameter(name='n', value='2', format='int')
1042 1069 #
1043 1070 # opObj11 = procUnitConfObj2.addOperation(name='SpectraPlot', optype='external')
1044 1071 # opObj11.addParameter(name='idfigure', value='4', format='int')
1045 1072 # opObj11.addParameter(name='wintitle', value='SpectraPlot OBJ 2', format='str')
1046 1073 # opObj11.addParameter(name='zmin', value='70', format='int')
1047 1074 # opObj11.addParameter(name='zmax', value='90', format='int')
1048 1075
1049 1076 print "Escribiendo el archivo XML"
1050 1077
1051 1078 controllerObj.writeXml(filename)
1052 1079
1053 1080 print "Leyendo el archivo XML"
1054 1081 controllerObj.readXml(filename)
1055 1082 #controllerObj.printattr()
1056 1083
1057 1084 controllerObj.createObjects()
1058 1085 controllerObj.connectObjects()
1059 1086 controllerObj.run()
1060 1087
1061 1088 No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,115 +1,125
1 1 import threading
2 2 import Queue
3 3 try:
4 4 from gevent import sleep
5 5 except:
6 6 from time import sleep
7 7
8 8 from schainpy.controller import Project
9 9 from command import *
10 10
11 11 class ControllerThread(threading.Thread):
12
12 13 def __init__(self, filename, data_q=None):
14
13 15 super(ControllerThread, self).__init__()
16 self.setDaemon(True)
17
14 18 self.filename = filename
15 19 self.data_q = data_q
16 20 self.control = {'stop':False,'pause':False}
17 21
18 22 def stop(self):
19 23 self.control['stop'] = True
20 24
21 25 def pause(self):
22 26 self.control['pause'] = not(self.control['pause'])
23 27
24 28 def run(self):
25 29 self.control['stop'] = False
26 30 self.control['pause'] = False
27 31 self.controllerObj = Project(self.control, self.data_q)
28 32 self.controllerObj.readXml(self.filename)
29 33 self.controllerObj.createObjects()
30 34 self.controllerObj.connectObjects()
31 35 self.controllerObj.run()
32 36
33 37 class CommCtrlProcessThread(threading.Thread):
34 38 """ Implements the threading.Thread interface (start, join, etc.) and
35 39 can be controlled via the cmd_q Queue attribute. Replies are placed in
36 40 the reply_q Queue attribute.
37 41 """
38 42 def __init__(self, cmd_q=Queue.Queue(), reply_q=Queue.Queue()):
39 43 super(CommCtrlProcessThread, self).__init__()
40 44 self.cmd_q = cmd_q
41 45 # self.reply_q = reply_q
42 46
43 47 # self.print_q = Queue.Queue()
44 48 # self.data_q = Queue.Queue()
45 49
46 50
47 51 self.alive = threading.Event()
48 52 self.setDaemon(True)
49 53 self.alive.set()
50 54 self.socket = None
51 55
52 56 self.socketIO = None
53 57 self.mySocket = None
54 58
55 59 self.controllerObj = None
56 60
57 61 self.handlers = {
58 62 ProcessCommand.PROCESS: self._handle_ioPROCESSTHREAD,
59 63 ProcessCommand.MESSAGE: self._handle_ioMESSAGE,
60 64 ProcessCommand.DATA: self._handle_ioDATA,
61 65 ProcessCommand.STOP: self._handle_ioSTOP,
62 66 ProcessCommand.PAUSE: self._handle_ioPAUSE
63 67 }
64 68
65 69 def run(self):
66 70
67 71 while self.alive.isSet():
68 72 try:
69 73 cmd = self.cmd_q.get(True, 0.1)
70 74 self.handlers[cmd.type](cmd)
71 75 except Queue.Empty as e:
72 sleep(0.1)
73 76 continue
74 77
75 78 def isRunning(self):
76 79
77 80 if self.controllerObj == None:
78 81 return False
79 82
80 83 if self.controllerObj.isAlive():
81 84 return True
82 85
83 86 return False
84 87
85 88 def _handle_ioPROCESSTHREAD(self, cmd):
86 89 filename = cmd.data
87 90 self.controllerObj = ControllerThread(filename=filename)
88 91 self.controllerObj.start()
89 92
90 93 def _handle_ioPAUSE(self, cmd):
91 94 self.controllerObj.pause()
92 95
93 96 def _handle_ioSTOP(self, cmd):
94 97 self.controllerObj.stop()
98
99 while self.controllerObj.isAlive():
100 self.console.clear()
101 self.console.append("Close graphics before continue...")
102 sleep(0.1)
103
104
95 105 self.controllerObj.join()
96 106 # print "Process thread finished"
97 107
98 108 def _handle_ioDATA(self, cmd):
99 109 self.reply_q.put(self._success_reply_data(data=cmd.data))
100 110
101 111 def _handle_ioMESSAGE(self, cmd):
102 112 self.reply_q.put(self._success_reply_message(data=cmd.data))
103 113
104 114 def _success_reply_data(self, data=None):
105 115 return ClientReply(ClientReply.DATA, data)
106 116
107 117 def _success_reply_message(self, data=None):
108 118 return ClientReply(ClientReply.MESSAGE, data)
109 119
110 120 def join(self, timeout=None):
111 121 self.alive.clear()
112 122 threading.Thread.join(self, timeout)
113 123
114 124
115 125 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,171 +1,173
1 1
2 2 from PyQt4 import QtCore, QtGui
3 3
4 4 try:
5 5 _fromUtf8 = QtCore.QString.fromUtf8
6 6 except AttributeError:
7 7 def _fromUtf8(s):
8 8 return s
9 9
10 10 try:
11 11 _encoding = QtGui.QApplication.UnicodeUTF8
12 12 def _translate(context, text, disambig):
13 13 return QtGui.QApplication.translate(context, text, disambig, _encoding)
14 14 except AttributeError:
15 15 def _translate(context, text, disambig):
16 16 return QtGui.QApplication.translate(context, text, disambig)
17 17
18 18 class Ui_ProjectTab(object):
19 19
20 20 def setupUi(self):
21 21
22 22 self.tabProject = QtGui.QWidget()
23 23 self.tabProject.setObjectName(_fromUtf8("tabProject"))
24 24 self.gridLayout_15 = QtGui.QGridLayout(self.tabProject)
25 25 self.gridLayout_15.setObjectName(_fromUtf8("gridLayout_15"))
26 26 self.frame = QtGui.QFrame(self.tabProject)
27 27 self.frame.setFrameShape(QtGui.QFrame.StyledPanel)
28 28 self.frame.setFrameShadow(QtGui.QFrame.Raised)
29 29 self.frame.setObjectName(_fromUtf8("frame"))
30 30 self.gridLayout_2 = QtGui.QGridLayout(self.frame)
31 31 self.gridLayout_2.setObjectName(_fromUtf8("gridLayout_2"))
32 32 self.label = QtGui.QLabel(self.frame)
33 33 self.label.setObjectName(_fromUtf8("label"))
34 34 self.gridLayout_2.addWidget(self.label, 0, 0, 1, 1)
35 35 self.proName = QtGui.QLineEdit(self.frame)
36 36 self.proName.setObjectName(_fromUtf8("proName"))
37 37 self.gridLayout_2.addWidget(self.proName, 0, 1, 1, 8)
38 38 self.label_11 = QtGui.QLabel(self.frame)
39 39 self.label_11.setObjectName(_fromUtf8("label_11"))
40 40 self.gridLayout_2.addWidget(self.label_11, 1, 0, 1, 1)
41 41 self.proComDataType = QtGui.QComboBox(self.frame)
42 42 self.proComDataType.setObjectName(_fromUtf8("proComDataType"))
43 43 self.proComDataType.addItem(_fromUtf8(""))
44 44 self.proComDataType.addItem(_fromUtf8(""))
45 45 self.proComDataType.addItem(_fromUtf8(""))
46 self.proComDataType.addItem(_fromUtf8(""))
46 47 self.gridLayout_2.addWidget(self.proComDataType, 1, 1, 1, 5)
47 48 self.proDataType = QtGui.QLineEdit(self.frame)
48 49 self.proDataType.setObjectName(_fromUtf8("proDataType"))
49 50 self.gridLayout_2.addWidget(self.proDataType, 1, 6, 1, 3)
50 51 self.label_15 = QtGui.QLabel(self.frame)
51 52 self.label_15.setObjectName(_fromUtf8("label_15"))
52 53 self.gridLayout_2.addWidget(self.label_15, 2, 0, 1, 1)
53 54 self.proToolPath = QtGui.QToolButton(self.frame)
54 55 self.proToolPath.setObjectName(_fromUtf8("proToolPath"))
55 56 self.gridLayout_2.addWidget(self.proToolPath, 2, 1, 1, 1)
56 57 self.proDataPath = QtGui.QLineEdit(self.frame)
57 58 self.proDataPath.setObjectName(_fromUtf8("proDataPath"))
58 59 self.gridLayout_2.addWidget(self.proDataPath, 2, 2, 1, 7)
59 60 self.label_23 = QtGui.QLabel(self.frame)
60 61 self.label_23.setObjectName(_fromUtf8("label_23"))
61 62 self.gridLayout_2.addWidget(self.label_23, 3, 0, 1, 1)
62 63 self.proComReadMode = QtGui.QComboBox(self.frame)
63 64 self.proComReadMode.setObjectName(_fromUtf8("proComReadMode"))
64 65 self.proComReadMode.addItem(_fromUtf8(""))
65 66 self.proComReadMode.addItem(_fromUtf8(""))
66 67 self.gridLayout_2.addWidget(self.proComReadMode, 3, 1, 1, 2)
67 68 self.label_33 = QtGui.QLabel(self.frame)
68 69 self.label_33.setObjectName(_fromUtf8("label_33"))
69 70 self.gridLayout_2.addWidget(self.label_33, 3, 5, 1, 2)
70 71 self.proDelay = QtGui.QLineEdit(self.frame)
71 72 self.proDelay.setObjectName(_fromUtf8("proDelay"))
72 73 self.gridLayout_2.addWidget(self.proDelay, 3, 8, 1, 1)
73 74 self.label_32 = QtGui.QLabel(self.frame)
74 75 self.label_32.setObjectName(_fromUtf8("label_32"))
75 76 self.gridLayout_2.addWidget(self.label_32, 4, 0, 1, 1)
76 77 self.proComWalk = QtGui.QComboBox(self.frame)
77 78 self.proComWalk.setObjectName(_fromUtf8("proComWalk"))
78 79 self.proComWalk.addItem(_fromUtf8(""))
79 80 self.proComWalk.addItem(_fromUtf8(""))
80 81 self.gridLayout_2.addWidget(self.proComWalk, 4, 1, 1, 8)
81 82 self.proLoadButton = QtGui.QPushButton(self.frame)
82 83 self.proLoadButton.setObjectName(_fromUtf8("proLoadButton"))
83 84 self.gridLayout_2.addWidget(self.proLoadButton, 5, 0, 1, 9)
84 85 self.label_10 = QtGui.QLabel(self.frame)
85 86 self.label_10.setObjectName(_fromUtf8("label_10"))
86 87 self.gridLayout_2.addWidget(self.label_10, 3, 3, 1, 1)
87 88 self.proSet = QtGui.QLineEdit(self.frame)
88 89 self.proSet.setObjectName(_fromUtf8("proSet"))
89 90 self.gridLayout_2.addWidget(self.proSet, 3, 4, 1, 1)
90 91 self.gridLayout_15.addWidget(self.frame, 0, 0, 1, 1)
91 92 self.frame_2 = QtGui.QFrame(self.tabProject)
92 93 self.frame_2.setFrameShape(QtGui.QFrame.StyledPanel)
93 94 self.frame_2.setFrameShadow(QtGui.QFrame.Raised)
94 95 self.frame_2.setObjectName(_fromUtf8("frame_2"))
95 96 self.gridLayout_10 = QtGui.QGridLayout(self.frame_2)
96 97 self.gridLayout_10.setObjectName(_fromUtf8("gridLayout_10"))
97 98 self.label_27 = QtGui.QLabel(self.frame_2)
98 99 self.label_27.setObjectName(_fromUtf8("label_27"))
99 100 self.gridLayout_10.addWidget(self.label_27, 0, 0, 1, 1)
100 101 self.proComStartDate = QtGui.QComboBox(self.frame_2)
101 102 self.proComStartDate.setObjectName(_fromUtf8("proComStartDate"))
102 103 self.gridLayout_10.addWidget(self.proComStartDate, 0, 1, 1, 1)
103 104 self.label_28 = QtGui.QLabel(self.frame_2)
104 105 self.label_28.setObjectName(_fromUtf8("label_28"))
105 106 self.gridLayout_10.addWidget(self.label_28, 1, 0, 1, 1)
106 107 self.proComEndDate = QtGui.QComboBox(self.frame_2)
107 108 self.proComEndDate.setObjectName(_fromUtf8("proComEndDate"))
108 109 self.gridLayout_10.addWidget(self.proComEndDate, 1, 1, 1, 1)
109 110 self.label_2 = QtGui.QLabel(self.frame_2)
110 111 self.label_2.setObjectName(_fromUtf8("label_2"))
111 112 self.gridLayout_10.addWidget(self.label_2, 2, 0, 1, 1)
112 113 self.proStartTime = QtGui.QTimeEdit(self.frame_2)
113 114 self.proStartTime.setObjectName(_fromUtf8("proStartTime"))
114 115 self.gridLayout_10.addWidget(self.proStartTime, 2, 1, 1, 1)
115 116 self.label_3 = QtGui.QLabel(self.frame_2)
116 117 self.label_3.setObjectName(_fromUtf8("label_3"))
117 118 self.gridLayout_10.addWidget(self.label_3, 3, 0, 1, 1)
118 119 self.proEndTime = QtGui.QTimeEdit(self.frame_2)
119 120 self.proEndTime.setObjectName(_fromUtf8("proEndTime"))
120 121 self.gridLayout_10.addWidget(self.proEndTime, 3, 1, 1, 1)
121 122 self.label_30 = QtGui.QLabel(self.frame_2)
122 123 self.label_30.setObjectName(_fromUtf8("label_30"))
123 124 self.gridLayout_10.addWidget(self.label_30, 4, 0, 1, 1)
124 125 self.proDescription = QtGui.QTextEdit(self.frame_2)
125 126 self.proDescription.setObjectName(_fromUtf8("proDescription"))
126 127 self.gridLayout_10.addWidget(self.proDescription, 4, 1, 1, 1)
127 128 self.gridLayout_15.addWidget(self.frame_2, 1, 0, 1, 1)
128 129 self.frame_3 = QtGui.QFrame(self.tabProject)
129 130 self.frame_3.setFrameShape(QtGui.QFrame.StyledPanel)
130 131 self.frame_3.setFrameShadow(QtGui.QFrame.Raised)
131 132 self.frame_3.setObjectName(_fromUtf8("frame_3"))
132 133 self.gridLayout_14 = QtGui.QGridLayout(self.frame_3)
133 134 self.gridLayout_14.setObjectName(_fromUtf8("gridLayout_14"))
134 135 self.proOk = QtGui.QPushButton(self.frame_3)
135 136 self.proOk.setObjectName(_fromUtf8("proOk"))
136 137 self.gridLayout_14.addWidget(self.proOk, 0, 0, 1, 1)
137 138 self.proClear = QtGui.QPushButton(self.frame_3)
138 139 self.proClear.setObjectName(_fromUtf8("proClear"))
139 140 self.gridLayout_14.addWidget(self.proClear, 0, 1, 1, 1)
140 141 self.gridLayout_15.addWidget(self.frame_3, 2, 0, 1, 1)
141 142
142 143 self.tabWidgetProject.addTab(self.tabProject, _fromUtf8(""))
143 144
144 145 def retranslateUi(self):
145 146
146 147 self.label.setText(_translate("MainWindow", "Project Name :", None))
147 148 self.label_11.setText(_translate("MainWindow", "DataType :", None))
148 149 self.proComDataType.setItemText(0, _translate("MainWindow", "Voltage", None))
149 150 self.proComDataType.setItemText(1, _translate("MainWindow", "Spectra", None))
150 151 self.proComDataType.setItemText(2, _translate("MainWindow", "Fits", None))
152 self.proComDataType.setItemText(3, _translate("MainWindow", "USRP", None))
151 153 self.label_15.setText(_translate("MainWindow", "DataPath :", None))
152 154 self.proToolPath.setText(_translate("MainWindow", "...", None))
153 155 self.label_23.setText(_translate("MainWindow", "Read Mode:", None))
154 156 self.proComReadMode.setItemText(0, _translate("MainWindow", "Offline", None))
155 157 self.proComReadMode.setItemText(1, _translate("MainWindow", "Online", None))
156 158 self.label_33.setText(_translate("MainWindow", "Delay:", None))
157 159 self.label_32.setText(_translate("MainWindow", "Walk :", None))
158 160 self.proComWalk.setItemText(0, _translate("MainWindow", "On Files", None))
159 161 self.proComWalk.setItemText(1, _translate("MainWindow", "On Folder", None))
160 162 self.proLoadButton.setText(_translate("MainWindow", "Load", None))
161 163 self.label_10.setText(_translate("MainWindow", "Set:", None))
162 164 self.label_27.setText(_translate("MainWindow", "Star Date:", None))
163 165 self.label_28.setText(_translate("MainWindow", "End Date:", None))
164 166 self.label_2.setText(_translate("MainWindow", "Start Time:", None))
165 167 self.label_3.setText(_translate("MainWindow", "End Time:", None))
166 168 self.label_30.setText(_translate("MainWindow", "Description:", None))
167 169 self.proOk.setText(_translate("MainWindow", "Ok", None))
168 170 self.proClear.setText(_translate("MainWindow", "Clear", None))
169 171
170 172 self.tabWidgetProject.setTabText(self.tabWidgetProject.indexOf(self.tabProject), _translate("MainWindow", "Project", None))
171 173 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,903 +1,903
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6
7 7 from schainpy.model.data.jrodata import *
8 8 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 9 from schainpy.model.io.jroIO_base import *
10 10
11 11
12 12 class HDF5Reader(ProcessingUnit):
13 13
14 14 ext = ".hdf5"
15 15
16 16 optchar = "D"
17 17
18 18 timezone = None
19 19
20 20 secStart = None
21 21
22 22 secEnd = None
23 23
24 24 fileIndex = None
25 25
26 26 blockIndex = None
27 27
28 28 blocksPerFile = None
29 29
30 30 path = None
31 31
32 32 #List of Files
33 33
34 34 filenameList = None
35 35
36 36 datetimeList = None
37 37
38 38 #Hdf5 File
39 39
40 40 fpMetadata = None
41 41
42 42 pathMeta = None
43 43
44 44 listMetaname = None
45 45
46 46 listMeta = None
47 47
48 48 listDataname = None
49 49
50 50 listData = None
51 51
52 52 listShapes = None
53 53
54 54 fp = None
55 55
56 56 #dataOut reconstruction
57 57
58 58 dataOut = None
59 59
60 60 nRecords = None
61 61
62 62
63 63 def __init__(self):
64 64 self.dataOut = self.__createObjByDefault()
65 65 return
66 66
67 67 def __createObjByDefault(self):
68 68
69 69 dataObj = Parameters()
70 70
71 71 return dataObj
72 72
73 73 def setup(self,path=None,
74 74 startDate=None,
75 75 endDate=None,
76 76 startTime=datetime.time(0,0,0),
77 77 endTime=datetime.time(23,59,59),
78 78 walk=True,
79 79 timezone='ut',
80 80 all=0,
81 81 online=False,
82 82 ext=None):
83 83
84 84 if ext==None:
85 85 ext = self.ext
86 86 self.timezone = timezone
87 87 # self.all = all
88 88 # self.online = online
89 89 self.path = path
90 90
91 91 startDateTime = datetime.datetime.combine(startDate,startTime)
92 92 endDateTime = datetime.datetime.combine(endDate,endTime)
93 93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
94 94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
95 95
96 96 self.secStart = secStart
97 97 self.secEnd = secEnd
98 98
99 99 if not(online):
100 100 #Busqueda de archivos offline
101 101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
102 102 else:
103 103 self.__searchFilesOnline(path, walk)
104 104
105 105 if not(self.filenameList):
106 106 print "There is no files into the folder: %s"%(path)
107 107 sys.exit(-1)
108 108
109 109 # self.__getExpParameters()
110 110
111 111 self.fileIndex = -1
112 112
113 113 self.__setNextFileOffline()
114 114
115 115 self.__readMetadata()
116 116
117 117 self.blockIndex = 0
118 118
119 119 return
120 120
121 121 def __searchFilesOffline(self,
122 122 path,
123 123 startDate,
124 124 endDate,
125 125 ext,
126 126 startTime=datetime.time(0,0,0),
127 127 endTime=datetime.time(23,59,59),
128 128 secStart = 0,
129 129 secEnd = numpy.inf,
130 130 walk=True):
131 131
132 132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
133 133 #
134 134 # self.__checkPath()
135 135 #
136 136 # self.__findDataForDates()
137 137 #
138 138 # self.__selectDataForTimes()
139 139 #
140 140 # for i in range(len(self.filenameList)):
141 141 # print "%s" %(self.filenameList[i])
142 142
143 143 pathList = []
144 144
145 145 if not walk:
146 146 #pathList.append(path)
147 147 multi_path = path.split(',')
148 148 for single_path in multi_path:
149 149 pathList.append(single_path)
150 150
151 151 else:
152 152 #dirList = []
153 153 multi_path = path.split(',')
154 154 for single_path in multi_path:
155 155 dirList = []
156 156 for thisPath in os.listdir(single_path):
157 157 if not os.path.isdir(os.path.join(single_path,thisPath)):
158 158 continue
159 if not isDoyFolder(thisPath):
159 if not isRadarFolder(thisPath):
160 160 continue
161 161
162 162 dirList.append(thisPath)
163 163
164 164 if not(dirList):
165 165 return None, None
166 166
167 167 thisDate = startDate
168 168
169 169 while(thisDate <= endDate):
170 170 year = thisDate.timetuple().tm_year
171 171 doy = thisDate.timetuple().tm_yday
172 172
173 173 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
174 174 if len(matchlist) == 0:
175 175 thisDate += datetime.timedelta(1)
176 176 continue
177 177 for match in matchlist:
178 178 pathList.append(os.path.join(single_path,match))
179 179
180 180 thisDate += datetime.timedelta(1)
181 181
182 182 if pathList == []:
183 183 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
184 184 return None, None
185 185
186 186 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
187 187
188 188 filenameList = []
189 189 datetimeList = []
190 190 pathDict = {}
191 191 filenameList_to_sort = []
192 192
193 193 for i in range(len(pathList)):
194 194
195 195 thisPath = pathList[i]
196 196
197 197 fileList = glob.glob1(thisPath, "*%s" %ext)
198 198 fileList.sort()
199 199 pathDict.setdefault(fileList[0])
200 200 pathDict[fileList[0]] = i
201 201 filenameList_to_sort.append(fileList[0])
202 202
203 203 filenameList_to_sort.sort()
204 204
205 205 for file in filenameList_to_sort:
206 206 thisPath = pathList[pathDict[file]]
207 207
208 208 fileList = glob.glob1(thisPath, "*%s" %ext)
209 209 fileList.sort()
210 210
211 211 for file in fileList:
212 212
213 213 filename = os.path.join(thisPath,file)
214 214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
215 215
216 216 if not(thisDatetime):
217 217 continue
218 218
219 219 filenameList.append(filename)
220 220 datetimeList.append(thisDatetime)
221 221
222 222 if not(filenameList):
223 223 print "Any file was found for the time range %s - %s" %(startTime, endTime)
224 224 return None, None
225 225
226 226 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
227 227 print
228 228
229 229 for i in range(len(filenameList)):
230 230 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
231 231
232 232 self.filenameList = filenameList
233 233 self.datetimeList = datetimeList
234 234
235 235 return pathList, filenameList
236 236
237 237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
238 238 """
239 239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
240 240
241 241 Inputs:
242 242 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
243 243
244 244 startTime : tiempo inicial del rango seleccionado en formato datetime.time
245 245
246 246 endTime : tiempo final del rango seleccionado en formato datetime.time
247 247
248 248 Return:
249 249 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
250 250 fecha especificado, de lo contrario retorna False.
251 251
252 252 Excepciones:
253 253 Si el archivo no existe o no puede ser abierto
254 254 Si la cabecera no puede ser leida.
255 255
256 256 """
257 257
258 258 try:
259 259 fp = fp = h5py.File(filename,'r')
260 260 except IOError:
261 261 traceback.print_exc()
262 262 raise IOError, "The file %s can't be opened" %(filename)
263 263
264 264 grp = fp['Data']
265 265 timeAux = grp['time']
266 266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
267 267
268 268 fp.close()
269 269
270 270 if self.timezone == 'lt':
271 271 time0 -= 5*3600
272 272
273 273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
274 274
275 275 if not (numpy.any(boolTimer)):
276 276 return None
277 277
278 278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
279 279 return thisDatetime
280 280
281 281 def __checkPath(self):
282 282 if os.path.exists(self.path):
283 283 self.status = 1
284 284 else:
285 285 self.status = 0
286 286 print 'Path:%s does not exists'%self.path
287 287
288 288 return
289 289
290 290 def __setNextFileOffline(self):
291 291 idFile = self.fileIndex
292 292 idFile += 1
293 293
294 294 if not(idFile < len(self.filenameList)):
295 295 print "No more Files"
296 296 return 0
297 297
298 298 filename = self.filenameList[idFile]
299 299
300 300 filePointer = h5py.File(filename,'r')
301 301
302 302 self.flagIsNewFile = 1
303 303 self.fileIndex = idFile
304 304 self.filename = filename
305 305
306 306 self.fp = filePointer
307 307
308 308 print "Setting the file: %s"%self.filename
309 309
310 310 self.__readMetadata()
311 311 self.__setBlockList()
312 312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
313 313 self.nRecords = self.fp['Data'].attrs['nRecords']
314 314 self.blockIndex = 0
315 315 return 1
316 316
317 317 def __setBlockList(self):
318 318 '''
319 319 self.fp
320 320 self.startDateTime
321 321 self.endDateTime
322 322
323 323 self.blockList
324 324 self.blocksPerFile
325 325
326 326 '''
327 327 filePointer = self.fp
328 328 secStart = self.secStart
329 329 secEnd = self.secEnd
330 330
331 331 grp = filePointer['Data']
332 332 timeVector = grp['time'].value.astype(numpy.float)[0]
333 333
334 334 if self.timezone == 'lt':
335 335 timeVector -= 5*3600
336 336
337 337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
338 338
339 339 self.blockList = ind
340 340 self.blocksPerFile = len(ind)
341 341
342 342 return
343 343
344 344 def __readMetadata(self):
345 345 '''
346 346 self.pathMeta
347 347
348 348 self.listShapes
349 349 self.listMetaname
350 350 self.listMeta
351 351
352 352 '''
353 353
354 354 grp = self.fp['Data']
355 355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
356 356
357 357 if pathMeta == self.pathMeta:
358 358 return
359 359 else:
360 360 self.pathMeta = pathMeta
361 361
362 362 filePointer = h5py.File(self.pathMeta,'r')
363 363 groupPointer = filePointer['Metadata']
364 364
365 365 listMetaname = []
366 366 listMetadata = []
367 367 for item in groupPointer.items():
368 368 name = item[0]
369 369
370 370 if name=='array dimensions':
371 371 table = groupPointer[name][:]
372 372 listShapes = {}
373 373 for shapes in table:
374 374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
375 375 else:
376 376 data = groupPointer[name].value
377 377 listMetaname.append(name)
378 378 listMetadata.append(data)
379 379
380 380 if name=='type':
381 381 self.__initDataOut(data)
382 382
383 383 filePointer.close()
384 384
385 385 self.listShapes = listShapes
386 386 self.listMetaname = listMetaname
387 387 self.listMeta = listMetadata
388 388
389 389 return
390 390
391 391 def __readData(self):
392 392 grp = self.fp['Data']
393 393 listdataname = []
394 394 listdata = []
395 395
396 396 for item in grp.items():
397 397 name = item[0]
398 398
399 399 if name == 'time':
400 400 listdataname.append('utctime')
401 401 timeAux = grp[name].value.astype(numpy.float)[0]
402 402 listdata.append(timeAux)
403 403 continue
404 404
405 405 listdataname.append(name)
406 406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
407 407 listdata.append(array)
408 408
409 409 self.listDataname = listdataname
410 410 self.listData = listdata
411 411 return
412 412
413 413 def __setDataArray(self, nRecords, dataset, shapes):
414 414
415 415 nChannels = shapes[0] #Dimension 0
416 416
417 417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
418 418
419 419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
420 420
421 421 mode = shapes[3]
422 422
423 423 # if nPoints>1:
424 424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
425 425 # else:
426 426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
427 427 #
428 428 # chn = 'channel'
429 429 #
430 430 # for i in range(nChannels):
431 431 #
432 432 # data = dataset[chn + str(i)].value
433 433 #
434 434 # if nPoints>1:
435 435 # data = numpy.rollaxis(data,2)
436 436 #
437 437 # arrayData[:,i,:] = data
438 438
439 439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
440 440 doSqueeze = False
441 441 if mode == 0:
442 442 strds = 'channel'
443 443 nDatas = nChannels
444 444 newShapes = (nRecords,nPoints,nSamples)
445 445 if nPoints == 1:
446 446 doSqueeze = True
447 447 axisSqueeze = 2
448 448 else:
449 449 strds = 'param'
450 450 nDatas = nPoints
451 451 newShapes = (nRecords,nChannels,nSamples)
452 452 if nChannels == 1:
453 453 doSqueeze = True
454 454 axisSqueeze = 1
455 455
456 456 for i in range(nDatas):
457 457
458 458 data = dataset[strds + str(i)].value
459 459 data = data.reshape(newShapes)
460 460
461 461 if mode == 0:
462 462 arrayData[:,i,:,:] = data
463 463 else:
464 464 arrayData[:,:,i,:] = data
465 465
466 466 if doSqueeze:
467 467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
468 468
469 469 return arrayData
470 470
471 471 def __initDataOut(self, type):
472 472
473 473 # if type =='Parameters':
474 474 # self.dataOut = Parameters()
475 475 # elif type =='Spectra':
476 476 # self.dataOut = Spectra()
477 477 # elif type =='Voltage':
478 478 # self.dataOut = Voltage()
479 479 # elif type =='Correlation':
480 480 # self.dataOut = Correlation()
481 481
482 482 return
483 483
484 484 def __setDataOut(self):
485 485 listMeta = self.listMeta
486 486 listMetaname = self.listMetaname
487 487 listDataname = self.listDataname
488 488 listData = self.listData
489 489
490 490 blockIndex = self.blockIndex
491 491 blockList = self.blockList
492 492
493 493 for i in range(len(listMeta)):
494 494 setattr(self.dataOut,listMetaname[i],listMeta[i])
495 495
496 496 for j in range(len(listData)):
497 497 if listDataname[j]=='utctime':
498 498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
499 499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
500 500 continue
501 501
502 502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
503 503
504 504 return self.dataOut.data_param
505 505
506 506 def getData(self):
507 507
508 508 # if self.flagNoMoreFiles:
509 509 # self.dataOut.flagNoData = True
510 510 # print 'Process finished'
511 511 # return 0
512 512 #
513 513 if self.blockIndex==self.blocksPerFile:
514 514 if not( self.__setNextFileOffline() ):
515 515 self.dataOut.flagNoData = True
516 516 return 0
517 517
518 518 #
519 519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
520 520 # self.dataOut.flagNoData = True
521 521 # return 0
522 522
523 523 self.__readData()
524 524 self.__setDataOut()
525 525 self.dataOut.flagNoData = False
526 526
527 527 self.blockIndex += 1
528 528
529 529 return
530 530
531 531 def run(self, **kwargs):
532 532
533 533 if not(self.isConfig):
534 534 self.setup(**kwargs)
535 535 # self.setObjProperties()
536 536 self.isConfig = True
537 537
538 538 self.getData()
539 539
540 540 return
541 541
542 542 class HDF5Writer(Operation):
543 543
544 544 ext = ".hdf5"
545 545
546 546 optchar = "D"
547 547
548 548 metaoptchar = "M"
549 549
550 550 metaFile = None
551 551
552 552 path = None
553 553
554 554 setFile = None
555 555
556 556 fp = None
557 557
558 558 grp = None
559 559
560 560 ds = None
561 561
562 562 firsttime = True
563 563
564 564 #Configurations
565 565
566 566 blocksPerFile = None
567 567
568 568 blockIndex = None
569 569
570 570 dataOut = None
571 571
572 572 #Data Arrays
573 573
574 574 dataList = None
575 575
576 576 metadataList = None
577 577
578 578 arrayDim = None
579 579
580 580 tableDim = None
581 581
582 582 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
583 583
584 584 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
585 585
586 586 mode = None
587 587
588 588 nDatas = None #Number of datasets to be stored per array
589 589
590 590 nDims = None #Number Dimensions in each dataset
591 591
592 592 def __init__(self):
593 593
594 594 Operation.__init__(self)
595 595 self.isConfig = False
596 596 return
597 597
598 598
599 599 def setup(self, dataOut, **kwargs):
600 600
601 601 self.path = kwargs['path']
602 602
603 603 if kwargs.has_key('ext'):
604 604 self.ext = kwargs['ext']
605 605
606 606 if kwargs.has_key('blocksPerFile'):
607 607 self.blocksPerFile = kwargs['blocksPerFile']
608 608 else:
609 609 self.blocksPerFile = 10
610 610
611 611 self.metadataList = kwargs['metadataList']
612 612
613 613 self.dataList = kwargs['dataList']
614 614
615 615 self.dataOut = dataOut
616 616
617 617 if kwargs.has_key('mode'):
618 618 mode = kwargs['mode']
619 619
620 620 if type(mode) == int:
621 621 mode = numpy.zeros(len(self.dataList)) + mode
622 622 else:
623 623 mode = numpy.zeros(len(self.dataList))
624 624
625 625 self.mode = mode
626 626
627 627 arrayDim = numpy.zeros((len(self.dataList),5))
628 628
629 629 #Table dimensions
630 630
631 631 dtype0 = self.dtype
632 632
633 633 tableList = []
634 634
635 635 for i in range(len(self.dataList)):
636 636
637 637 dataAux = getattr(self.dataOut, self.dataList[i])
638 638
639 639 if type(dataAux)==float or type(dataAux)==int:
640 640 arrayDim[i,0] = 1
641 641 else:
642 642 arrayDim0 = dataAux.shape
643 643 arrayDim[i,0] = len(arrayDim0)
644 644 arrayDim[i,4] = mode[i]
645 645
646 646 if len(arrayDim0) == 3:
647 647 arrayDim[i,1:-1] = numpy.array(arrayDim0)
648 648 elif len(arrayDim0) == 2:
649 649 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
650 650 elif len(arrayDim0) == 1:
651 651 arrayDim[i,3] = arrayDim0
652 652 elif len(arrayDim0) == 0:
653 653 arrayDim[i,0] = 1
654 654 arrayDim[i,3] = 1
655 655
656 656 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
657 657 tableList.append(table)
658 658
659 659 self.arrayDim = arrayDim
660 660 self.tableDim = numpy.array(tableList, dtype = dtype0)
661 661 self.blockIndex = 0
662 662
663 663 return
664 664
665 665 def putMetadata(self):
666 666
667 667 fp = self.createMetadataFile()
668 668 self.writeMetadata(fp)
669 669 fp.close()
670 670 return
671 671
672 672 def createMetadataFile(self):
673 673 ext = self.ext
674 674 path = self.path
675 675 setFile = self.setFile
676 676
677 677 timeTuple = time.localtime(self.dataOut.utctime)
678 678 subfolder = ''
679 679
680 680 fullpath = os.path.join( path, subfolder )
681 681 if not( os.path.exists(fullpath) ):
682 682 os.mkdir(fullpath)
683 683 setFile = -1 #inicializo mi contador de seteo
684 684 else:
685 685 filesList = os.listdir( fullpath )
686 686 if len( filesList ) > 0:
687 687 filesList = sorted( filesList, key=str.lower )
688 688 filen = filesList[-1]
689 689 # el filename debera tener el siguiente formato
690 690 # 0 1234 567 89A BCDE (hex)
691 691 # x YYYY DDD SSS .ext
692 692 if isNumber( filen[8:11] ):
693 693 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
694 694 else:
695 695 setFile = -1
696 696 else:
697 697 setFile = -1 #inicializo mi contador de seteo
698 698
699 699 setFile += 1
700 700
701 701 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
702 702 timeTuple.tm_year,
703 703 timeTuple.tm_yday,
704 704 setFile,
705 705 ext )
706 706
707 707 filename = os.path.join( path, subfolder, file )
708 708 self.metaFile = file
709 709 #Setting HDF5 File
710 710 fp = h5py.File(filename,'w')
711 711
712 712 return fp
713 713
714 714 def writeMetadata(self, fp):
715 715
716 716 grp = fp.create_group("Metadata")
717 717 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
718 718
719 719 for i in range(len(self.metadataList)):
720 720 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
721 721 return
722 722
723 723 def setNextFile(self):
724 724
725 725 ext = self.ext
726 726 path = self.path
727 727 setFile = self.setFile
728 728 mode = self.mode
729 729
730 730 if self.fp != None:
731 731 self.fp.close()
732 732
733 733 timeTuple = time.localtime(self.dataOut.utctime)
734 734 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
735 735
736 736 fullpath = os.path.join( path, subfolder )
737 737 if not( os.path.exists(fullpath) ):
738 738 os.mkdir(fullpath)
739 739 setFile = -1 #inicializo mi contador de seteo
740 740 else:
741 741 filesList = os.listdir( fullpath )
742 742 if len( filesList ) > 0:
743 743 filesList = sorted( filesList, key=str.lower )
744 744 filen = filesList[-1]
745 745 # el filename debera tener el siguiente formato
746 746 # 0 1234 567 89A BCDE (hex)
747 747 # x YYYY DDD SSS .ext
748 748 if isNumber( filen[8:11] ):
749 749 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
750 750 else:
751 751 setFile = -1
752 752 else:
753 753 setFile = -1 #inicializo mi contador de seteo
754 754
755 755 setFile += 1
756 756
757 757 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
758 758 timeTuple.tm_year,
759 759 timeTuple.tm_yday,
760 760 setFile,
761 761 ext )
762 762
763 763 filename = os.path.join( path, subfolder, file )
764 764
765 765 #Setting HDF5 File
766 766 fp = h5py.File(filename,'w')
767 767 grp = fp.create_group("Data")
768 768 grp.attrs['metadata'] = self.metaFile
769 769
770 770 # grp.attrs['blocksPerFile'] = 0
771 771
772 772 ds = []
773 773 data = []
774 774
775 775 nDatas = numpy.zeros(len(self.dataList))
776 776 nDims = self.arrayDim[:,0]
777 777
778 778 for i in range(len(self.dataList)):
779 779
780 780 if nDims[i]==1:
781 781 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,None) , chunks = True, dtype='S20')
782 782 ds.append(ds0)
783 783 data.append([])
784 784
785 785 else:
786 786
787 787 if mode[i]==0:
788 788 strMode = "channel"
789 789 nDatas[i] = self.arrayDim[i,1]
790 790 else:
791 791 strMode = "param"
792 792 nDatas[i] = self.arrayDim[i,2]
793 793
794 794 if nDims[i]==2:
795 795 nDatas[i] = self.arrayDim[i,2]
796 796
797 797 grp0 = grp.create_group(self.dataList[i])
798 798
799 799 for j in range(int(nDatas[i])):
800 800 tableName = strMode + str(j)
801 801
802 802 if nDims[i] == 3:
803 803 ds0 = grp0.create_dataset(tableName, (1,1,1) , maxshape=(None,None,None), chunks=True)
804 804 else:
805 805 ds0 = grp0.create_dataset(tableName, (1,1) , maxshape=(None,None), chunks=True)
806 806
807 807 ds.append(ds0)
808 808 data.append([])
809 809
810 810 self.nDatas = nDatas
811 811 self.nDims = nDims
812 812
813 813 #Saving variables
814 814 print 'Writing the file: %s'%filename
815 815 self.fp = fp
816 816 self.grp = grp
817 817 self.ds = ds
818 818 self.data = data
819 819
820 820 self.setFile = setFile
821 821 self.firsttime = True
822 822 self.blockIndex = 0
823 823 return
824 824
825 825 def putData(self):
826 826 self.setBlock()
827 827 self.writeBlock()
828 828
829 829 if self.blockIndex == self.blocksPerFile:
830 830 self.setNextFile()
831 831 return
832 832
833 833 def setBlock(self):
834 834 '''
835 835 data Array configured
836 836
837 837
838 838 self.data
839 839 '''
840 840 #Creating Arrays
841 841 data = self.data
842 842 nDatas = self.nDatas
843 843 nDims = self.nDims
844 844 mode = self.mode
845 845 ind = 0
846 846
847 847 for i in range(len(self.dataList)):
848 848 dataAux = getattr(self.dataOut,self.dataList[i])
849 849
850 850 if nDims[i] == 1:
851 851 data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
852 852 if not self.firsttime:
853 853 data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
854 854 ind += 1
855 855
856 856 else:
857 857 for j in range(int(nDatas[i])):
858 858 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
859 859 data[ind] = dataAux[j,:]
860 860 else:
861 861 data[ind] = dataAux[:,j,:]
862 862
863 863 if nDims[i] == 3:
864 864 data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
865 865
866 866 if not self.firsttime:
867 867 data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
868 868
869 869 else:
870 870 data[ind] = data[ind].reshape((1,data[ind].shape[0]))
871 871
872 872 if not self.firsttime:
873 873 data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
874 874 ind += 1
875 875
876 876 self.data = data
877 877 return
878 878
879 879 def writeBlock(self):
880 880 '''
881 881 Saves the block in the HDF5 file
882 882 '''
883 883 for i in range(len(self.ds)):
884 884 self.ds[i].resize(self.data[i].shape)
885 885 self.ds[i][:] = self.data[i]
886 886
887 887 self.blockIndex += 1
888 888
889 889 self.grp.attrs.modify('nRecords', self.blockIndex)
890 890
891 891 self.firsttime = False
892 892 return
893 893
894 894 def run(self, dataOut, **kwargs):
895 895 if not(self.isConfig):
896 896 self.setup(dataOut, **kwargs)
897 897 self.isConfig = True
898 898 self.putMetadata()
899 899 self.setNextFile()
900 900
901 901 self.putData()
902 902 return
903 903
Show file before | Show file after | Hide comments
@@ -1,1348 +1,1451
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22
23 23 LOCALTIME = True
24 24
25 25 def isNumber(cad):
26 26 """
27 27 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 28
29 29 Excepciones:
30 30 Si un determinado string no puede ser convertido a numero
31 31 Input:
32 32 str, string al cual se le analiza para determinar si convertible a un numero o no
33 33
34 34 Return:
35 35 True : si el string es uno numerico
36 36 False : no es un string numerico
37 37 """
38 38 try:
39 39 float( cad )
40 40 return True
41 41 except:
42 42 return False
43 43
44 44 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 45 """
46 46 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 47
48 48 Inputs:
49 49 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 50
51 51 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 52 segundos contados desde 01/01/1970.
53 53 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55
56 56 Return:
57 57 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 58 fecha especificado, de lo contrario retorna False.
59 59
60 60 Excepciones:
61 61 Si el archivo no existe o no puede ser abierto
62 62 Si la cabecera no puede ser leida.
63 63
64 64 """
65 65 basicHeaderObj = BasicHeader(LOCALTIME)
66 66
67 67 try:
68 68 fp = open(filename,'rb')
69 69 except IOError:
70 70 traceback.print_exc()
71 71 raise IOError, "The file %s can't be opened" %(filename)
72 72
73 73 sts = basicHeaderObj.read(fp)
74 74 fp.close()
75 75
76 76 if not(sts):
77 77 print "Skipping the file %s because it has not a valid header" %(filename)
78 78 return 0
79 79
80 80 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 81 return 0
82 82
83 83 return 1
84 84
85 85 def isFileinThisTime(filename, startTime, endTime):
86 86 """
87 87 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 88
89 89 Inputs:
90 90 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 91
92 92 startTime : tiempo inicial del rango seleccionado en formato datetime.time
93 93
94 94 endTime : tiempo final del rango seleccionado en formato datetime.time
95 95
96 96 Return:
97 97 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
98 98 fecha especificado, de lo contrario retorna False.
99 99
100 100 Excepciones:
101 101 Si el archivo no existe o no puede ser abierto
102 102 Si la cabecera no puede ser leida.
103 103
104 104 """
105 105
106 106
107 107 try:
108 108 fp = open(filename,'rb')
109 109 except IOError:
110 110 traceback.print_exc()
111 111 raise IOError, "The file %s can't be opened" %(filename)
112 112
113 113 basicHeaderObj = BasicHeader(LOCALTIME)
114 114 sts = basicHeaderObj.read(fp)
115 115 fp.close()
116 116
117 117 thisDatetime = basicHeaderObj.datatime
118 118 thisTime = thisDatetime.time()
119 119
120 120 if not(sts):
121 121 print "Skipping the file %s because it has not a valid header" %(filename)
122 122 return None
123 123
124 124 if not ((startTime <= thisTime) and (endTime > thisTime)):
125 125 return None
126 126
127 127 return thisDatetime
128 128
129 129 def getFileFromSet(path, ext, set):
130 130 validFilelist = []
131 131 fileList = os.listdir(path)
132 132
133 133 # 0 1234 567 89A BCDE
134 134 # H YYYY DDD SSS .ext
135 135
136 136 for thisFile in fileList:
137 137 try:
138 138 year = int(thisFile[1:5])
139 139 doy = int(thisFile[5:8])
140 140 except:
141 141 continue
142 142
143 143 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
144 144 continue
145 145
146 146 validFilelist.append(thisFile)
147 147
148 148 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
149 149
150 150 if len(myfile)!= 0:
151 151 return myfile[0]
152 152 else:
153 153 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
154 154 print 'the filename %s does not exist'%filename
155 155 print '...going to the last file: '
156 156
157 157 if validFilelist:
158 158 validFilelist = sorted( validFilelist, key=str.lower )
159 159 return validFilelist[-1]
160 160
161 161 return None
162 162
163 163 def getlastFileFromPath(path, ext):
164 164 """
165 165 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
166 166 al final de la depuracion devuelve el ultimo file de la lista que quedo.
167 167
168 168 Input:
169 169 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
170 170 ext : extension de los files contenidos en una carpeta
171 171
172 172 Return:
173 173 El ultimo file de una determinada carpeta, no se considera el path.
174 174 """
175 175 validFilelist = []
176 176 fileList = os.listdir(path)
177 177
178 178 # 0 1234 567 89A BCDE
179 179 # H YYYY DDD SSS .ext
180 180
181 181 for thisFile in fileList:
182 182
183 183 year = thisFile[1:5]
184 184 if not isNumber(year):
185 185 continue
186 186
187 187 doy = thisFile[5:8]
188 188 if not isNumber(doy):
189 189 continue
190 190
191 191 year = int(year)
192 192 doy = int(doy)
193 193
194 194 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
195 195 continue
196 196
197 197 validFilelist.append(thisFile)
198 198
199 199 if validFilelist:
200 200 validFilelist = sorted( validFilelist, key=str.lower )
201 201 return validFilelist[-1]
202 202
203 203 return None
204 204
205 205 def checkForRealPath(path, foldercounter, year, doy, set, ext):
206 206 """
207 207 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
208 208 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
209 209 el path exacto de un determinado file.
210 210
211 211 Example :
212 212 nombre correcto del file es .../.../D2009307/P2009307367.ext
213 213
214 214 Entonces la funcion prueba con las siguientes combinaciones
215 215 .../.../y2009307367.ext
216 216 .../.../Y2009307367.ext
217 217 .../.../x2009307/y2009307367.ext
218 218 .../.../x2009307/Y2009307367.ext
219 219 .../.../X2009307/y2009307367.ext
220 220 .../.../X2009307/Y2009307367.ext
221 221 siendo para este caso, la ultima combinacion de letras, identica al file buscado
222 222
223 223 Return:
224 224 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
225 225 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
226 226 para el filename
227 227 """
228 228 fullfilename = None
229 229 find_flag = False
230 230 filename = None
231 231
232 232 prefixDirList = [None,'d','D']
233 233 if ext.lower() == ".r": #voltage
234 234 prefixFileList = ['d','D']
235 235 elif ext.lower() == ".pdata": #spectra
236 236 prefixFileList = ['p','P']
237 237 else:
238 238 return None, filename
239 239
240 240 #barrido por las combinaciones posibles
241 241 for prefixDir in prefixDirList:
242 242 thispath = path
243 243 if prefixDir != None:
244 244 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
245 245 if foldercounter == 0:
246 246 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
247 247 else:
248 248 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
249 249 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
250 250 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
251 251 fullfilename = os.path.join( thispath, filename ) #formo el path completo
252 252
253 253 if os.path.exists( fullfilename ): #verifico que exista
254 254 find_flag = True
255 255 break
256 256 if find_flag:
257 257 break
258 258
259 259 if not(find_flag):
260 260 return None, filename
261 261
262 262 return fullfilename, filename
263 263
264 def isDoyFolder(folder):
264 def isRadarFolder(folder):
265 265 try:
266 266 year = int(folder[1:5])
267 doy = int(folder[5:8])
267 268 except:
268 269 return 0
269 270
271 return 1
272
273 def isRadarFile(file):
270 274 try:
271 doy = int(folder[5:8])
275 year = int(file[1:5])
276 doy = int(file[5:8])
277 set = int(file[8:11])
272 278 except:
273 279 return 0
274 280
275 281 return 1
276 282
283 def getDateFromRadarFile(file):
284 try:
285 year = int(file[1:5])
286 doy = int(file[5:8])
287 set = int(file[8:11])
288 except:
289 return None
290
291 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
292 return thisDate
293
277 294 class JRODataIO:
278 295
279 296 c = 3E8
280 297
281 298 isConfig = False
282 299
283 300 basicHeaderObj = None
284 301
285 302 systemHeaderObj = None
286 303
287 304 radarControllerHeaderObj = None
288 305
289 306 processingHeaderObj = None
290 307
291 308 online = 0
292 309
293 310 dtype = None
294 311
295 312 pathList = []
296 313
297 314 filenameList = []
298 315
299 316 filename = None
300 317
301 318 ext = None
302 319
303 320 flagIsNewFile = 1
304 321
305 322 flagDiscontinuousBlock = 0
306 323
307 324 flagIsNewBlock = 0
308 325
309 326 fp = None
310 327
311 328 firstHeaderSize = 0
312 329
313 330 basicHeaderSize = 24
314 331
315 332 versionFile = 1103
316 333
317 334 fileSize = None
318 335
319 336 # ippSeconds = None
320 337
321 338 fileSizeByHeader = None
322 339
323 340 fileIndex = None
324 341
325 342 profileIndex = None
326 343
327 344 blockIndex = None
328 345
329 346 nTotalBlocks = None
330 347
331 348 maxTimeStep = 30
332 349
333 350 lastUTTime = None
334 351
335 352 datablock = None
336 353
337 354 dataOut = None
338 355
339 356 blocksize = None
340 357
341 358 getByBlock = False
342 359
343 360 def __init__(self):
344 361
345 362 raise ValueError, "Not implemented"
346 363
347 364 def run(self):
348 365
349 366 raise ValueError, "Not implemented"
350 367
351 368 class JRODataReader(JRODataIO):
352 369
353 370 nReadBlocks = 0
354 371
355 372 delay = 10 #number of seconds waiting a new file
356 373
357 374 nTries = 3 #quantity tries
358 375
359 376 nFiles = 3 #number of files for searching
360 377
361 378 path = None
362 379
363 380 foldercounter = 0
364 381
365 382 flagNoMoreFiles = 0
366 383
367 384 datetimeList = []
368 385
369 386 __isFirstTimeOnline = 1
370 387
371 388 __printInfo = True
372 389
373 390 profileIndex = None
374 391
375 392 nTxs = 1
376 393
377 394 txIndex = None
378 395
379 396 def __init__(self):
380 397
381 398 """
382 399
383 400 """
384 401
385 raise ValueError, "This method has not been implemented"
402 # raise NotImplementedError, "This method has not been implemented"
386 403
387 404
388 405 def createObjByDefault(self):
389 406 """
390 407
391 408 """
392 raise ValueError, "This method has not been implemented"
409 raise NotImplementedError, "This method has not been implemented"
393 410
394 411 def getBlockDimension(self):
395 412
396 raise ValueError, "No implemented"
413 raise NotImplementedError, "No implemented"
397 414
398 415 def __searchFilesOffLine(self,
399 416 path,
400 startDate,
401 endDate,
417 startDate=None,
418 endDate=None,
402 419 startTime=datetime.time(0,0,0),
403 420 endTime=datetime.time(23,59,59),
404 421 set=None,
405 422 expLabel='',
406 423 ext='.r',
407 424 walk=True):
408 425
426 self.filenameList = []
427 self.datetimeList = []
428
409 429 pathList = []
410 430
411 431 if not walk:
412 432 #pathList.append(path)
413 433 multi_path = path.split(',')
414 434 for single_path in multi_path:
415 435 pathList.append(single_path)
416 436
417 437 else:
418 438 #dirList = []
419 439 multi_path = path.split(',')
420 440 for single_path in multi_path:
421 441 dirList = []
422 442 for thisPath in os.listdir(single_path):
423 443 if not os.path.isdir(os.path.join(single_path,thisPath)):
424 444 continue
425 if not isDoyFolder(thisPath):
445 if not isRadarFolder(thisPath):
426 446 continue
427 447
428 448 dirList.append(thisPath)
429 449
430 450 if not(dirList):
431 451 return None, None
432 452
453 if startDate and endDate:
433 454 thisDate = startDate
434 455
435 456 while(thisDate <= endDate):
436 457 year = thisDate.timetuple().tm_year
437 458 doy = thisDate.timetuple().tm_yday
438 459
439 460 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
440 461 if len(matchlist) == 0:
441 462 thisDate += datetime.timedelta(1)
442 463 continue
443 464 for match in matchlist:
444 465 pathList.append(os.path.join(single_path,match,expLabel))
445 466
446 467 thisDate += datetime.timedelta(1)
468 else:
469 for thiDir in dirList:
470 pathList.append(os.path.join(single_path,thiDir,expLabel))
447 471
448 472 if pathList == []:
449 473 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
450 474 return None, None
451 475
452 476 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
453 477
454 478 filenameList = []
455 479 datetimeList = []
456 480 pathDict = {}
457 481 filenameList_to_sort = []
458 482
459 483 for i in range(len(pathList)):
460 484
461 485 thisPath = pathList[i]
462 486
463 487 fileList = glob.glob1(thisPath, "*%s" %ext)
464 488 if len(fileList) < 1:
465 489 continue
466 490 fileList.sort()
467 491 pathDict.setdefault(fileList[0])
468 492 pathDict[fileList[0]] = i
469 493 filenameList_to_sort.append(fileList[0])
470 494
471 495 filenameList_to_sort.sort()
472 496
473 497 for file in filenameList_to_sort:
474 498 thisPath = pathList[pathDict[file]]
475 499
476 500 fileList = glob.glob1(thisPath, "*%s" %ext)
477 501 fileList.sort()
478 502
479 503 for file in fileList:
480 504
481 505 filename = os.path.join(thisPath,file)
482 506 thisDatetime = isFileinThisTime(filename, startTime, endTime)
483 507
484 508 if not(thisDatetime):
485 509 continue
486 510
487 511 filenameList.append(filename)
488 512 datetimeList.append(thisDatetime)
489 513
490 514 if not(filenameList):
491 515 print "Any file was found for the time range %s - %s" %(startTime, endTime)
492 516 return None, None
493 517
494 518 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
495 519 print
496 520
497 521 for i in range(len(filenameList)):
498 522 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
499 523
500 524 self.filenameList = filenameList
501 525 self.datetimeList = datetimeList
502 526
503 527 return pathList, filenameList
504 528
505 529 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
506 530
507 531 """
508 532 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
509 533 devuelve el archivo encontrado ademas de otros datos.
510 534
511 535 Input:
512 536 path : carpeta donde estan contenidos los files que contiene data
513 537
514 538 expLabel : Nombre del subexperimento (subfolder)
515 539
516 540 ext : extension de los files
517 541
518 542 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
519 543
520 544 Return:
521 545 directory : eL directorio donde esta el file encontrado
522 546 filename : el ultimo file de una determinada carpeta
523 547 year : el anho
524 548 doy : el numero de dia del anho
525 549 set : el set del archivo
526 550
527 551
528 552 """
529 553 dirList = []
530 554
531 555 if not walk:
532 556 fullpath = path
533 557 foldercounter = 0
534 558 else:
535 559 #Filtra solo los directorios
536 560 for thisPath in os.listdir(path):
537 561 if not os.path.isdir(os.path.join(path,thisPath)):
538 562 continue
539 if not isDoyFolder(thisPath):
563 if not isRadarFolder(thisPath):
540 564 continue
541 565
542 566 dirList.append(thisPath)
543 567
544 568 if not(dirList):
545 569 return None, None, None, None, None, None
546 570
547 571 dirList = sorted( dirList, key=str.lower )
548 572
549 573 doypath = dirList[-1]
550 574 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
551 575 fullpath = os.path.join(path, doypath, expLabel)
552 576
553 577
554 578 print "%s folder was found: " %(fullpath )
555 579
556 580 if set == None:
557 581 filename = getlastFileFromPath(fullpath, ext)
558 582 else:
559 583 filename = getFileFromSet(fullpath, ext, set)
560 584
561 585 if not(filename):
562 586 return None, None, None, None, None, None
563 587
564 588 print "%s file was found" %(filename)
565 589
566 590 if not(self.__verifyFile(os.path.join(fullpath, filename))):
567 591 return None, None, None, None, None, None
568 592
569 593 year = int( filename[1:5] )
570 594 doy = int( filename[5:8] )
571 595 set = int( filename[8:11] )
572 596
573 597 return fullpath, foldercounter, filename, year, doy, set
574 598
575 599 def __setNextFileOffline(self):
576 600
577 601 idFile = self.fileIndex
578 602
579 603 while (True):
580 604 idFile += 1
581 605 if not(idFile < len(self.filenameList)):
582 606 self.flagNoMoreFiles = 1
583 607 # print "[Reading] No more Files"
584 608 return 0
585 609
586 610 filename = self.filenameList[idFile]
587 611
588 612 if not(self.__verifyFile(filename)):
589 613 continue
590 614
591 615 fileSize = os.path.getsize(filename)
592 616 fp = open(filename,'rb')
593 617 break
594 618
595 619 self.flagIsNewFile = 1
596 620 self.fileIndex = idFile
597 621 self.filename = filename
598 622 self.fileSize = fileSize
599 623 self.fp = fp
600 624
601 625 # print "[Reading] Setting the file: %s"%self.filename
602 626
603 627 return 1
604 628
605 629 def __setNextFileOnline(self):
606 630 """
607 631 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
608 632 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
609 633 siguientes.
610 634
611 635 Affected:
612 636 self.flagIsNewFile
613 637 self.filename
614 638 self.fileSize
615 639 self.fp
616 640 self.set
617 641 self.flagNoMoreFiles
618 642
619 643 Return:
620 644 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
621 645 1 : si el file fue abierto con exito y esta listo a ser leido
622 646
623 647 Excepciones:
624 648 Si un determinado file no puede ser abierto
625 649 """
626 650 nFiles = 0
627 651 fileOk_flag = False
628 652 firstTime_flag = True
629 653
630 654 self.set += 1
631 655
632 656 if self.set > 999:
633 657 self.set = 0
634 658 self.foldercounter += 1
635 659
636 660 #busca el 1er file disponible
637 661 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
638 662 if fullfilename:
639 663 if self.__verifyFile(fullfilename, False):
640 664 fileOk_flag = True
641 665
642 666 #si no encuentra un file entonces espera y vuelve a buscar
643 667 if not(fileOk_flag):
644 668 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
645 669
646 670 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
647 671 tries = self.nTries
648 672 else:
649 673 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
650 674
651 675 for nTries in range( tries ):
652 676 if firstTime_flag:
653 677 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
654 678 sleep( self.delay )
655 679 else:
656 680 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
657 681
658 682 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
659 683 if fullfilename:
660 684 if self.__verifyFile(fullfilename):
661 685 fileOk_flag = True
662 686 break
663 687
664 688 if fileOk_flag:
665 689 break
666 690
667 691 firstTime_flag = False
668 692
669 693 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
670 694 self.set += 1
671 695
672 696 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
673 697 self.set = 0
674 698 self.doy += 1
675 699 self.foldercounter = 0
676 700
677 701 if fileOk_flag:
678 702 self.fileSize = os.path.getsize( fullfilename )
679 703 self.filename = fullfilename
680 704 self.flagIsNewFile = 1
681 705 if self.fp != None: self.fp.close()
682 706 self.fp = open(fullfilename, 'rb')
683 707 self.flagNoMoreFiles = 0
684 708 # print '[Reading] Setting the file: %s' % fullfilename
685 709 else:
686 710 self.fileSize = 0
687 711 self.filename = None
688 712 self.flagIsNewFile = 0
689 713 self.fp = None
690 714 self.flagNoMoreFiles = 1
691 715 # print '[Reading] No more files to read'
692 716
693 717 return fileOk_flag
694 718
695 719 def setNextFile(self):
696 720 if self.fp != None:
697 721 self.fp.close()
698 722
699 723 if self.online:
700 724 newFile = self.__setNextFileOnline()
701 725 else:
702 726 newFile = self.__setNextFileOffline()
703 727
704 728 if not(newFile):
705 729 print '[Reading] No more files to read'
706 730 return 0
707 731
708 732 print '[Reading] Setting the file: %s' % self.filename
709 733
710 734 self.__readFirstHeader()
711 735 self.nReadBlocks = 0
712 736 return 1
713 737
714 738 def __waitNewBlock(self):
715 739 """
716 740 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
717 741
718 742 Si el modo de lectura es OffLine siempre retorn 0
719 743 """
720 744 if not self.online:
721 745 return 0
722 746
723 747 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
724 748 return 0
725 749
726 750 currentPointer = self.fp.tell()
727 751
728 752 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
729 753
730 754 for nTries in range( self.nTries ):
731 755
732 756 self.fp.close()
733 757 self.fp = open( self.filename, 'rb' )
734 758 self.fp.seek( currentPointer )
735 759
736 760 self.fileSize = os.path.getsize( self.filename )
737 761 currentSize = self.fileSize - currentPointer
738 762
739 763 if ( currentSize >= neededSize ):
740 764 self.basicHeaderObj.read(self.fp)
741 765 return 1
742 766
743 767 if self.fileSize == self.fileSizeByHeader:
744 768 # self.flagEoF = True
745 769 return 0
746 770
747 771 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
748 772 sleep( self.delay )
749 773
750 774
751 775 return 0
752 776
753 777 def waitDataBlock(self,pointer_location):
754 778
755 779 currentPointer = pointer_location
756 780
757 781 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
758 782
759 783 for nTries in range( self.nTries ):
760 784 self.fp.close()
761 785 self.fp = open( self.filename, 'rb' )
762 786 self.fp.seek( currentPointer )
763 787
764 788 self.fileSize = os.path.getsize( self.filename )
765 789 currentSize = self.fileSize - currentPointer
766 790
767 791 if ( currentSize >= neededSize ):
768 792 return 1
769 793
770 794 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
771 795 sleep( self.delay )
772 796
773 797 return 0
774 798
775 799 def __jumpToLastBlock(self):
776 800
777 801 if not(self.__isFirstTimeOnline):
778 802 return
779 803
780 804 csize = self.fileSize - self.fp.tell()
781 805 blocksize = self.processingHeaderObj.blockSize
782 806
783 807 #salta el primer bloque de datos
784 808 if csize > self.processingHeaderObj.blockSize:
785 809 self.fp.seek(self.fp.tell() + blocksize)
786 810 else:
787 811 return
788 812
789 813 csize = self.fileSize - self.fp.tell()
790 814 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
791 815 while True:
792 816
793 817 if self.fp.tell()<self.fileSize:
794 818 self.fp.seek(self.fp.tell() + neededsize)
795 819 else:
796 820 self.fp.seek(self.fp.tell() - neededsize)
797 821 break
798 822
799 823 # csize = self.fileSize - self.fp.tell()
800 824 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
801 825 # factor = int(csize/neededsize)
802 826 # if factor > 0:
803 827 # self.fp.seek(self.fp.tell() + factor*neededsize)
804 828
805 829 self.flagIsNewFile = 0
806 830 self.__isFirstTimeOnline = 0
807 831
808 832 def __setNewBlock(self):
809 833
810 834 if self.fp == None:
811 835 return 0
812 836
813 837 if self.online:
814 838 self.__jumpToLastBlock()
815 839
816 840 if self.flagIsNewFile:
817 841 return 1
818 842
819 843 self.lastUTTime = self.basicHeaderObj.utc
820 844 currentSize = self.fileSize - self.fp.tell()
821 845 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
822 846
823 847 if (currentSize >= neededSize):
824 848 self.basicHeaderObj.read(self.fp)
825 849 return 1
826 850
827 851 if self.__waitNewBlock():
828 852 return 1
829 853
830 854 if not(self.setNextFile()):
831 855 return 0
832 856
833 857 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
834 858
835 859 self.flagDiscontinuousBlock = 0
836 860
837 861 if deltaTime > self.maxTimeStep:
838 862 self.flagDiscontinuousBlock = 1
839 863
840 864 return 1
841 865
842 866 def readNextBlock(self):
843 867 if not(self.__setNewBlock()):
844 868 return 0
845 869
846 870 if not(self.readBlock()):
847 871 return 0
848 872
849 873 return 1
850 874
851 875 def __readFirstHeader(self):
852 876
853 877 self.basicHeaderObj.read(self.fp)
854 878 self.systemHeaderObj.read(self.fp)
855 879 self.radarControllerHeaderObj.read(self.fp)
856 880 self.processingHeaderObj.read(self.fp)
857 881
858 882 self.firstHeaderSize = self.basicHeaderObj.size
859 883
860 884 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
861 885 if datatype == 0:
862 886 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
863 887 elif datatype == 1:
864 888 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
865 889 elif datatype == 2:
866 890 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
867 891 elif datatype == 3:
868 892 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
869 893 elif datatype == 4:
870 894 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
871 895 elif datatype == 5:
872 896 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
873 897 else:
874 898 raise ValueError, 'Data type was not defined'
875 899
876 900 self.dtype = datatype_str
877 901 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
878 902 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
879 903 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
880 904 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
881 905 self.getBlockDimension()
882 906
883 907 def __verifyFile(self, filename, msgFlag=True):
884 908 msg = None
885 909 try:
886 910 fp = open(filename, 'rb')
887 911 currentPosition = fp.tell()
888 912 except IOError:
889 913 traceback.print_exc()
890 914 if msgFlag:
891 915 print "[Reading] The file %s can't be opened" % (filename)
892 916 return False
893 917
894 918 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
895 919
896 920 if neededSize == 0:
897 921 basicHeaderObj = BasicHeader(LOCALTIME)
898 922 systemHeaderObj = SystemHeader()
899 923 radarControllerHeaderObj = RadarControllerHeader()
900 924 processingHeaderObj = ProcessingHeader()
901 925
902 926 try:
903 927 if not( basicHeaderObj.read(fp) ): raise IOError
904 928 if not( systemHeaderObj.read(fp) ): raise IOError
905 929 if not( radarControllerHeaderObj.read(fp) ): raise IOError
906 930 if not( processingHeaderObj.read(fp) ): raise IOError
907 931 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
908 932
909 933 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
910 934
911 935 except IOError:
912 936 traceback.print_exc()
913 937 if msgFlag:
914 938 print "[Reading] The file %s is empty or it hasn't enough data" % filename
915 939
916 940 fp.close()
917 941 return False
918 942 else:
919 943 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
920 944
921 945 fp.close()
922 946 fileSize = os.path.getsize(filename)
923 947 currentSize = fileSize - currentPosition
924 948 if currentSize < neededSize:
925 949 if msgFlag and (msg != None):
926 950 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
927 951 return False
928 952
929 953 return True
930 954
955 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
956
957 dateList = []
958 pathList = []
959
960 if not walk:
961 #pathList.append(path)
962 multi_path = path.split(',')
963 for single_path in multi_path:
964
965 ok = False
966 fileList = glob.glob1(single_path, "*"+ext)
967
968 for thisFile in fileList:
969
970 if not os.path.isfile(os.path.join(single_path, thisFile)):
971 continue
972
973 if not isRadarFile(thisFile):
974 continue
975
976 ok = True
977 thisDate = getDateFromRadarFile(thisFile)
978
979 if thisDate not in dateList:
980 dateList.append(thisDate)
981
982 if ok:
983 pathList.append(single_path)
984
985 return dateList
986
987 multi_path = path.split(',')
988 for single_path in multi_path:
989
990 dirList = []
991
992 for thisPath in os.listdir(single_path):
993
994 if not os.path.isdir(os.path.join(single_path,thisPath)):
995 continue
996
997 if not isRadarFolder(thisPath):
998 continue
999
1000 dirList.append(thisPath)
1001
1002 if not dirList:
1003 return dateList
1004
1005 if startDate and endDate:
1006 thisDate = startDate
1007
1008 while(thisDate <= endDate):
1009 year = thisDate.timetuple().tm_year
1010 doy = thisDate.timetuple().tm_yday
1011
1012 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1013 if len(matchlist) == 0:
1014 thisDate += datetime.timedelta(1)
1015 continue
1016
1017 for match in matchlist:
1018 pathList.append(os.path.join(single_path,match,expLabel))
1019 dateList.append(thisDate)
1020
1021 thisDate += datetime.timedelta(1)
1022 else:
1023 for thiDir in dirList:
1024 year = int(folder[1:5])
1025 doy = int(folder[5:8])
1026 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1027
1028 pathList.append(os.path.join(single_path,thiDir,expLabel))
1029 dateList.append(thisDate)
1030
1031 return dateList
1032
1033
931 1034 def setup(self,
932 1035 path=None,
933 1036 startDate=None,
934 1037 endDate=None,
935 1038 startTime=datetime.time(0,0,0),
936 1039 endTime=datetime.time(23,59,59),
937 1040 set=None,
938 1041 expLabel = "",
939 1042 ext = None,
940 1043 online = False,
941 1044 delay = 60,
942 1045 walk = True,
943 1046 getblock = False,
944 1047 nTxs = 1):
945 1048
946 1049 if path == None:
947 1050 raise ValueError, "[Reading] The path is not valid"
948 1051
949 1052 if ext == None:
950 1053 ext = self.ext
951 1054
952 1055 if online:
953 1056 print "[Reading] Searching files in online mode..."
954 1057
955 1058 for nTries in range( self.nTries ):
956 1059 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
957 1060
958 1061 if fullpath:
959 1062 break
960 1063
961 1064 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
962 1065 sleep( self.delay )
963 1066
964 1067 if not(fullpath):
965 1068 print "[Reading] There 'isn't any valid file in %s" % path
966 1069 return None
967 1070
968 1071 self.year = year
969 1072 self.doy = doy
970 1073 self.set = set - 1
971 1074 self.path = path
972 1075 self.foldercounter = foldercounter
973 1076 last_set = None
974 1077
975 1078 else:
976 1079 print "[Reading] Searching files in offline mode ..."
977 1080 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
978 1081 startTime=startTime, endTime=endTime,
979 1082 set=set, expLabel=expLabel, ext=ext,
980 1083 walk=walk)
981 1084
982 1085 if not(pathList):
983 1086 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
984 1087 datetime.datetime.combine(startDate,startTime).ctime(),
985 1088 datetime.datetime.combine(endDate,endTime).ctime())
986 1089
987 1090 sys.exit(-1)
988 1091
989 1092
990 1093 self.fileIndex = -1
991 1094 self.pathList = pathList
992 1095 self.filenameList = filenameList
993 1096 file_name = os.path.basename(filenameList[-1])
994 1097 basename, ext = os.path.splitext(file_name)
995 1098 last_set = int(basename[-3:])
996 1099
997 1100 self.online = online
998 1101 self.delay = delay
999 1102 ext = ext.lower()
1000 1103 self.ext = ext
1001 1104 self.getByBlock = getblock
1002 1105 self.nTxs = int(nTxs)
1003 1106
1004 1107 if not(self.setNextFile()):
1005 1108 if (startDate!=None) and (endDate!=None):
1006 1109 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1007 1110 elif startDate != None:
1008 1111 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1009 1112 else:
1010 1113 print "[Reading] No files"
1011 1114
1012 1115 sys.exit(-1)
1013 1116
1014 1117 # self.updateDataHeader()
1015 1118 if last_set != None:
1016 1119 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1017 1120 return
1018 1121
1019 1122 def getBasicHeader(self):
1020 1123
1021 1124 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1022 1125
1023 1126 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1024 1127
1025 1128 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1026 1129
1027 1130 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1028 1131
1029 1132 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1030 1133
1031 1134 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1032 1135
1033 1136 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1034 1137
1035 1138 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1036 1139
1037 1140
1038 1141 def getFirstHeader(self):
1039 1142
1040 1143 raise ValueError, "This method has not been implemented"
1041 1144
1042 1145 def getData(self):
1043 1146
1044 1147 raise ValueError, "This method has not been implemented"
1045 1148
1046 1149 def hasNotDataInBuffer(self):
1047 1150
1048 1151 raise ValueError, "This method has not been implemented"
1049 1152
1050 1153 def readBlock(self):
1051 1154
1052 1155 raise ValueError, "This method has not been implemented"
1053 1156
1054 1157 def isEndProcess(self):
1055 1158
1056 1159 return self.flagNoMoreFiles
1057 1160
1058 1161 def printReadBlocks(self):
1059 1162
1060 1163 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1061 1164
1062 1165 def printTotalBlocks(self):
1063 1166
1064 1167 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1065 1168
1066 1169 def printNumberOfBlock(self):
1067 1170
1068 1171 if self.flagIsNewBlock:
1069 1172 print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1070 1173 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1071 1174
1072 1175 def printInfo(self):
1073 1176
1074 1177 if self.__printInfo == False:
1075 1178 return
1076 1179
1077 1180 self.basicHeaderObj.printInfo()
1078 1181 self.systemHeaderObj.printInfo()
1079 1182 self.radarControllerHeaderObj.printInfo()
1080 1183 self.processingHeaderObj.printInfo()
1081 1184
1082 1185 self.__printInfo = False
1083 1186
1084 1187
1085 1188 def run(self, **kwargs):
1086 1189
1087 1190 if not(self.isConfig):
1088 1191
1089 1192 # self.dataOut = dataOut
1090 1193 self.setup(**kwargs)
1091 1194 self.isConfig = True
1092 1195
1093 1196 self.getData()
1094 1197
1095 1198 class JRODataWriter(JRODataIO):
1096 1199
1097 1200 """
1098 1201 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1099 1202 de los datos siempre se realiza por bloques.
1100 1203 """
1101 1204
1102 1205 blockIndex = 0
1103 1206
1104 1207 path = None
1105 1208
1106 1209 setFile = None
1107 1210
1108 1211 profilesPerBlock = None
1109 1212
1110 1213 blocksPerFile = None
1111 1214
1112 1215 nWriteBlocks = 0
1113 1216
1114 1217 def __init__(self, dataOut=None):
1115 1218 raise ValueError, "Not implemented"
1116 1219
1117 1220
1118 1221 def hasAllDataInBuffer(self):
1119 1222 raise ValueError, "Not implemented"
1120 1223
1121 1224
1122 1225 def setBlockDimension(self):
1123 1226 raise ValueError, "Not implemented"
1124 1227
1125 1228
1126 1229 def writeBlock(self):
1127 1230 raise ValueError, "No implemented"
1128 1231
1129 1232
1130 1233 def putData(self):
1131 1234 raise ValueError, "No implemented"
1132 1235
1133 1236
1134 1237 def setBasicHeader(self):
1135 1238
1136 1239 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1137 1240 self.basicHeaderObj.version = self.versionFile
1138 1241 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1139 1242
1140 1243 utc = numpy.floor(self.dataOut.utctime)
1141 1244 milisecond = (self.dataOut.utctime - utc)* 1000.0
1142 1245
1143 1246 self.basicHeaderObj.utc = utc
1144 1247 self.basicHeaderObj.miliSecond = milisecond
1145 1248 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1146 1249 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1147 1250 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1148 1251
1149 1252 def setFirstHeader(self):
1150 1253 """
1151 1254 Obtiene una copia del First Header
1152 1255
1153 1256 Affected:
1154 1257
1155 1258 self.basicHeaderObj
1156 1259 self.systemHeaderObj
1157 1260 self.radarControllerHeaderObj
1158 1261 self.processingHeaderObj self.
1159 1262
1160 1263 Return:
1161 1264 None
1162 1265 """
1163 1266
1164 1267 raise ValueError, "No implemented"
1165 1268
1166 1269 def __writeFirstHeader(self):
1167 1270 """
1168 1271 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1169 1272
1170 1273 Affected:
1171 1274 __dataType
1172 1275
1173 1276 Return:
1174 1277 None
1175 1278 """
1176 1279
1177 1280 # CALCULAR PARAMETROS
1178 1281
1179 1282 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1180 1283 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1181 1284
1182 1285 self.basicHeaderObj.write(self.fp)
1183 1286 self.systemHeaderObj.write(self.fp)
1184 1287 self.radarControllerHeaderObj.write(self.fp)
1185 1288 self.processingHeaderObj.write(self.fp)
1186 1289
1187 1290 self.dtype = self.dataOut.dtype
1188 1291
1189 1292 def __setNewBlock(self):
1190 1293 """
1191 1294 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1192 1295
1193 1296 Return:
1194 1297 0 : si no pudo escribir nada
1195 1298 1 : Si escribio el Basic el First Header
1196 1299 """
1197 1300 if self.fp == None:
1198 1301 self.setNextFile()
1199 1302
1200 1303 if self.flagIsNewFile:
1201 1304 return 1
1202 1305
1203 1306 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1204 1307 self.basicHeaderObj.write(self.fp)
1205 1308 return 1
1206 1309
1207 1310 if not( self.setNextFile() ):
1208 1311 return 0
1209 1312
1210 1313 return 1
1211 1314
1212 1315
1213 1316 def writeNextBlock(self):
1214 1317 """
1215 1318 Selecciona el bloque siguiente de datos y los escribe en un file
1216 1319
1217 1320 Return:
1218 1321 0 : Si no hizo pudo escribir el bloque de datos
1219 1322 1 : Si no pudo escribir el bloque de datos
1220 1323 """
1221 1324 if not( self.__setNewBlock() ):
1222 1325 return 0
1223 1326
1224 1327 self.writeBlock()
1225 1328
1226 1329 return 1
1227 1330
1228 1331 def setNextFile(self):
1229 1332 """
1230 1333 Determina el siguiente file que sera escrito
1231 1334
1232 1335 Affected:
1233 1336 self.filename
1234 1337 self.subfolder
1235 1338 self.fp
1236 1339 self.setFile
1237 1340 self.flagIsNewFile
1238 1341
1239 1342 Return:
1240 1343 0 : Si el archivo no puede ser escrito
1241 1344 1 : Si el archivo esta listo para ser escrito
1242 1345 """
1243 1346 ext = self.ext
1244 1347 path = self.path
1245 1348
1246 1349 if self.fp != None:
1247 1350 self.fp.close()
1248 1351
1249 1352 timeTuple = time.localtime( self.dataOut.utctime)
1250 1353 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1251 1354
1252 1355 fullpath = os.path.join( path, subfolder )
1253 1356 if not( os.path.exists(fullpath) ):
1254 1357 os.mkdir(fullpath)
1255 1358 self.setFile = -1 #inicializo mi contador de seteo
1256 1359 else:
1257 1360 filesList = os.listdir( fullpath )
1258 1361 if len( filesList ) > 0:
1259 1362 filesList = sorted( filesList, key=str.lower )
1260 1363 filen = filesList[-1]
1261 1364 # el filename debera tener el siguiente formato
1262 1365 # 0 1234 567 89A BCDE (hex)
1263 1366 # x YYYY DDD SSS .ext
1264 1367 if isNumber( filen[8:11] ):
1265 1368 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1266 1369 else:
1267 1370 self.setFile = -1
1268 1371 else:
1269 1372 self.setFile = -1 #inicializo mi contador de seteo
1270 1373
1271 1374 setFile = self.setFile
1272 1375 setFile += 1
1273 1376
1274 1377 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1275 1378 timeTuple.tm_year,
1276 1379 timeTuple.tm_yday,
1277 1380 setFile,
1278 1381 ext )
1279 1382
1280 1383 filename = os.path.join( path, subfolder, filen )
1281 1384
1282 1385 fp = open( filename,'wb' )
1283 1386
1284 1387 self.blockIndex = 0
1285 1388
1286 1389 #guardando atributos
1287 1390 self.filename = filename
1288 1391 self.subfolder = subfolder
1289 1392 self.fp = fp
1290 1393 self.setFile = setFile
1291 1394 self.flagIsNewFile = 1
1292 1395
1293 1396 self.setFirstHeader()
1294 1397
1295 1398 print '[Writing] file: %s'%self.filename
1296 1399
1297 1400 self.__writeFirstHeader()
1298 1401
1299 1402 return 1
1300 1403
1301 1404 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1302 1405 """
1303 1406 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1304 1407
1305 1408 Inputs:
1306 1409 path : el path destino en el cual se escribiran los files a crear
1307 1410 format : formato en el cual sera salvado un file
1308 1411 set : el setebo del file
1309 1412
1310 1413 Return:
1311 1414 0 : Si no realizo un buen seteo
1312 1415 1 : Si realizo un buen seteo
1313 1416 """
1314 1417
1315 1418 if ext == None:
1316 1419 ext = self.ext
1317 1420
1318 1421 ext = ext.lower()
1319 1422
1320 1423 self.ext = ext
1321 1424
1322 1425 self.path = path
1323 1426
1324 1427 self.setFile = set - 1
1325 1428
1326 1429 self.blocksPerFile = blocksPerFile
1327 1430
1328 1431 self.profilesPerBlock = profilesPerBlock
1329 1432
1330 1433 self.dataOut = dataOut
1331 1434
1332 1435 if not(self.setNextFile()):
1333 1436 print "[Writing] There isn't a next file"
1334 1437 return 0
1335 1438
1336 1439 self.setBlockDimension()
1337 1440
1338 1441 return 1
1339 1442
1340 1443 def run(self, dataOut, **kwargs):
1341 1444
1342 1445 if not(self.isConfig):
1343 1446
1344 1447 self.setup(dataOut, **kwargs)
1345 1448 self.isConfig = True
1346 1449
1347 1450 self.putData()
1348 1451
Show file before | Show file after | Hide comments
@@ -1,849 +1,849
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import os, sys
8 8 import time, datetime
9 9 import numpy
10 10 import fnmatch
11 11 import glob
12 12
13 13 try:
14 14 from gevent import sleep
15 15 except:
16 16 from time import sleep
17 17
18 18 try:
19 19 import pyfits
20 20 except:
21 21 """
22 22 """
23 23
24 24 from xml.etree.ElementTree import ElementTree
25 25
26 from jroIO_base import isDoyFolder, isNumber
26 from jroIO_base import isRadarFolder, isNumber
27 27 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
28 28
29 29 class Fits:
30 30 name=None
31 31 format=None
32 32 array =None
33 33 data =None
34 34 thdulist=None
35 35 prihdr=None
36 36 hdu=None
37 37
38 38 def __init__(self):
39 39
40 40 pass
41 41
42 42 def setColF(self,name,format,array):
43 43 self.name=name
44 44 self.format=format
45 45 self.array=array
46 46 a1=numpy.array([self.array],dtype=numpy.float32)
47 47 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
48 48 return self.col1
49 49
50 50 # def setColP(self,name,format,data):
51 51 # self.name=name
52 52 # self.format=format
53 53 # self.data=data
54 54 # a2=numpy.array([self.data],dtype=numpy.float32)
55 55 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
56 56 # return self.col2
57 57
58 58
59 59 def writeData(self,name,format,data):
60 60 self.name=name
61 61 self.format=format
62 62 self.data=data
63 63 a2=numpy.array([self.data],dtype=numpy.float32)
64 64 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
65 65 return self.col2
66 66
67 67 def cFImage(self,idblock,year,month,day,hour,minute,second):
68 68 self.hdu= pyfits.PrimaryHDU(idblock)
69 69 self.hdu.header.set("Year",year)
70 70 self.hdu.header.set("Month",month)
71 71 self.hdu.header.set("Day",day)
72 72 self.hdu.header.set("Hour",hour)
73 73 self.hdu.header.set("Minute",minute)
74 74 self.hdu.header.set("Second",second)
75 75 return self.hdu
76 76
77 77
78 78 def Ctable(self,colList):
79 79 self.cols=pyfits.ColDefs(colList)
80 80 self.tbhdu = pyfits.new_table(self.cols)
81 81 return self.tbhdu
82 82
83 83
84 84 def CFile(self,hdu,tbhdu):
85 85 self.thdulist=pyfits.HDUList([hdu,tbhdu])
86 86
87 87 def wFile(self,filename):
88 88 if os.path.isfile(filename):
89 89 os.remove(filename)
90 90 self.thdulist.writeto(filename)
91 91
92 92
93 93 class ParameterConf:
94 94 ELEMENTNAME = 'Parameter'
95 95 def __init__(self):
96 96 self.name = ''
97 97 self.value = ''
98 98
99 99 def readXml(self, parmElement):
100 100 self.name = parmElement.get('name')
101 101 self.value = parmElement.get('value')
102 102
103 103 def getElementName(self):
104 104 return self.ELEMENTNAME
105 105
106 106 class Metadata:
107 107
108 108 def __init__(self, filename):
109 109 self.parmConfObjList = []
110 110 self.readXml(filename)
111 111
112 112 def readXml(self, filename):
113 113 self.projectElement = None
114 114 self.procUnitConfObjDict = {}
115 115 self.projectElement = ElementTree().parse(filename)
116 116 self.project = self.projectElement.tag
117 117
118 118 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
119 119
120 120 for parmElement in parmElementList:
121 121 parmConfObj = ParameterConf()
122 122 parmConfObj.readXml(parmElement)
123 123 self.parmConfObjList.append(parmConfObj)
124 124
125 125 class FitsWriter(Operation):
126 126
127 127 def __init__(self):
128 128 self.isConfig = False
129 129 self.dataBlocksPerFile = None
130 130 self.blockIndex = 0
131 131 self.flagIsNewFile = 1
132 132 self.fitsObj = None
133 133 self.optchar = 'P'
134 134 self.ext = '.fits'
135 135 self.setFile = 0
136 136
137 137 def setFitsHeader(self, dataOut, metadatafile):
138 138
139 139 header_data = pyfits.PrimaryHDU()
140 140
141 141 metadata4fits = Metadata(metadatafile)
142 142 for parameter in metadata4fits.parmConfObjList:
143 143 parm_name = parameter.name
144 144 parm_value = parameter.value
145 145
146 146 # if parm_value == 'fromdatadatetime':
147 147 # value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
148 148 # elif parm_value == 'fromdataheights':
149 149 # value = dataOut.nHeights
150 150 # elif parm_value == 'fromdatachannel':
151 151 # value = dataOut.nChannels
152 152 # elif parm_value == 'fromdatasamples':
153 153 # value = dataOut.nFFTPoints
154 154 # else:
155 155 # value = parm_value
156 156
157 157 header_data.header[parm_name] = parm_value
158 158
159 159
160 160 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
161 161 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
162 162 header_data.header['NCHANNELS'] = dataOut.nChannels
163 163 #header_data.header['HEIGHTS'] = dataOut.heightList
164 164 header_data.header['NHEIGHTS'] = dataOut.nHeights
165 165
166 166 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
167 167 header_data.header['NCOHINT'] = dataOut.nCohInt
168 168 header_data.header['NINCOHINT'] = dataOut.nIncohInt
169 169 header_data.header['TIMEZONE'] = dataOut.timeZone
170 170 header_data.header['NBLOCK'] = self.blockIndex
171 171
172 172 header_data.writeto(self.filename)
173 173
174 174 self.addExtension(dataOut.heightList,'HEIGHTLIST')
175 175
176 176
177 177 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
178 178
179 179 self.path = path
180 180 self.dataOut = dataOut
181 181 self.metadatafile = metadatafile
182 182 self.dataBlocksPerFile = dataBlocksPerFile
183 183
184 184 def open(self):
185 185 self.fitsObj = pyfits.open(self.filename, mode='update')
186 186
187 187
188 188 def addExtension(self, data, tagname):
189 189 self.open()
190 190 extension = pyfits.ImageHDU(data=data, name=tagname)
191 191 #extension.header['TAG'] = tagname
192 192 self.fitsObj.append(extension)
193 193 self.write()
194 194
195 195 def addData(self, data):
196 196 self.open()
197 197 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
198 198 extension.header['UTCTIME'] = self.dataOut.utctime
199 199 self.fitsObj.append(extension)
200 200 self.blockIndex += 1
201 201 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
202 202
203 203 self.write()
204 204
205 205 def write(self):
206 206
207 207 self.fitsObj.flush(verbose=True)
208 208 self.fitsObj.close()
209 209
210 210
211 211 def setNextFile(self):
212 212
213 213 ext = self.ext
214 214 path = self.path
215 215
216 216 timeTuple = time.localtime( self.dataOut.utctime)
217 217 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
218 218
219 219 fullpath = os.path.join( path, subfolder )
220 220 if not( os.path.exists(fullpath) ):
221 221 os.mkdir(fullpath)
222 222 self.setFile = -1 #inicializo mi contador de seteo
223 223 else:
224 224 filesList = os.listdir( fullpath )
225 225 if len( filesList ) > 0:
226 226 filesList = sorted( filesList, key=str.lower )
227 227 filen = filesList[-1]
228 228
229 229 if isNumber( filen[8:11] ):
230 230 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
231 231 else:
232 232 self.setFile = -1
233 233 else:
234 234 self.setFile = -1 #inicializo mi contador de seteo
235 235
236 236 setFile = self.setFile
237 237 setFile += 1
238 238
239 239 thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
240 240 timeTuple.tm_year,
241 241 timeTuple.tm_yday,
242 242 setFile,
243 243 ext )
244 244
245 245 filename = os.path.join( path, subfolder, thisFile )
246 246
247 247 self.blockIndex = 0
248 248 self.filename = filename
249 249 self.setFile = setFile
250 250 self.flagIsNewFile = 1
251 251
252 252 print 'Writing the file: %s'%self.filename
253 253
254 254 self.setFitsHeader(self.dataOut, self.metadatafile)
255 255
256 256 return 1
257 257
258 258 def writeBlock(self):
259 259 self.addData(self.dataOut.data_spc)
260 260 self.flagIsNewFile = 0
261 261
262 262
263 263 def __setNewBlock(self):
264 264
265 265 if self.flagIsNewFile:
266 266 return 1
267 267
268 268 if self.blockIndex < self.dataBlocksPerFile:
269 269 return 1
270 270
271 271 if not( self.setNextFile() ):
272 272 return 0
273 273
274 274 return 1
275 275
276 276 def writeNextBlock(self):
277 277 if not( self.__setNewBlock() ):
278 278 return 0
279 279 self.writeBlock()
280 280 return 1
281 281
282 282 def putData(self):
283 283 if self.flagIsNewFile:
284 284 self.setNextFile()
285 285 self.writeNextBlock()
286 286
287 287 def run(self, dataOut, **kwargs):
288 288 if not(self.isConfig):
289 289 self.setup(dataOut, **kwargs)
290 290 self.isConfig = True
291 291 self.putData()
292 292
293 293
294 294 class FitsReader(ProcessingUnit):
295 295
296 296 # __TIMEZONE = time.timezone
297 297
298 298 expName = None
299 299 datetimestr = None
300 300 utc = None
301 301 nChannels = None
302 302 nSamples = None
303 303 dataBlocksPerFile = None
304 304 comments = None
305 305 lastUTTime = None
306 306 header_dict = None
307 307 data = None
308 308 data_header_dict = None
309 309
310 310 def __init__(self):
311 311 self.isConfig = False
312 312 self.ext = '.fits'
313 313 self.setFile = 0
314 314 self.flagNoMoreFiles = 0
315 315 self.flagIsNewFile = 1
316 316 self.flagDiscontinuousBlock = None
317 317 self.fileIndex = None
318 318 self.filename = None
319 319 self.fileSize = None
320 320 self.fitsObj = None
321 321 self.timeZone = None
322 322 self.nReadBlocks = 0
323 323 self.nTotalBlocks = 0
324 324 self.dataOut = self.createObjByDefault()
325 325 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
326 326 self.blockIndex = 1
327 327
328 328 def createObjByDefault(self):
329 329
330 330 dataObj = Fits()
331 331
332 332 return dataObj
333 333
334 334 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
335 335 try:
336 336 fitsObj = pyfits.open(filename,'readonly')
337 337 except:
338 338 raise IOError, "The file %s can't be opened" %(filename)
339 339
340 340 header = fitsObj[0].header
341 341 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
342 342 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
343 343
344 344 ltc = utc
345 345 if useLocalTime:
346 346 ltc -= time.timezone
347 347 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
348 348 thisTime = thisDatetime.time()
349 349
350 350 if not ((startTime <= thisTime) and (endTime > thisTime)):
351 351 return None
352 352
353 353 return thisDatetime
354 354
355 355 def __setNextFileOnline(self):
356 356 raise ValueError, "No implemented"
357 357
358 358 def __setNextFileOffline(self):
359 359 idFile = self.fileIndex
360 360
361 361 while (True):
362 362 idFile += 1
363 363 if not(idFile < len(self.filenameList)):
364 364 self.flagNoMoreFiles = 1
365 365 print "No more Files"
366 366 return 0
367 367
368 368 filename = self.filenameList[idFile]
369 369
370 370 # if not(self.__verifyFile(filename)):
371 371 # continue
372 372
373 373 fileSize = os.path.getsize(filename)
374 374 fitsObj = pyfits.open(filename,'readonly')
375 375 break
376 376
377 377 self.flagIsNewFile = 1
378 378 self.fileIndex = idFile
379 379 self.filename = filename
380 380 self.fileSize = fileSize
381 381 self.fitsObj = fitsObj
382 382 self.blockIndex = 0
383 383 print "Setting the file: %s"%self.filename
384 384
385 385 return 1
386 386
387 387 def __setValuesFromHeader(self):
388 388
389 389 self.dataOut.header = self.header_dict
390 390 self.dataOut.expName = self.expName
391 391 self.dataOut.nChannels = self.nChannels
392 392 self.dataOut.timeZone = self.timeZone
393 393 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
394 394 self.dataOut.comments = self.comments
395 395 # self.dataOut.timeInterval = self.timeInterval
396 396 self.dataOut.channelList = self.channelList
397 397 self.dataOut.heightList = self.heightList
398 398
399 399 self.dataOut.nCohInt = self.nCohInt
400 400 self.dataOut.nIncohInt = self.nIncohInt
401 401
402 402 def readHeader(self):
403 403 headerObj = self.fitsObj[0]
404 404
405 405 self.header_dict = headerObj.header
406 406 if 'EXPNAME' in headerObj.header.keys():
407 407 self.expName = headerObj.header['EXPNAME']
408 408
409 409 if 'DATATYPE' in headerObj.header.keys():
410 410 self.dataType = headerObj.header['DATATYPE']
411 411
412 412 self.datetimestr = headerObj.header['DATETIME']
413 413 channelList = headerObj.header['CHANNELLIST']
414 414 channelList = channelList.split('[')
415 415 channelList = channelList[1].split(']')
416 416 channelList = channelList[0].split(',')
417 417 channelList = [int(ch) for ch in channelList]
418 418 self.channelList = channelList
419 419 self.nChannels = headerObj.header['NCHANNELS']
420 420 self.nHeights = headerObj.header['NHEIGHTS']
421 421 self.ippSeconds = headerObj.header['IPPSECONDS']
422 422 self.nCohInt = headerObj.header['NCOHINT']
423 423 self.nIncohInt = headerObj.header['NINCOHINT']
424 424 self.dataBlocksPerFile = headerObj.header['NBLOCK']
425 425 self.timeZone = headerObj.header['TIMEZONE']
426 426
427 427 # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
428 428
429 429 if 'COMMENT' in headerObj.header.keys():
430 430 self.comments = headerObj.header['COMMENT']
431 431
432 432 self.readHeightList()
433 433
434 434 def readHeightList(self):
435 435 self.blockIndex = self.blockIndex + 1
436 436 obj = self.fitsObj[self.blockIndex]
437 437 self.heightList = obj.data
438 438 self.blockIndex = self.blockIndex + 1
439 439
440 440 def readExtension(self):
441 441 obj = self.fitsObj[self.blockIndex]
442 442 self.heightList = obj.data
443 443 self.blockIndex = self.blockIndex + 1
444 444
445 445 def setNextFile(self):
446 446
447 447 if self.online:
448 448 newFile = self.__setNextFileOnline()
449 449 else:
450 450 newFile = self.__setNextFileOffline()
451 451
452 452 if not(newFile):
453 453 return 0
454 454
455 455 self.readHeader()
456 456 self.__setValuesFromHeader()
457 457 self.nReadBlocks = 0
458 458 # self.blockIndex = 1
459 459 return 1
460 460
461 461 def __searchFilesOffLine(self,
462 462 path,
463 463 startDate,
464 464 endDate,
465 465 startTime=datetime.time(0,0,0),
466 466 endTime=datetime.time(23,59,59),
467 467 set=None,
468 468 expLabel='',
469 469 ext='.fits',
470 470 walk=True):
471 471
472 472 pathList = []
473 473
474 474 if not walk:
475 475 pathList.append(path)
476 476
477 477 else:
478 478 dirList = []
479 479 for thisPath in os.listdir(path):
480 480 if not os.path.isdir(os.path.join(path,thisPath)):
481 481 continue
482 if not isDoyFolder(thisPath):
482 if not isRadarFolder(thisPath):
483 483 continue
484 484
485 485 dirList.append(thisPath)
486 486
487 487 if not(dirList):
488 488 return None, None
489 489
490 490 thisDate = startDate
491 491
492 492 while(thisDate <= endDate):
493 493 year = thisDate.timetuple().tm_year
494 494 doy = thisDate.timetuple().tm_yday
495 495
496 496 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
497 497 if len(matchlist) == 0:
498 498 thisDate += datetime.timedelta(1)
499 499 continue
500 500 for match in matchlist:
501 501 pathList.append(os.path.join(path,match,expLabel))
502 502
503 503 thisDate += datetime.timedelta(1)
504 504
505 505 if pathList == []:
506 506 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
507 507 return None, None
508 508
509 509 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
510 510
511 511 filenameList = []
512 512 datetimeList = []
513 513
514 514 for i in range(len(pathList)):
515 515
516 516 thisPath = pathList[i]
517 517
518 518 fileList = glob.glob1(thisPath, "*%s" %ext)
519 519 fileList.sort()
520 520
521 521 for thisFile in fileList:
522 522
523 523 filename = os.path.join(thisPath,thisFile)
524 524 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
525 525
526 526 if not(thisDatetime):
527 527 continue
528 528
529 529 filenameList.append(filename)
530 530 datetimeList.append(thisDatetime)
531 531
532 532 if not(filenameList):
533 533 print "Any file was found for the time range %s - %s" %(startTime, endTime)
534 534 return None, None
535 535
536 536 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
537 537 print
538 538
539 539 for i in range(len(filenameList)):
540 540 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
541 541
542 542 self.filenameList = filenameList
543 543 self.datetimeList = datetimeList
544 544
545 545 return pathList, filenameList
546 546
547 547 def setup(self, path=None,
548 548 startDate=None,
549 549 endDate=None,
550 550 startTime=datetime.time(0,0,0),
551 551 endTime=datetime.time(23,59,59),
552 552 set=0,
553 553 expLabel = "",
554 554 ext = None,
555 555 online = False,
556 556 delay = 60,
557 557 walk = True):
558 558
559 559 if path == None:
560 560 raise ValueError, "The path is not valid"
561 561
562 562 if ext == None:
563 563 ext = self.ext
564 564
565 565 if not(online):
566 566 print "Searching files in offline mode ..."
567 567 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
568 568 startTime=startTime, endTime=endTime,
569 569 set=set, expLabel=expLabel, ext=ext,
570 570 walk=walk)
571 571
572 572 if not(pathList):
573 573 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
574 574 datetime.datetime.combine(startDate,startTime).ctime(),
575 575 datetime.datetime.combine(endDate,endTime).ctime())
576 576
577 577 sys.exit(-1)
578 578
579 579 self.fileIndex = -1
580 580 self.pathList = pathList
581 581 self.filenameList = filenameList
582 582
583 583 self.online = online
584 584 self.delay = delay
585 585 ext = ext.lower()
586 586 self.ext = ext
587 587
588 588 if not(self.setNextFile()):
589 589 if (startDate!=None) and (endDate!=None):
590 590 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
591 591 elif startDate != None:
592 592 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
593 593 else:
594 594 print "No files"
595 595
596 596 sys.exit(-1)
597 597
598 598
599 599
600 600 def readBlock(self):
601 601 dataObj = self.fitsObj[self.blockIndex]
602 602
603 603 self.data = dataObj.data
604 604 self.data_header_dict = dataObj.header
605 605 self.utc = self.data_header_dict['UTCTIME']
606 606
607 607 self.flagIsNewFile = 0
608 608 self.blockIndex += 1
609 609 self.nTotalBlocks += 1
610 610 self.nReadBlocks += 1
611 611
612 612 return 1
613 613
614 614 def __jumpToLastBlock(self):
615 615 raise ValueError, "No implemented"
616 616
617 617 def __waitNewBlock(self):
618 618 """
619 619 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
620 620
621 621 Si el modo de lectura es OffLine siempre retorn 0
622 622 """
623 623 if not self.online:
624 624 return 0
625 625
626 626 if (self.nReadBlocks >= self.dataBlocksPerFile):
627 627 return 0
628 628
629 629 currentPointer = self.fp.tell()
630 630
631 631 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
632 632
633 633 for nTries in range( self.nTries ):
634 634
635 635 self.fp.close()
636 636 self.fp = open( self.filename, 'rb' )
637 637 self.fp.seek( currentPointer )
638 638
639 639 self.fileSize = os.path.getsize( self.filename )
640 640 currentSize = self.fileSize - currentPointer
641 641
642 642 if ( currentSize >= neededSize ):
643 643 self.__rdBasicHeader()
644 644 return 1
645 645
646 646 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
647 647 sleep( self.delay )
648 648
649 649
650 650 return 0
651 651
652 652 def __setNewBlock(self):
653 653
654 654 if self.online:
655 655 self.__jumpToLastBlock()
656 656
657 657 if self.flagIsNewFile:
658 658 return 1
659 659
660 660 self.lastUTTime = self.utc
661 661
662 662 if self.online:
663 663 if self.__waitNewBlock():
664 664 return 1
665 665
666 666 if self.nReadBlocks < self.dataBlocksPerFile:
667 667 return 1
668 668
669 669 if not(self.setNextFile()):
670 670 return 0
671 671
672 672 deltaTime = self.utc - self.lastUTTime
673 673
674 674 self.flagDiscontinuousBlock = 0
675 675
676 676 if deltaTime > self.maxTimeStep:
677 677 self.flagDiscontinuousBlock = 1
678 678
679 679 return 1
680 680
681 681
682 682 def readNextBlock(self):
683 683 if not(self.__setNewBlock()):
684 684 return 0
685 685
686 686 if not(self.readBlock()):
687 687 return 0
688 688
689 689 return 1
690 690
691 691
692 692 def getData(self):
693 693
694 694 if self.flagNoMoreFiles:
695 695 self.dataOut.flagNoData = True
696 696 print 'Process finished'
697 697 return 0
698 698
699 699 self.flagDiscontinuousBlock = 0
700 700 self.flagIsNewBlock = 0
701 701
702 702 if not(self.readNextBlock()):
703 703 return 0
704 704
705 705 if self.data == None:
706 706 self.dataOut.flagNoData = True
707 707 return 0
708 708
709 709 self.dataOut.data = self.data
710 710 self.dataOut.data_header = self.data_header_dict
711 711 self.dataOut.utctime = self.utc
712 712
713 713 # self.dataOut.header = self.header_dict
714 714 # self.dataOut.expName = self.expName
715 715 # self.dataOut.nChannels = self.nChannels
716 716 # self.dataOut.timeZone = self.timeZone
717 717 # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
718 718 # self.dataOut.comments = self.comments
719 719 # # self.dataOut.timeInterval = self.timeInterval
720 720 # self.dataOut.channelList = self.channelList
721 721 # self.dataOut.heightList = self.heightList
722 722 self.dataOut.flagNoData = False
723 723
724 724 return self.dataOut.data
725 725
726 726 def run(self, **kwargs):
727 727
728 728 if not(self.isConfig):
729 729 self.setup(**kwargs)
730 730 self.isConfig = True
731 731
732 732 self.getData()
733 733
734 734 class SpectraHeisWriter(Operation):
735 735 # set = None
736 736 setFile = None
737 737 idblock = None
738 738 doypath = None
739 739 subfolder = None
740 740
741 741 def __init__(self):
742 742 self.wrObj = Fits()
743 743 # self.dataOut = dataOut
744 744 self.nTotalBlocks=0
745 745 # self.set = None
746 746 self.setFile = None
747 747 self.idblock = 0
748 748 self.wrpath = None
749 749 self.doypath = None
750 750 self.subfolder = None
751 751 self.isConfig = False
752 752
753 753 def isNumber(str):
754 754 """
755 755 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
756 756
757 757 Excepciones:
758 758 Si un determinado string no puede ser convertido a numero
759 759 Input:
760 760 str, string al cual se le analiza para determinar si convertible a un numero o no
761 761
762 762 Return:
763 763 True : si el string es uno numerico
764 764 False : no es un string numerico
765 765 """
766 766 try:
767 767 float( str )
768 768 return True
769 769 except:
770 770 return False
771 771
772 772 def setup(self, dataOut, wrpath):
773 773
774 774 if not(os.path.exists(wrpath)):
775 775 os.mkdir(wrpath)
776 776
777 777 self.wrpath = wrpath
778 778 # self.setFile = 0
779 779 self.dataOut = dataOut
780 780
781 781 def putData(self):
782 782 name= time.localtime( self.dataOut.utctime)
783 783 ext=".fits"
784 784
785 785 if self.doypath == None:
786 786 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
787 787 self.doypath = os.path.join( self.wrpath, self.subfolder )
788 788 os.mkdir(self.doypath)
789 789
790 790 if self.setFile == None:
791 791 # self.set = self.dataOut.set
792 792 self.setFile = 0
793 793 # if self.set != self.dataOut.set:
794 794 ## self.set = self.dataOut.set
795 795 # self.setFile = 0
796 796
797 797 #make the filename
798 798 thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
799 799
800 800 filename = os.path.join(self.wrpath,self.subfolder, thisFile)
801 801
802 802 idblock = numpy.array([self.idblock],dtype="int64")
803 803 header=self.wrObj.cFImage(idblock=idblock,
804 804 year=time.gmtime(self.dataOut.utctime).tm_year,
805 805 month=time.gmtime(self.dataOut.utctime).tm_mon,
806 806 day=time.gmtime(self.dataOut.utctime).tm_mday,
807 807 hour=time.gmtime(self.dataOut.utctime).tm_hour,
808 808 minute=time.gmtime(self.dataOut.utctime).tm_min,
809 809 second=time.gmtime(self.dataOut.utctime).tm_sec)
810 810
811 811 c=3E8
812 812 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
813 813 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
814 814
815 815 colList = []
816 816
817 817 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
818 818
819 819 colList.append(colFreq)
820 820
821 821 nchannel=self.dataOut.nChannels
822 822
823 823 for i in range(nchannel):
824 824 col = self.wrObj.writeData(name="PCh"+str(i+1),
825 825 format=str(self.dataOut.nFFTPoints)+'E',
826 826 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
827 827
828 828 colList.append(col)
829 829
830 830 data=self.wrObj.Ctable(colList=colList)
831 831
832 832 self.wrObj.CFile(header,data)
833 833
834 834 self.wrObj.wFile(filename)
835 835
836 836 #update the setFile
837 837 self.setFile += 1
838 838 self.idblock += 1
839 839
840 840 return 1
841 841
842 842 def run(self, dataOut, **kwargs):
843 843
844 844 if not(self.isConfig):
845 845
846 846 self.setup(dataOut, **kwargs)
847 847 self.isConfig = True
848 848
849 849 self.putData() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,517 +1,577
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import datetime
7 7 import numpy
8 8
9 9 try:
10 10 from gevent import sleep
11 11 except:
12 12 from time import sleep
13 13
14 14 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 15 from schainpy.model.data.jrodata import Voltage
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
17 17
18 18 try:
19 19 import digital_rf_hdf5
20 20 except:
21 21 print 'You should install "digital_rf_hdf5" module if you want to read USRP data'
22 22
23 23 class USRPReader(ProcessingUnit):
24 24 '''
25 25 classdocs
26 26 '''
27 27
28 28 def __init__(self):
29 29 '''
30 30 Constructor
31 31 '''
32 32
33 33 ProcessingUnit.__init__(self)
34 34
35 35 self.dataOut = Voltage()
36 36 self.__printInfo = True
37 37 self.__flagDiscontinuousBlock = False
38 38 self.__bufferIndex = 9999999
39 39
40 40 self.__ippKm = None
41 41 self.__codeType = 0
42 42 self.__nCode = None
43 43 self.__nBaud = None
44 44 self.__code = None
45 45
46 46 def __getCurrentSecond(self):
47 47
48 48 return self.__thisUnixSample/self.__sample_rate
49 49
50 50 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
51 51
52 52 def __setFileHeader(self):
53 53 '''
54 54 In this method will be initialized every parameter of dataOut object (header, no data)
55 55 '''
56 56
57 57 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
58 58 txA=0,
59 59 txB=0,
60 60 nWindows=1,
61 61 nHeights=self.__nSamples,
62 62 firstHeight=self.__firstHeigth,
63 63 deltaHeight=self.__deltaHeigth,
64 64 codeType=self.__codeType,
65 65 nCode=self.__nCode, nBaud=self.__nBaud,
66 66 code = self.__code)
67 67
68 68 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
69 69 nProfiles=1024,
70 70 nChannels=len(self.__channelList),
71 71 adcResolution=14)
72 72
73 73 self.dataOut.type = "Voltage"
74 74
75 75 self.dataOut.data = None
76 76
77 77 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
78 78
79 79 # self.dataOut.nChannels = 0
80 80
81 81 # self.dataOut.nHeights = 0
82 82
83 83 self.dataOut.nProfiles = 1
84 84
85 85 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
86 86
87 87 self.dataOut.channelList = self.__channelList
88 88
89 89 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
90 90
91 91 # self.dataOut.channelIndexList = None
92 92
93 93 self.dataOut.flagNoData = True
94 94
95 95 #Set to TRUE if the data is discontinuous
96 96 self.dataOut.flagDiscontinuousBlock = False
97 97
98 98 self.dataOut.utctime = None
99 99
100 100 self.dataOut.timeZone = self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
101 101
102 102 self.dataOut.dstFlag = 0
103 103
104 104 self.dataOut.errorCount = 0
105 105
106 106 self.dataOut.nCohInt = 1
107 107
108 108 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
109 109
110 110 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
111 111
112 112 self.dataOut.flagShiftFFT = False
113 113
114 114 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
115 115
116 116 #Time interval between profiles
117 117 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
118 118
119 119 self.dataOut.frequency = self.__frequency
120 120
121 121 self.dataOut.realtime = self.__online
122 122
123 def findDatafiles(self, path, startDate=None, endDate=None):
124
125 try:
126 digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
127 except:
128 digitalReadObj = digital_rf_hdf5.read_hdf5(path)
129
130 channelNameList = digitalReadObj.get_channels()
131
132 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
133
134 sample_rate = metadata_dict['sample_rate'][0]
135
136 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
137
138 try:
139 timezone = this_metadata_file['timezone'].value
140 except:
141 timezone = 0
142
143 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
144
145 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
146 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
147
148 if not startDate:
149 startDate = startDatetime.date()
150
151 if not endDate:
152 endDate = endDatatime.date()
153
154 dateList = []
155
156 thisDatetime = startDatetime
157
158 while(thisDatetime<=endDatatime):
159
160 thisDate = thisDatetime.date()
161
162 if thisDate < startDate:
163 continue
164
165 if thisDate > endDate:
166 break
167
168 dateList.append(thisDate)
169 thisDatetime += datetime.timedelta(1)
170
171 return dateList
172
123 173 def setup(self, path = None,
124 174 startDate = None,
125 175 endDate = None,
126 176 startTime = datetime.time(0,0,0),
127 177 endTime = datetime.time(23,59,59),
128 178 channelList = None,
129 179 nSamples = None,
130 ippKm = None,
180 ippKm = 60,
131 181 online = False,
132 wait = 60,
133 nbuffer = 1024*4):
182 delay = 60,
183 buffer_size = None,
184 nbuffer = 1024,
185 **kwargs):
134 186 '''
135 187 In this method we should set all initial parameters.
136 188
137 189 Inputs:
138 190 path
139 191 startDate
140 192 endDate
141 193 startTime
142 194 endTime
143 195 set
144 196 expLabel
145 197 ext
146 198 online
147 wait
199 delay
148 200 '''
201
202 if not buffer_size:
203 buffer_size = nbuffer
204
149 205 try:
150 206 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
151 207 except:
152 208 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
153 209
154 210 channelNameList = self.digitalReadObj.get_channels()
155 211
156 212 if not channelNameList:
157 213 raise IOError, "[Reading] The path doesn,t have any files .. "
158 214
159 215 if not channelList:
160 216 channelList = range(len(channelNameList))
161 217
162 218 ########## Reading metadata ######################
163 219
164 220 metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
165 221
166 222 self.__sample_rate = metadata_dict['sample_rate'][0]
167 223 self.__samples_per_file = metadata_dict['samples_per_file'][0]
168 224 self.__deltaHeigth = 1e6*0.15/self.__sample_rate
169 225
170 226 this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
171 227
172 228 self.__frequency = this_metadata_file['center_frequencies'].value
173 229 try:
174 230 self.__timezone = this_metadata_file['timezone'].value
175 231 except:
176 232 self.__timezone = 0
177 233
178 234 self.__firstHeigth = 0
179 235
180 236 try:
181 237 codeType = this_metadata_file['codeType'].value
182 238 except:
183 239 codeType = 0
184 240
185 241 nCode = 0
186 242 nBaud = 0
187 243 code = None
188 244
189 245 if codeType:
190 246 nCode = this_metadata_file['nCode'].value
191 247 nBaud = this_metadata_file['nBaud'].value
192 248 code = this_metadata_file['code'].value
193 249
194 250 if not ippKm:
195 251 try:
196 252 #seconds to km
197 253 ippKm = 1e6*0.15*this_metadata_file['ipp'].value
198 254 except:
199 255 ippKm = None
200 256
201 257 ####################################################
202 258 startUTCSecond = None
203 259 endUTCSecond = None
204 260
205 261 if startDate:
206 262 startDatetime = datetime.datetime.combine(startDate, startTime)
207 263 startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
208 264
209 265 if endDate:
210 266 endDatetime = datetime.datetime.combine(endDate, endTime)
211 267 endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
212 268
213 269 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
214 270
215 271 if not startUTCSecond:
216 272 startUTCSecond = start_index/self.__sample_rate
217 273
218 274 if start_index > startUTCSecond*self.__sample_rate:
219 275 startUTCSecond = start_index/self.__sample_rate
220 276
221 277 if not endUTCSecond:
222 278 endUTCSecond = end_index/self.__sample_rate
223 279
224 280 if end_index < endUTCSecond*self.__sample_rate:
225 281 endUTCSecond = end_index/self.__sample_rate
226 282
227 283 if not nSamples:
228 284 if not ippKm:
229 285 raise ValueError, "[Reading] nSamples or ippKm should be defined"
230 286
231 287 nSamples = ippKm / (1e6*0.15/self.__sample_rate)
232 288
233 289 channelBoundList = []
234 290 channelNameListFiltered = []
235 291
236 292 for thisIndexChannel in channelList:
237 293 thisChannelName = channelNameList[thisIndexChannel]
238 294 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
239 295 channelBoundList.append((start_index, end_index))
240 296 channelNameListFiltered.append(thisChannelName)
241 297
242 298 self.profileIndex = 0
243 299
300 self.__delay = delay
244 301 self.__ippKm = ippKm
245 302 self.__codeType = codeType
246 303 self.__nCode = nCode
247 304 self.__nBaud = nBaud
248 305 self.__code = code
249 306
250 307 self.__datapath = path
251 308 self.__online = online
252 309 self.__channelList = channelList
253 310 self.__channelNameList = channelNameListFiltered
254 311 self.__channelBoundList = channelBoundList
255 312 self.__nSamples = nSamples
256 self.__samples_to_read = nbuffer*nSamples
313 self.__samples_to_read = buffer_size*nSamples
257 314 self.__nChannels = len(self.__channelList)
258 315
259 316 self.__startUTCSecond = startUTCSecond
260 317 self.__endUTCSecond = endUTCSecond
261 318
262 319 self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
263 320
264 321 if online:
265 322 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
266 323 startUTCSecond = numpy.floor(endUTCSecond)
267 324
268 325 self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
269 326
270 327 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
271 328
272 329 self.__setFileHeader()
273 330 self.isConfig = True
274 331
275 332 print "[Reading] USRP Data was found from %s to %s " %(
276 333 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
277 334 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
278 335 )
279 336
280 337 print "[Reading] Starting process from ", datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone), " to ", datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
281 338
282 339 def __reload(self):
283 340
284 341 if not self.__online:
285 342 return
286 343
287 344 # print
288 345 # print "%s not in range [%s, %s]" %(
289 346 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
290 347 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
291 348 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
292 349 # )
293 350 print "[Reading] reloading metadata ..."
294 351
352 try:
295 353 self.digitalReadObj.reload(complete_update=True)
354 except:
355 self.digitalReadObj.reload()
296 356
297 357 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
298 358
299 359 if start_index > self.__startUTCSecond*self.__sample_rate:
300 360 self.__startUTCSecond = 1.0*start_index/self.__sample_rate
301 361
302 362 if end_index > self.__endUTCSecond*self.__sample_rate:
303 363 self.__endUTCSecond = 1.0*end_index/self.__sample_rate
304 364 print
305 365 print "[Reading] New timerange found [%s, %s] " %(
306 366 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
307 367 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
308 368 )
309 369
310 370 return True
311 371
312 372 return False
313 373
314 374 def __readNextBlock(self, seconds=30, volt_scale = 218776):
315 375 '''
316 376 '''
317 377
318 378 #Set the next data
319 379 self.__flagDiscontinuousBlock = False
320 380 self.__thisUnixSample += self.__samples_to_read
321 381
322 382 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
323 383 print "[Reading] There are no more data into selected timerange"
324 384
325 385 self.__reload()
326 386
327 387 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
328 388 self.__thisUnixSample -= self.__samples_to_read
329 389 return False
330 390
331 391 indexChannel = 0
332 392
333 393 dataOk = False
334 394
335 395 for thisChannelName in self.__channelNameList:
336 396
337 397 try:
338 398 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
339 399 self.__samples_to_read,
340 400 thisChannelName)
341 401
342 402 except IOError, e:
343 403 #read next profile
344 404 self.__flagDiscontinuousBlock = True
345 405 print e
346 406 break
347 407
348 408 if result.shape[0] != self.__samples_to_read:
349 409 self.__flagDiscontinuousBlock = True
350 410 print "[Reading] %s: Too few samples were found, just %d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
351 411 result.shape[0])
352 412 break
353 413
354 414 self.__data_buffer[indexChannel,:] = result*volt_scale
355 415
356 416 indexChannel += 1
357 417
358 418 dataOk = True
359 419
360 420 self.__utctime = self.__thisUnixSample/self.__sample_rate
361 421
362 422 if not dataOk:
363 423 return False
364 424
365 425 print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
366 426 self.__samples_to_read,
367 427 self.__timeInterval)
368 428
369 429 self.__bufferIndex = 0
370 430
371 431 return True
372 432
373 433 def __isBufferEmpty(self):
374 434
375 435 if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
376 436 return False
377 437
378 438 return True
379 439
380 440 def getData(self, seconds=30, nTries=5):
381 441
382 442 '''
383 443 This method gets the data from files and put the data into the dataOut object
384 444
385 445 In addition, increase el the buffer counter in one.
386 446
387 447 Return:
388 448 data : retorna un perfil de voltages (alturas * canales) copiados desde el
389 449 buffer. Si no hay mas archivos a leer retorna None.
390 450
391 451 Affected:
392 452 self.dataOut
393 453 self.profileIndex
394 454 self.flagDiscontinuousBlock
395 455 self.flagIsNewBlock
396 456 '''
397 457
398 458 err_counter = 0
399 459 self.dataOut.flagNoData = True
400 460
401 461 if self.__isBufferEmpty():
402 462
403 463 self.__flagDiscontinuousBlock = False
404 464
405 465 while True:
406 466 if self.__readNextBlock():
407 467 break
408 468
409 469 if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
410 470 return False
411 471
412 472 if self.__flagDiscontinuousBlock:
413 473 print '[Reading] discontinuous block found ... continue with the next block'
414 474 continue
415 475
416 476 if not self.__online:
417 477 return False
418 478
419 479 err_counter += 1
420 480 if err_counter > nTries:
421 481 return False
422 482
423 483 print '[Reading] waiting %d seconds to read a new block' %seconds
424 484 sleep(seconds)
425 485
426 486 self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
427 487 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
428 488 self.dataOut.flagNoData = False
429 489 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
430 490
431 491 self.__bufferIndex += self.__nSamples
432 492 self.profileIndex += 1
433 493
434 494 return True
435 495
436 496 def printInfo(self):
437 497 '''
438 498 '''
439 499 if self.__printInfo == False:
440 500 return
441 501
442 502 # self.systemHeaderObj.printInfo()
443 503 # self.radarControllerHeaderObj.printInfo()
444 504
445 505 self.__printInfo = False
446 506
447 507 def printNumberOfBlock(self):
448 508 '''
449 509 '''
450 510
451 511 print self.profileIndex
452 512
453 513 def run(self, **kwargs):
454 514 '''
455 515 This method will be called many times so here you should put all your code
456 516 '''
457 517
458 518 if not self.isConfig:
459 519 self.setup(**kwargs)
460 520
461 self.getData()
521 self.getData(seconds=self.__delay)
462 522
463 523 return
464 524
465 525 class USRPWriter(Operation):
466 526 '''
467 527 classdocs
468 528 '''
469 529
470 530 def __init__(self):
471 531 '''
472 532 Constructor
473 533 '''
474 534 self.dataOut = None
475 535
476 536 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
477 537 '''
478 538 In this method we should set all initial parameters.
479 539
480 540 Input:
481 541 dataIn : Input data will also be outputa data
482 542
483 543 '''
484 544 self.dataOut = dataIn
485 545
486 546
487 547
488 548
489 549
490 550 self.isConfig = True
491 551
492 552 return
493 553
494 554 def run(self, dataIn, **kwargs):
495 555 '''
496 556 This method will be called many times so here you should put all your code
497 557
498 558 Inputs:
499 559
500 560 dataIn : object with the data
501 561
502 562 '''
503 563
504 564 if not self.isConfig:
505 565 self.setup(dataIn, **kwargs)
506 566
507 567
508 568 if __name__ == '__main__':
509 569
510 570 readObj = USRPReader()
511 571
512 572 while True:
513 573 readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
514 574 # readObj.printInfo()
515 575 readObj.printNumberOfBlock()
516 576
517 577 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now