##// END OF EJS Templates
schain-jc
RSS
Change multiSchain by MPProject
Juan C. Espinoza -
r1052:3fb7c359028f
parent child
Show More
Side by Side Unified
Context file:
r1052:3fb7c359028f Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1324 +1,1297
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, cpu_count
13
14 import schainpy
15 import schainpy.admin
16 from schainpy.utils.log import logToFile
17
13
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 from xml.dom import minidom
15 from xml.dom import minidom
20
16
17 import schainpy
18 import schainpy.admin
21 from schainpy.model import *
19 from schainpy.model import *
22 from time import sleep
20 from schainpy.utils import log
23
21
22 DTYPES = {
23 'Voltage': '.r',
24 'Spectra': '.pdata'
25 }
24
26
27 def MPProject(project, n=cpu_count()):
28 '''
29 Project wrapper to run schain in n processes
30 '''
25
31
26 def prettify(elem):
32 rconf = project.getReadUnitObj()
27 """Return a pretty-printed XML string for the Element.
33 op = rconf.getOperationObj('run')
28 """
34 dt1 = op.getParameterValue('startDate')
29 rough_string = tostring(elem, 'utf-8')
35 dt2 = op.getParameterValue('endDate')
30 reparsed = minidom.parseString(rough_string)
31 return reparsed.toprettyxml(indent=" ")
32
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 skip = 0
35 cursor = 0
36 nFiles = None
37 processes = []
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
40 days = (dt2 - dt1).days
36 days = (dt2 - dt1).days
41
37
42 for day in range(days+1):
38 for day in range(days+1):
43 skip = 0
39 skip = 0
44 cursor = 0
40 cursor = 0
45 q = Queue()
46 processes = []
41 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
42 dt = dt1 + datetime.timedelta(day)
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
43 dt_str = dt.strftime('%Y/%m/%d')
49 firstProcess.start()
44 reader = JRODataReader()
50 if by_day:
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
51 continue
46 startDate=dt,
52 nFiles = q.get()
47 endDate=dt,
48 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
53 if nFiles==0:
50 if nFiles == 0:
54 continue
51 continue
55 firstProcess.terminate()
52 skip = int(math.ceil(nFiles/n))
56 skip = int(math.ceil(nFiles/nProcess))
53 while nFiles > cursor*skip:
57 while True:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
55 skip=skip)
59 processes[cursor].start()
56 p = project.clone()
60 if nFiles < cursor*skip:
57 p.start()
61 break
58 processes.append(p)
62 cursor += 1
59 cursor += 1
63
60
64 def beforeExit(exctype, value, trace):
61 def beforeExit(exctype, value, trace):
65 for process in processes:
62 for process in processes:
66 process.terminate()
63 process.terminate()
67 process.join()
64 process.join()
68 print traceback.print_tb(trace)
65 print traceback.print_tb(trace)
69
66
70 sys.excepthook = beforeExit
67 sys.excepthook = beforeExit
71
68
72 for process in processes:
69 for process in processes:
73 process.join()
70 process.join()
74 process.terminate()
71 process.terminate()
75
72
76 time.sleep(3)
73 time.sleep(3)
77
74
78
79 class ParameterConf():
75 class ParameterConf():
80
76
81 id = None
77 id = None
82 name = None
78 name = None
83 value = None
79 value = None
84 format = None
80 format = None
85
81
86 __formated_value = None
82 __formated_value = None
87
83
88 ELEMENTNAME = 'Parameter'
84 ELEMENTNAME = 'Parameter'
89
85
90 def __init__(self):
86 def __init__(self):
91
87
92 self.format = 'str'
88 self.format = 'str'
93
89
94 def getElementName(self):
90 def getElementName(self):
95
91
96 return self.ELEMENTNAME
92 return self.ELEMENTNAME
97
93
98 def getValue(self):
94 def getValue(self):
99
95
100 value = self.value
96 value = self.value
101 format = self.format
97 format = self.format
102
98
103 if self.__formated_value != None:
99 if self.__formated_value != None:
104
100
105 return self.__formated_value
101 return self.__formated_value
106
102
107 if format == 'obj':
103 if format == 'obj':
108 return value
104 return value
109
105
110 if format == 'str':
106 if format == 'str':
111 self.__formated_value = str(value)
107 self.__formated_value = str(value)
112 return self.__formated_value
108 return self.__formated_value
113
109
114 if value == '':
110 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, '%s: This parameter value is empty' %self.name
116
112
117 if format == 'list':
113 if format == 'list':
118 strList = value.split(',')
114 strList = value.split(',')
119
115
120 self.__formated_value = strList
116 self.__formated_value = strList
121
117
122 return self.__formated_value
118 return self.__formated_value
123
119
124 if format == 'intlist':
120 if format == 'intlist':
125 """
121 '''
126 Example:
122 Example:
127 value = (0,1,2)
123 value = (0,1,2)
128 """
124 '''
129
125
130 new_value = ast.literal_eval(value)
126 new_value = ast.literal_eval(value)
131
127
132 if type(new_value) not in (tuple, list):
128 if type(new_value) not in (tuple, list):
133 new_value = [int(new_value)]
129 new_value = [int(new_value)]
134
130
135 self.__formated_value = new_value
131 self.__formated_value = new_value
136
132
137 return self.__formated_value
133 return self.__formated_value
138
134
139 if format == 'floatlist':
135 if format == 'floatlist':
140 """
136 '''
141 Example:
137 Example:
142 value = (0.5, 1.4, 2.7)
138 value = (0.5, 1.4, 2.7)
143 """
139 '''
144
140
145 new_value = ast.literal_eval(value)
141 new_value = ast.literal_eval(value)
146
142
147 if type(new_value) not in (tuple, list):
143 if type(new_value) not in (tuple, list):
148 new_value = [float(new_value)]
144 new_value = [float(new_value)]
149
145
150 self.__formated_value = new_value
146 self.__formated_value = new_value
151
147
152 return self.__formated_value
148 return self.__formated_value
153
149
154 if format == 'date':
150 if format == 'date':
155 strList = value.split('/')
151 strList = value.split('/')
156 intList = [int(x) for x in strList]
152 intList = [int(x) for x in strList]
157 date = datetime.date(intList[0], intList[1], intList[2])
153 date = datetime.date(intList[0], intList[1], intList[2])
158
154
159 self.__formated_value = date
155 self.__formated_value = date
160
156
161 return self.__formated_value
157 return self.__formated_value
162
158
163 if format == 'time':
159 if format == 'time':
164 strList = value.split(':')
160 strList = value.split(':')
165 intList = [int(x) for x in strList]
161 intList = [int(x) for x in strList]
166 time = datetime.time(intList[0], intList[1], intList[2])
162 time = datetime.time(intList[0], intList[1], intList[2])
167
163
168 self.__formated_value = time
164 self.__formated_value = time
169
165
170 return self.__formated_value
166 return self.__formated_value
171
167
172 if format == 'pairslist':
168 if format == 'pairslist':
173 """
169 '''
174 Example:
170 Example:
175 value = (0,1),(1,2)
171 value = (0,1),(1,2)
176 """
172 '''
177
173
178 new_value = ast.literal_eval(value)
174 new_value = ast.literal_eval(value)
179
175
180 if type(new_value) not in (tuple, list):
176 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
182
178
183 if type(new_value[0]) not in (tuple, list):
179 if type(new_value[0]) not in (tuple, list):
184 if len(new_value) != 2:
180 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
186 new_value = [new_value]
182 new_value = [new_value]
187
183
188 for thisPair in new_value:
184 for thisPair in new_value:
189 if len(thisPair) != 2:
185 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
191
187
192 self.__formated_value = new_value
188 self.__formated_value = new_value
193
189
194 return self.__formated_value
190 return self.__formated_value
195
191
196 if format == 'multilist':
192 if format == 'multilist':
197 """
193 '''
198 Example:
194 Example:
199 value = (0,1,2),(3,4,5)
195 value = (0,1,2),(3,4,5)
200 """
196 '''
201 multiList = ast.literal_eval(value)
197 multiList = ast.literal_eval(value)
202
198
203 if type(multiList[0]) == int:
199 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval('(' + value + ')')
205
201
206 self.__formated_value = multiList
202 self.__formated_value = multiList
207
203
208 return self.__formated_value
204 return self.__formated_value
209
205
210 if format == 'bool':
206 if format == 'bool':
211 value = int(value)
207 value = int(value)
212
208
213 if format == 'int':
209 if format == 'int':
214 value = float(value)
210 value = float(value)
215
211
216 format_func = eval(format)
212 format_func = eval(format)
217
213
218 self.__formated_value = format_func(value)
214 self.__formated_value = format_func(value)
219
215
220 return self.__formated_value
216 return self.__formated_value
221
217
222 def updateId(self, new_id):
218 def updateId(self, new_id):
223
219
224 self.id = str(new_id)
220 self.id = str(new_id)
225
221
226 def setup(self, id, name, value, format='str'):
222 def setup(self, id, name, value, format='str'):
227 self.id = str(id)
223 self.id = str(id)
228 self.name = name
224 self.name = name
229 if format == 'obj':
225 if format == 'obj':
230 self.value = value
226 self.value = value
231 else:
227 else:
232 self.value = str(value)
228 self.value = str(value)
233 self.format = str.lower(format)
229 self.format = str.lower(format)
234
230
235 self.getValue()
231 self.getValue()
236
232
237 return 1
233 return 1
238
234
239 def update(self, name, value, format='str'):
235 def update(self, name, value, format='str'):
240
236
241 self.name = name
237 self.name = name
242 self.value = str(value)
238 self.value = str(value)
243 self.format = format
239 self.format = format
244
240
245 def makeXml(self, opElement):
241 def makeXml(self, opElement):
246 if self.name not in ('queue',):
242 if self.name not in ('queue',):
247 parmElement = SubElement(opElement, self.ELEMENTNAME)
243 parmElement = SubElement(opElement, self.ELEMENTNAME)
248 parmElement.set('id', str(self.id))
244 parmElement.set('id', str(self.id))
249 parmElement.set('name', self.name)
245 parmElement.set('name', self.name)
250 parmElement.set('value', self.value)
246 parmElement.set('value', self.value)
251 parmElement.set('format', self.format)
247 parmElement.set('format', self.format)
252
248
253 def readXml(self, parmElement):
249 def readXml(self, parmElement):
254
250
255 self.id = parmElement.get('id')
251 self.id = parmElement.get('id')
256 self.name = parmElement.get('name')
252 self.name = parmElement.get('name')
257 self.value = parmElement.get('value')
253 self.value = parmElement.get('value')
258 self.format = str.lower(parmElement.get('format'))
254 self.format = str.lower(parmElement.get('format'))
259
255
260 #Compatible with old signal chain version
256 #Compatible with old signal chain version
261 if self.format == 'int' and self.name == 'idfigure':
257 if self.format == 'int' and self.name == 'idfigure':
262 self.name = 'id'
258 self.name = 'id'
263
259
264 def printattr(self):
260 def printattr(self):
265
261
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
267
263
268 class OperationConf():
264 class OperationConf():
269
265
270 id = None
266 id = None
271 name = None
267 name = None
272 priority = None
268 priority = None
273 type = None
269 type = None
274
270
275 parmConfObjList = []
271 parmConfObjList = []
276
272
277 ELEMENTNAME = 'Operation'
273 ELEMENTNAME = 'Operation'
278
274
279 def __init__(self):
275 def __init__(self):
280
276
281 self.id = '0'
277 self.id = '0'
282 self.name = None
278 self.name = None
283 self.priority = None
279 self.priority = None
284 self.type = 'self'
280 self.type = 'self'
285
281
286
282
287 def __getNewId(self):
283 def __getNewId(self):
288
284
289 return int(self.id)*10 + len(self.parmConfObjList) + 1
285 return int(self.id)*10 + len(self.parmConfObjList) + 1
290
286
291 def updateId(self, new_id):
287 def updateId(self, new_id):
292
288
293 self.id = str(new_id)
289 self.id = str(new_id)
294
290
295 n = 1
291 n = 1
296 for parmObj in self.parmConfObjList:
292 for parmObj in self.parmConfObjList:
297
293
298 idParm = str(int(new_id)*10 + n)
294 idParm = str(int(new_id)*10 + n)
299 parmObj.updateId(idParm)
295 parmObj.updateId(idParm)
300
296
301 n += 1
297 n += 1
302
298
303 def getElementName(self):
299 def getElementName(self):
304
300
305 return self.ELEMENTNAME
301 return self.ELEMENTNAME
306
302
307 def getParameterObjList(self):
303 def getParameterObjList(self):
308
304
309 return self.parmConfObjList
305 return self.parmConfObjList
310
306
311 def getParameterObj(self, parameterName):
307 def getParameterObj(self, parameterName):
312
308
313 for parmConfObj in self.parmConfObjList:
309 for parmConfObj in self.parmConfObjList:
314
310
315 if parmConfObj.name != parameterName:
311 if parmConfObj.name != parameterName:
316 continue
312 continue
317
313
318 return parmConfObj
314 return parmConfObj
319
315
320 return None
316 return None
321
317
322 def getParameterObjfromValue(self, parameterValue):
318 def getParameterObjfromValue(self, parameterValue):
323
319
324 for parmConfObj in self.parmConfObjList:
320 for parmConfObj in self.parmConfObjList:
325
321
326 if parmConfObj.getValue() != parameterValue:
322 if parmConfObj.getValue() != parameterValue:
327 continue
323 continue
328
324
329 return parmConfObj.getValue()
325 return parmConfObj.getValue()
330
326
331 return None
327 return None
332
328
333 def getParameterValue(self, parameterName):
329 def getParameterValue(self, parameterName):
334
330
335 parameterObj = self.getParameterObj(parameterName)
331 parameterObj = self.getParameterObj(parameterName)
336
332
337 # if not parameterObj:
333 # if not parameterObj:
338 # return None
334 # return None
339
335
340 value = parameterObj.getValue()
336 value = parameterObj.getValue()
341
337
342 return value
338 return value
343
339
344
340
345 def getKwargs(self):
341 def getKwargs(self):
346
342
347 kwargs = {}
343 kwargs = {}
348
344
349 for parmConfObj in self.parmConfObjList:
345 for parmConfObj in self.parmConfObjList:
350 if self.name == 'run' and parmConfObj.name == 'datatype':
346 if self.name == 'run' and parmConfObj.name == 'datatype':
351 continue
347 continue
352
348
353 kwargs[parmConfObj.name] = parmConfObj.getValue()
349 kwargs[parmConfObj.name] = parmConfObj.getValue()
354
350
355 return kwargs
351 return kwargs
356
352
357 def setup(self, id, name, priority, type):
353 def setup(self, id, name, priority, type):
358
354
359 self.id = str(id)
355 self.id = str(id)
360 self.name = name
356 self.name = name
361 self.type = type
357 self.type = type
362 self.priority = priority
358 self.priority = priority
363
359
364 self.parmConfObjList = []
360 self.parmConfObjList = []
365
361
366 def removeParameters(self):
362 def removeParameters(self):
367
363
368 for obj in self.parmConfObjList:
364 for obj in self.parmConfObjList:
369 del obj
365 del obj
370
366
371 self.parmConfObjList = []
367 self.parmConfObjList = []
372
368
373 def addParameter(self, name, value, format='str'):
369 def addParameter(self, name, value, format='str'):
374
370
371 if value is None:
372 return None
375 id = self.__getNewId()
373 id = self.__getNewId()
376
374
377 parmConfObj = ParameterConf()
375 parmConfObj = ParameterConf()
378 if not parmConfObj.setup(id, name, value, format):
376 if not parmConfObj.setup(id, name, value, format):
379 return None
377 return None
380
378
381 self.parmConfObjList.append(parmConfObj)
379 self.parmConfObjList.append(parmConfObj)
382
380
383 return parmConfObj
381 return parmConfObj
384
382
385 def changeParameter(self, name, value, format='str'):
383 def changeParameter(self, name, value, format='str'):
386
384
387 parmConfObj = self.getParameterObj(name)
385 parmConfObj = self.getParameterObj(name)
388 parmConfObj.update(name, value, format)
386 parmConfObj.update(name, value, format)
389
387
390 return parmConfObj
388 return parmConfObj
391
389
392 def makeXml(self, procUnitElement):
390 def makeXml(self, procUnitElement):
393
391
394 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
392 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
395 opElement.set('id', str(self.id))
393 opElement.set('id', str(self.id))
396 opElement.set('name', self.name)
394 opElement.set('name', self.name)
397 opElement.set('type', self.type)
395 opElement.set('type', self.type)
398 opElement.set('priority', str(self.priority))
396 opElement.set('priority', str(self.priority))
399
397
400 for parmConfObj in self.parmConfObjList:
398 for parmConfObj in self.parmConfObjList:
401 parmConfObj.makeXml(opElement)
399 parmConfObj.makeXml(opElement)
402
400
403 def readXml(self, opElement):
401 def readXml(self, opElement):
404
402
405 self.id = opElement.get('id')
403 self.id = opElement.get('id')
406 self.name = opElement.get('name')
404 self.name = opElement.get('name')
407 self.type = opElement.get('type')
405 self.type = opElement.get('type')
408 self.priority = opElement.get('priority')
406 self.priority = opElement.get('priority')
409
407
410 #Compatible with old signal chain version
408 #Compatible with old signal chain version
411 #Use of 'run' method instead 'init'
409 #Use of 'run' method instead 'init'
412 if self.type == 'self' and self.name == 'init':
410 if self.type == 'self' and self.name == 'init':
413 self.name = 'run'
411 self.name = 'run'
414
412
415 self.parmConfObjList = []
413 self.parmConfObjList = []
416
414
417 parmElementList = opElement.iter(ParameterConf().getElementName())
415 parmElementList = opElement.iter(ParameterConf().getElementName())
418
416
419 for parmElement in parmElementList:
417 for parmElement in parmElementList:
420 parmConfObj = ParameterConf()
418 parmConfObj = ParameterConf()
421 parmConfObj.readXml(parmElement)
419 parmConfObj.readXml(parmElement)
422
420
423 #Compatible with old signal chain version
421 #Compatible with old signal chain version
424 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
422 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
425 if self.type != 'self' and self.name == 'Plot':
423 if self.type != 'self' and self.name == 'Plot':
426 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
424 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
427 self.name = parmConfObj.value
425 self.name = parmConfObj.value
428 continue
426 continue
429
427
430 self.parmConfObjList.append(parmConfObj)
428 self.parmConfObjList.append(parmConfObj)
431
429
432 def printattr(self):
430 def printattr(self):
433
431
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
435 self.id,
433 self.id,
436 self.name,
434 self.name,
437 self.type,
435 self.type,
438 self.priority)
436 self.priority)
439
437
440 for parmConfObj in self.parmConfObjList:
438 for parmConfObj in self.parmConfObjList:
441 parmConfObj.printattr()
439 parmConfObj.printattr()
442
440
443 def createObject(self, plotter_queue=None):
441 def createObject(self, plotter_queue=None):
444
442
445
443
446 if self.type == 'self':
444 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
445 raise ValueError, 'This operation type cannot be created'
448
446
449 if self.type == 'plotter':
447 if self.type == 'plotter':
450 #Plotter(plotter_name)
451 if not plotter_queue:
448 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453
450
454 opObj = Plotter(self.name, plotter_queue)
451 opObj = Plotter(self.name, plotter_queue)
455
452
456 if self.type == 'external' or self.type == 'other':
453 if self.type == 'external' or self.type == 'other':
457
454
458 className = eval(self.name)
455 className = eval(self.name)
459 kwargs = self.getKwargs()
456 kwargs = self.getKwargs()
460
457
461 opObj = className(**kwargs)
458 opObj = className(**kwargs)
462
459
463 return opObj
460 return opObj
464
461
465
462
466 class ProcUnitConf():
463 class ProcUnitConf():
467
464
468 id = None
465 id = None
469 name = None
466 name = None
470 datatype = None
467 datatype = None
471 inputId = None
468 inputId = None
472 parentId = None
469 parentId = None
473
470
474 opConfObjList = []
471 opConfObjList = []
475
472
476 procUnitObj = None
473 procUnitObj = None
477 opObjList = []
474 opObjList = []
478
475
479 ELEMENTNAME = 'ProcUnit'
476 ELEMENTNAME = 'ProcUnit'
480
477
481 def __init__(self):
478 def __init__(self):
482
479
483 self.id = None
480 self.id = None
484 self.datatype = None
481 self.datatype = None
485 self.name = None
482 self.name = None
486 self.inputId = None
483 self.inputId = None
487
484
488 self.opConfObjList = []
485 self.opConfObjList = []
489
486
490 self.procUnitObj = None
487 self.procUnitObj = None
491 self.opObjDict = {}
488 self.opObjDict = {}
492
489
493 def __getPriority(self):
490 def __getPriority(self):
494
491
495 return len(self.opConfObjList)+1
492 return len(self.opConfObjList)+1
496
493
497 def __getNewId(self):
494 def __getNewId(self):
498
495
499 return int(self.id)*10 + len(self.opConfObjList) + 1
496 return int(self.id)*10 + len(self.opConfObjList) + 1
500
497
501 def getElementName(self):
498 def getElementName(self):
502
499
503 return self.ELEMENTNAME
500 return self.ELEMENTNAME
504
501
505 def getId(self):
502 def getId(self):
506
503
507 return self.id
504 return self.id
508
505
509 def updateId(self, new_id, parentId=parentId):
506 def updateId(self, new_id, parentId=parentId):
510
507
511
508
512 new_id = int(parentId)*10 + (int(self.id) % 10)
509 new_id = int(parentId)*10 + (int(self.id) % 10)
513 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
510 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
514
511
515 #If this proc unit has not inputs
512 #If this proc unit has not inputs
516 if self.inputId == '0':
513 if self.inputId == '0':
517 new_inputId = 0
514 new_inputId = 0
518
515
519 n = 1
516 n = 1
520 for opConfObj in self.opConfObjList:
517 for opConfObj in self.opConfObjList:
521
518
522 idOp = str(int(new_id)*10 + n)
519 idOp = str(int(new_id)*10 + n)
523 opConfObj.updateId(idOp)
520 opConfObj.updateId(idOp)
524
521
525 n += 1
522 n += 1
526
523
527 self.parentId = str(parentId)
524 self.parentId = str(parentId)
528 self.id = str(new_id)
525 self.id = str(new_id)
529 self.inputId = str(new_inputId)
526 self.inputId = str(new_inputId)
530
527
531
528
532 def getInputId(self):
529 def getInputId(self):
533
530
534 return self.inputId
531 return self.inputId
535
532
536 def getOperationObjList(self):
533 def getOperationObjList(self):
537
534
538 return self.opConfObjList
535 return self.opConfObjList
539
536
540 def getOperationObj(self, name=None):
537 def getOperationObj(self, name=None):
541
538
542 for opConfObj in self.opConfObjList:
539 for opConfObj in self.opConfObjList:
543
540
544 if opConfObj.name != name:
541 if opConfObj.name != name:
545 continue
542 continue
546
543
547 return opConfObj
544 return opConfObj
548
545
549 return None
546 return None
550
547
551 def getOpObjfromParamValue(self, value=None):
548 def getOpObjfromParamValue(self, value=None):
552
549
553 for opConfObj in self.opConfObjList:
550 for opConfObj in self.opConfObjList:
554 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
551 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
555 continue
552 continue
556 return opConfObj
553 return opConfObj
557 return None
554 return None
558
555
559 def getProcUnitObj(self):
556 def getProcUnitObj(self):
560
557
561 return self.procUnitObj
558 return self.procUnitObj
562
559
563 def setup(self, id, name, datatype, inputId, parentId=None):
560 def setup(self, id, name, datatype, inputId, parentId=None):
564
561
565 #Compatible with old signal chain version
562 #Compatible with old signal chain version
566 if datatype==None and name==None:
563 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
564 raise ValueError, 'datatype or name should be defined'
568
565
569 if name==None:
566 if name==None:
570 if 'Proc' in datatype:
567 if 'Proc' in datatype:
571 name = datatype
568 name = datatype
572 else:
569 else:
573 name = '%sProc' %(datatype)
570 name = '%sProc' %(datatype)
574
571
575 if datatype==None:
572 if datatype==None:
576 datatype = name.replace('Proc','')
573 datatype = name.replace('Proc','')
577
574
578 self.id = str(id)
575 self.id = str(id)
579 self.name = name
576 self.name = name
580 self.datatype = datatype
577 self.datatype = datatype
581 self.inputId = inputId
578 self.inputId = inputId
582 self.parentId = parentId
579 self.parentId = parentId
583
580
584 self.opConfObjList = []
581 self.opConfObjList = []
585
582
586 self.addOperation(name='run', optype='self')
583 self.addOperation(name='run', optype='self')
587
584
588 def removeOperations(self):
585 def removeOperations(self):
589
586
590 for obj in self.opConfObjList:
587 for obj in self.opConfObjList:
591 del obj
588 del obj
592
589
593 self.opConfObjList = []
590 self.opConfObjList = []
594 self.addOperation(name='run')
591 self.addOperation(name='run')
595
592
596 def addParameter(self, **kwargs):
593 def addParameter(self, **kwargs):
597 '''
594 '''
598 Add parameters to "run" operation
595 Add parameters to 'run' operation
599 '''
596 '''
600 opObj = self.opConfObjList[0]
597 opObj = self.opConfObjList[0]
601
598
602 opObj.addParameter(**kwargs)
599 opObj.addParameter(**kwargs)
603
600
604 return opObj
601 return opObj
605
602
606 def addOperation(self, name, optype='self'):
603 def addOperation(self, name, optype='self'):
607
604
608 id = self.__getNewId()
605 id = self.__getNewId()
609 priority = self.__getPriority()
606 priority = self.__getPriority()
610
607
611 opConfObj = OperationConf()
608 opConfObj = OperationConf()
612 opConfObj.setup(id, name=name, priority=priority, type=optype)
609 opConfObj.setup(id, name=name, priority=priority, type=optype)
613
610
614 self.opConfObjList.append(opConfObj)
611 self.opConfObjList.append(opConfObj)
615
612
616 return opConfObj
613 return opConfObj
617
614
618 def makeXml(self, projectElement):
615 def makeXml(self, projectElement):
619
616
620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
617 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
621 procUnitElement.set('id', str(self.id))
618 procUnitElement.set('id', str(self.id))
622 procUnitElement.set('name', self.name)
619 procUnitElement.set('name', self.name)
623 procUnitElement.set('datatype', self.datatype)
620 procUnitElement.set('datatype', self.datatype)
624 procUnitElement.set('inputId', str(self.inputId))
621 procUnitElement.set('inputId', str(self.inputId))
625
622
626 for opConfObj in self.opConfObjList:
623 for opConfObj in self.opConfObjList:
627 opConfObj.makeXml(procUnitElement)
624 opConfObj.makeXml(procUnitElement)
628
625
629 def readXml(self, upElement):
626 def readXml(self, upElement):
630
627
631 self.id = upElement.get('id')
628 self.id = upElement.get('id')
632 self.name = upElement.get('name')
629 self.name = upElement.get('name')
633 self.datatype = upElement.get('datatype')
630 self.datatype = upElement.get('datatype')
634 self.inputId = upElement.get('inputId')
631 self.inputId = upElement.get('inputId')
635
632
636 if self.ELEMENTNAME == "ReadUnit":
633 if self.ELEMENTNAME == 'ReadUnit':
637 self.datatype = self.datatype.replace("Reader", "")
634 self.datatype = self.datatype.replace('Reader', '')
638
635
639 if self.ELEMENTNAME == "ProcUnit":
636 if self.ELEMENTNAME == 'ProcUnit':
640 self.datatype = self.datatype.replace("Proc", "")
637 self.datatype = self.datatype.replace('Proc', '')
641
638
642 if self.inputId == 'None':
639 if self.inputId == 'None':
643 self.inputId = '0'
640 self.inputId = '0'
644
641
645 self.opConfObjList = []
642 self.opConfObjList = []
646
643
647 opElementList = upElement.iter(OperationConf().getElementName())
644 opElementList = upElement.iter(OperationConf().getElementName())
648
645
649 for opElement in opElementList:
646 for opElement in opElementList:
650 opConfObj = OperationConf()
647 opConfObj = OperationConf()
651 opConfObj.readXml(opElement)
648 opConfObj.readXml(opElement)
652 self.opConfObjList.append(opConfObj)
649 self.opConfObjList.append(opConfObj)
653
650
654 def printattr(self):
651 def printattr(self):
655
652
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
657 self.id,
654 self.id,
658 self.name,
655 self.name,
659 self.datatype,
656 self.datatype,
660 self.inputId)
657 self.inputId)
661
658
662 for opConfObj in self.opConfObjList:
659 for opConfObj in self.opConfObjList:
663 opConfObj.printattr()
660 opConfObj.printattr()
664
661
665
662
666 def getKwargs(self):
663 def getKwargs(self):
667
664
668 opObj = self.opConfObjList[0]
665 opObj = self.opConfObjList[0]
669 kwargs = opObj.getKwargs()
666 kwargs = opObj.getKwargs()
670
667
671 return kwargs
668 return kwargs
672
669
673 def createObjects(self, plotter_queue=None):
670 def createObjects(self, plotter_queue=None):
674
671
675 className = eval(self.name)
672 className = eval(self.name)
676 kwargs = self.getKwargs()
673 kwargs = self.getKwargs()
677 procUnitObj = className(**kwargs)
674 procUnitObj = className(**kwargs)
678
675
679 for opConfObj in self.opConfObjList:
676 for opConfObj in self.opConfObjList:
680
677
681 if opConfObj.type=='self' and self.name=='run':
678 if opConfObj.type=='self' and self.name=='run':
682 continue
679 continue
683 elif opConfObj.type=='self':
680 elif opConfObj.type=='self':
684 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
681 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
685 continue
682 continue
686
683
687 opObj = opConfObj.createObject(plotter_queue)
684 opObj = opConfObj.createObject(plotter_queue)
688
685
689 self.opObjDict[opConfObj.id] = opObj
686 self.opObjDict[opConfObj.id] = opObj
690
687
691 procUnitObj.addOperation(opObj, opConfObj.id)
688 procUnitObj.addOperation(opObj, opConfObj.id)
692
689
693 self.procUnitObj = procUnitObj
690 self.procUnitObj = procUnitObj
694
691
695 return procUnitObj
692 return procUnitObj
696
693
697 def run(self):
694 def run(self):
698
695
699 is_ok = False
696 is_ok = False
700
697
701 for opConfObj in self.opConfObjList:
698 for opConfObj in self.opConfObjList:
702
699
703 kwargs = {}
700 kwargs = {}
704 for parmConfObj in opConfObj.getParameterObjList():
701 for parmConfObj in opConfObj.getParameterObjList():
705 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
702 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
706 continue
703 continue
707
704
708 kwargs[parmConfObj.name] = parmConfObj.getValue()
705 kwargs[parmConfObj.name] = parmConfObj.getValue()
709
706
710 #ini = time.time()
711
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 sts = self.procUnitObj.call(opType = opConfObj.type,
707 sts = self.procUnitObj.call(opType = opConfObj.type,
714 opName = opConfObj.name,
708 opName = opConfObj.name,
715 opId = opConfObj.id)
709 opId = opConfObj.id)
716
710
717 # total_time = time.time() - ini
718 #
719 # if total_time > 0.002:
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
721
722 is_ok = is_ok or sts
711 is_ok = is_ok or sts
723
712
724 return is_ok
713 return is_ok
725
714
726 def close(self):
715 def close(self):
727
716
728 for opConfObj in self.opConfObjList:
717 for opConfObj in self.opConfObjList:
729 if opConfObj.type == 'self':
718 if opConfObj.type == 'self':
730 continue
719 continue
731
720
732 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
721 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
733 opObj.close()
722 opObj.close()
734
723
735 self.procUnitObj.close()
724 self.procUnitObj.close()
736
725
737 return
726 return
738
727
739 class ReadUnitConf(ProcUnitConf):
728 class ReadUnitConf(ProcUnitConf):
740
729
741 path = None
730 path = None
742 startDate = None
731 startDate = None
743 endDate = None
732 endDate = None
744 startTime = None
733 startTime = None
745 endTime = None
734 endTime = None
746
735
747 ELEMENTNAME = 'ReadUnit'
736 ELEMENTNAME = 'ReadUnit'
748
737
749 def __init__(self):
738 def __init__(self):
750
739
751 self.id = None
740 self.id = None
752 self.datatype = None
741 self.datatype = None
753 self.name = None
742 self.name = None
754 self.inputId = None
743 self.inputId = None
755
744
756 self.parentId = None
745 self.parentId = None
757
746
758 self.opConfObjList = []
747 self.opConfObjList = []
759 self.opObjList = []
748 self.opObjList = []
760
749
761 def getElementName(self):
750 def getElementName(self):
762
751
763 return self.ELEMENTNAME
752 return self.ELEMENTNAME
764
753
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
767 #Compatible with old signal chain version
757 #Compatible with old signal chain version
768 if datatype==None and name==None:
758 if datatype==None and name==None:
769 raise ValueError, "datatype or name should be defined"
759 raise ValueError, 'datatype or name should be defined'
770
760
771 if name==None:
761 if name==None:
772 if 'Reader' in datatype:
762 if 'Reader' in datatype:
773 name = datatype
763 name = datatype
774 else:
764 else:
775 name = '%sReader' %(datatype)
765 name = '%sReader' %(datatype)
776 if datatype==None:
766 if datatype==None:
777 datatype = name.replace('Reader','')
767 datatype = name.replace('Reader','')
778
768
779 self.id = id
769 self.id = id
780 self.name = name
770 self.name = name
781 self.datatype = datatype
771 self.datatype = datatype
782 if path != '':
772 if path != '':
783 self.path = os.path.abspath(path)
773 self.path = os.path.abspath(path)
784 self.startDate = startDate
774 self.startDate = startDate
785 self.endDate = endDate
775 self.endDate = endDate
786 self.startTime = startTime
776 self.startTime = startTime
787 self.endTime = endTime
777 self.endTime = endTime
788
789 self.inputId = '0'
778 self.inputId = '0'
790 self.parentId = parentId
779 self.parentId = parentId
791 self.queue = queue
792 self.server = server
780 self.server = server
793 self.addRunOperation(**kwargs)
781 self.addRunOperation(**kwargs)
794
782
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
783 def update(self, **kwargs):
796
797 #Compatible with old signal chain version
798 if datatype==None and name==None:
799 raise ValueError, "datatype or name should be defined"
800
784
801 if name==None:
785 if 'datatype' in kwargs:
786 datatype = kwargs.pop('datatype')
802 if 'Reader' in datatype:
787 if 'Reader' in datatype:
803 name = datatype
788 self.name = datatype
804 else:
789 else:
805 name = '%sReader' %(datatype)
790 self.name = '%sReader' %(datatype)
791 self.datatype = self.name.replace('Reader', '')
806
792
807 if datatype==None:
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
808 datatype = name.replace('Reader','')
809
794
810 self.datatype = datatype
795 for attr in attrs:
811 self.name = name
796 if attr in kwargs:
812 self.path = path
797 setattr(self, attr, kwargs.pop(attr))
813 self.startDate = startDate
814 self.endDate = endDate
815 self.startTime = startTime
816 self.endTime = endTime
817
798
818 self.inputId = '0'
799 self.inputId = '0'
819 self.parentId = parentId
820
821 self.updateRunOperation(**kwargs)
800 self.updateRunOperation(**kwargs)
822
801
823 def removeOperations(self):
802 def removeOperations(self):
824
803
825 for obj in self.opConfObjList:
804 for obj in self.opConfObjList:
826 del obj
805 del obj
827
806
828 self.opConfObjList = []
807 self.opConfObjList = []
829
808
830 def addRunOperation(self, **kwargs):
809 def addRunOperation(self, **kwargs):
831
810
832 opObj = self.addOperation(name = 'run', optype = 'self')
811 opObj = self.addOperation(name = 'run', optype = 'self')
833
812
834 if self.server is None:
813 if self.server is None:
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
820
842 for key, value in kwargs.items():
821 for key, value in kwargs.items():
843 opObj.addParameter(name=key, value=value, format=type(value).__name__)
822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 else:
823 else:
845 opObj.addParameter(name='server' , value=self.server, format='str')
824 opObj.addParameter(name='server' , value=self.server, format='str')
846
825
847
826
848 return opObj
827 return opObj
849
828
850 def updateRunOperation(self, **kwargs):
829 def updateRunOperation(self, **kwargs):
851
830
852 opObj = self.getOperationObj(name = 'run')
831 opObj = self.getOperationObj(name='run')
853 opObj.removeParameters()
832 opObj.removeParameters()
854
833
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
861
840
862 for key, value in kwargs.items():
841 for key, value in kwargs.items():
863 opObj.addParameter(name=key, value=value, format=type(value).__name__)
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
864
843
865 return opObj
844 return opObj
866
845
867 # def makeXml(self, projectElement):
868 #
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
870 # procUnitElement.set('id', str(self.id))
871 # procUnitElement.set('name', self.name)
872 # procUnitElement.set('datatype', self.datatype)
873 # procUnitElement.set('inputId', str(self.inputId))
874 #
875 # for opConfObj in self.opConfObjList:
876 # opConfObj.makeXml(procUnitElement)
877
878 def readXml(self, upElement):
846 def readXml(self, upElement):
879
847
880 self.id = upElement.get('id')
848 self.id = upElement.get('id')
881 self.name = upElement.get('name')
849 self.name = upElement.get('name')
882 self.datatype = upElement.get('datatype')
850 self.datatype = upElement.get('datatype')
883 self.inputId = upElement.get('inputId')
851 self.inputId = upElement.get('inputId')
884
852
885 if self.ELEMENTNAME == "ReadUnit":
853 if self.ELEMENTNAME == 'ReadUnit':
886 self.datatype = self.datatype.replace("Reader", "")
854 self.datatype = self.datatype.replace('Reader', '')
887
855
888 if self.inputId == 'None':
856 if self.inputId == 'None':
889 self.inputId = '0'
857 self.inputId = '0'
890
858
891 self.opConfObjList = []
859 self.opConfObjList = []
892
860
893 opElementList = upElement.iter(OperationConf().getElementName())
861 opElementList = upElement.iter(OperationConf().getElementName())
894
862
895 for opElement in opElementList:
863 for opElement in opElementList:
896 opConfObj = OperationConf()
864 opConfObj = OperationConf()
897 opConfObj.readXml(opElement)
865 opConfObj.readXml(opElement)
898 self.opConfObjList.append(opConfObj)
866 self.opConfObjList.append(opConfObj)
899
867
900 if opConfObj.name == 'run':
868 if opConfObj.name == 'run':
901 self.path = opConfObj.getParameterValue('path')
869 self.path = opConfObj.getParameterValue('path')
902 self.startDate = opConfObj.getParameterValue('startDate')
870 self.startDate = opConfObj.getParameterValue('startDate')
903 self.endDate = opConfObj.getParameterValue('endDate')
871 self.endDate = opConfObj.getParameterValue('endDate')
904 self.startTime = opConfObj.getParameterValue('startTime')
872 self.startTime = opConfObj.getParameterValue('startTime')
905 self.endTime = opConfObj.getParameterValue('endTime')
873 self.endTime = opConfObj.getParameterValue('endTime')
906
874
907 class Project(Process):
875 class Project(Process):
876
908 id = None
877 id = None
909 name = None
878 # name = None
910 description = None
879 description = None
911 filename = None
880 filename = None
912
881
913 procUnitConfObjDict = None
882 procUnitConfObjDict = None
914
883
915 ELEMENTNAME = 'Project'
884 ELEMENTNAME = 'Project'
916
885
917 plotterQueue = None
886 plotterQueue = None
918
887
919 def __init__(self, plotter_queue=None, logfile=None):
888 def __init__(self, plotter_queue=None):
889
920 Process.__init__(self)
890 Process.__init__(self)
921 self.id = None
891 self.id = None
922 self.name = None
892 # self.name = None
923 self.description = None
893 self.description = None
924 if logfile is not None:
894
925 logToFile(logfile)
926 self.plotterQueue = plotter_queue
895 self.plotterQueue = plotter_queue
927
896
928 self.procUnitConfObjDict = {}
897 self.procUnitConfObjDict = {}
929
898
930 def __getNewId(self):
899 def __getNewId(self):
931
900
932 idList = self.procUnitConfObjDict.keys()
901 idList = self.procUnitConfObjDict.keys()
933
902
934 id = int(self.id)*10
903 id = int(self.id)*10
935
904
936 while True:
905 while True:
937 id += 1
906 id += 1
938
907
939 if str(id) in idList:
908 if str(id) in idList:
940 continue
909 continue
941
910
942 break
911 break
943
912
944 return str(id)
913 return str(id)
945
914
946 def getElementName(self):
915 def getElementName(self):
947
916
948 return self.ELEMENTNAME
917 return self.ELEMENTNAME
949
918
950 def getId(self):
919 def getId(self):
951
920
952 return self.id
921 return self.id
953
922
954 def updateId(self, new_id):
923 def updateId(self, new_id):
955
924
956 self.id = str(new_id)
925 self.id = str(new_id)
957
926
958 keyList = self.procUnitConfObjDict.keys()
927 keyList = self.procUnitConfObjDict.keys()
959 keyList.sort()
928 keyList.sort()
960
929
961 n = 1
930 n = 1
962 newProcUnitConfObjDict = {}
931 newProcUnitConfObjDict = {}
963
932
964 for procKey in keyList:
933 for procKey in keyList:
965
934
966 procUnitConfObj = self.procUnitConfObjDict[procKey]
935 procUnitConfObj = self.procUnitConfObjDict[procKey]
967 idProcUnit = str(int(self.id)*10 + n)
936 idProcUnit = str(int(self.id)*10 + n)
968 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
937 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
969
938
970 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
939 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
971 n += 1
940 n += 1
972
941
973 self.procUnitConfObjDict = newProcUnitConfObjDict
942 self.procUnitConfObjDict = newProcUnitConfObjDict
974
943
975 def setup(self, id, name, description):
944 def setup(self, id, name='', description=''):
976
945
946 print
947 print '*'*60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
950 print
977 self.id = str(id)
951 self.id = str(id)
978 self.name = name
979 self.description = description
952 self.description = description
980
953
981 def update(self, name, description):
954 def update(self, name, description):
982
955
983 self.name = name
984 self.description = description
956 self.description = description
985
957
958 def clone(self):
959
960 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
963
986 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
987 if id is None:
966 if id is None:
988 idReadUnit = self.__getNewId()
967 idReadUnit = self.__getNewId()
989 else:
968 else:
990 idReadUnit = str(id)
969 idReadUnit = str(id)
991
970
992 readUnitConfObj = ReadUnitConf()
971 readUnitConfObj = ReadUnitConf()
993 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
972 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
994
973
995 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
974 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
996
975
997 return readUnitConfObj
976 return readUnitConfObj
998
977
999 def addProcUnit(self, inputId='0', datatype=None, name=None):
978 def addProcUnit(self, inputId='0', datatype=None, name=None):
1000
979
1001 idProcUnit = self.__getNewId()
980 idProcUnit = self.__getNewId()
1002
981
1003 procUnitConfObj = ProcUnitConf()
982 procUnitConfObj = ProcUnitConf()
1004 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
983 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1005
984
1006 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
985 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1007
986
1008 return procUnitConfObj
987 return procUnitConfObj
1009
988
1010 def removeProcUnit(self, id):
989 def removeProcUnit(self, id):
1011
990
1012 if id in self.procUnitConfObjDict.keys():
991 if id in self.procUnitConfObjDict.keys():
1013 self.procUnitConfObjDict.pop(id)
992 self.procUnitConfObjDict.pop(id)
1014
993
1015 def getReadUnitId(self):
994 def getReadUnitId(self):
1016
995
1017 readUnitConfObj = self.getReadUnitObj()
996 readUnitConfObj = self.getReadUnitObj()
1018
997
1019 return readUnitConfObj.id
998 return readUnitConfObj.id
1020
999
1021 def getReadUnitObj(self):
1000 def getReadUnitObj(self):
1022
1001
1023 for obj in self.procUnitConfObjDict.values():
1002 for obj in self.procUnitConfObjDict.values():
1024 if obj.getElementName() == "ReadUnit":
1003 if obj.getElementName() == 'ReadUnit':
1025 return obj
1004 return obj
1026
1005
1027 return None
1006 return None
1028
1007
1029 def getProcUnitObj(self, id=None, name=None):
1008 def getProcUnitObj(self, id=None, name=None):
1030
1009
1031 if id != None:
1010 if id != None:
1032 return self.procUnitConfObjDict[id]
1011 return self.procUnitConfObjDict[id]
1033
1012
1034 if name != None:
1013 if name != None:
1035 return self.getProcUnitObjByName(name)
1014 return self.getProcUnitObjByName(name)
1036
1015
1037 return None
1016 return None
1038
1017
1039 def getProcUnitObjByName(self, name):
1018 def getProcUnitObjByName(self, name):
1040
1019
1041 for obj in self.procUnitConfObjDict.values():
1020 for obj in self.procUnitConfObjDict.values():
1042 if obj.name == name:
1021 if obj.name == name:
1043 return obj
1022 return obj
1044
1023
1045 return None
1024 return None
1046
1025
1047 def procUnitItems(self):
1026 def procUnitItems(self):
1048
1027
1049 return self.procUnitConfObjDict.items()
1028 return self.procUnitConfObjDict.items()
1050
1029
1051 def makeXml(self):
1030 def makeXml(self):
1052
1031
1053 projectElement = Element('Project')
1032 projectElement = Element('Project')
1054 projectElement.set('id', str(self.id))
1033 projectElement.set('id', str(self.id))
1055 projectElement.set('name', self.name)
1034 projectElement.set('name', self.name)
1056 projectElement.set('description', self.description)
1035 projectElement.set('description', self.description)
1057
1036
1058 for procUnitConfObj in self.procUnitConfObjDict.values():
1037 for procUnitConfObj in self.procUnitConfObjDict.values():
1059 procUnitConfObj.makeXml(projectElement)
1038 procUnitConfObj.makeXml(projectElement)
1060
1039
1061 self.projectElement = projectElement
1040 self.projectElement = projectElement
1062
1041
1063 def writeXml(self, filename=None):
1042 def writeXml(self, filename=None):
1064
1043
1065 if filename == None:
1044 if filename == None:
1066 if self.filename:
1045 if self.filename:
1067 filename = self.filename
1046 filename = self.filename
1068 else:
1047 else:
1069 filename = "schain.xml"
1048 filename = 'schain.xml'
1070
1049
1071 if not filename:
1050 if not filename:
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1073 return 0
1052 return 0
1074
1053
1075 abs_file = os.path.abspath(filename)
1054 abs_file = os.path.abspath(filename)
1076
1055
1077 if not os.access(os.path.dirname(abs_file), os.W_OK):
1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1079 return 0
1058 return 0
1080
1059
1081 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1082 print "File %s already exists and it could not be overwriten" %abs_file
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1083 return 0
1062 return 0
1084
1063
1085 self.makeXml()
1064 self.makeXml()
1086
1065
1087 ElementTree(self.projectElement).write(abs_file, method='xml')
1066 ElementTree(self.projectElement).write(abs_file, method='xml')
1088
1067
1089 self.filename = abs_file
1068 self.filename = abs_file
1090
1069
1091 return 1
1070 return 1
1092
1071
1093 def readXml(self, filename = None):
1072 def readXml(self, filename = None):
1094
1073
1095 if not filename:
1074 if not filename:
1096 print "filename is not defined"
1075 print 'filename is not defined'
1097 return 0
1076 return 0
1098
1077
1099 abs_file = os.path.abspath(filename)
1078 abs_file = os.path.abspath(filename)
1100
1079
1101 if not os.path.isfile(abs_file):
1080 if not os.path.isfile(abs_file):
1102 print "%s file does not exist" %abs_file
1081 print '%s file does not exist' %abs_file
1103 return 0
1082 return 0
1104
1083
1105 self.projectElement = None
1084 self.projectElement = None
1106 self.procUnitConfObjDict = {}
1085 self.procUnitConfObjDict = {}
1107
1086
1108 try:
1087 try:
1109 self.projectElement = ElementTree().parse(abs_file)
1088 self.projectElement = ElementTree().parse(abs_file)
1110 except:
1089 except:
1111 print "Error reading %s, verify file format" %filename
1090 print 'Error reading %s, verify file format' %filename
1112 return 0
1091 return 0
1113
1092
1114 self.project = self.projectElement.tag
1093 self.project = self.projectElement.tag
1115
1094
1116 self.id = self.projectElement.get('id')
1095 self.id = self.projectElement.get('id')
1117 self.name = self.projectElement.get('name')
1096 self.name = self.projectElement.get('name')
1118 self.description = self.projectElement.get('description')
1097 self.description = self.projectElement.get('description')
1119
1098
1120 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1099 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1121
1100
1122 for readUnitElement in readUnitElementList:
1101 for readUnitElement in readUnitElementList:
1123 readUnitConfObj = ReadUnitConf()
1102 readUnitConfObj = ReadUnitConf()
1124 readUnitConfObj.readXml(readUnitElement)
1103 readUnitConfObj.readXml(readUnitElement)
1125
1104
1126 if readUnitConfObj.parentId == None:
1105 if readUnitConfObj.parentId == None:
1127 readUnitConfObj.parentId = self.id
1106 readUnitConfObj.parentId = self.id
1128
1107
1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1108 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130
1109
1131 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1110 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1132
1111
1133 for procUnitElement in procUnitElementList:
1112 for procUnitElement in procUnitElementList:
1134 procUnitConfObj = ProcUnitConf()
1113 procUnitConfObj = ProcUnitConf()
1135 procUnitConfObj.readXml(procUnitElement)
1114 procUnitConfObj.readXml(procUnitElement)
1136
1115
1137 if procUnitConfObj.parentId == None:
1116 if procUnitConfObj.parentId == None:
1138 procUnitConfObj.parentId = self.id
1117 procUnitConfObj.parentId = self.id
1139
1118
1140 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1119 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1141
1120
1142 self.filename = abs_file
1121 self.filename = abs_file
1143
1122
1144 return 1
1123 return 1
1145
1124
1146 def printattr(self):
1125 def printattr(self):
1147
1126
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1149 self.name,
1128 self.name,
1150 self.description)
1129 self.description)
1151
1130
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 procUnitConfObj.printattr()
1132 procUnitConfObj.printattr()
1154
1133
1155 def createObjects(self):
1134 def createObjects(self):
1156
1135
1157 for procUnitConfObj in self.procUnitConfObjDict.values():
1136 for procUnitConfObj in self.procUnitConfObjDict.values():
1158 procUnitConfObj.createObjects(self.plotterQueue)
1137 procUnitConfObj.createObjects(self.plotterQueue)
1159
1138
1160 def __connect(self, objIN, thisObj):
1139 def __connect(self, objIN, thisObj):
1161
1140
1162 thisObj.setInput(objIN.getOutputObj())
1141 thisObj.setInput(objIN.getOutputObj())
1163
1142
1164 def connectObjects(self):
1143 def connectObjects(self):
1165
1144
1166 for thisPUConfObj in self.procUnitConfObjDict.values():
1145 for thisPUConfObj in self.procUnitConfObjDict.values():
1167
1146
1168 inputId = thisPUConfObj.getInputId()
1147 inputId = thisPUConfObj.getInputId()
1169
1148
1170 if int(inputId) == 0:
1149 if int(inputId) == 0:
1171 continue
1150 continue
1172
1151
1173 #Get input object
1152 #Get input object
1174 puConfINObj = self.procUnitConfObjDict[inputId]
1153 puConfINObj = self.procUnitConfObjDict[inputId]
1175 puObjIN = puConfINObj.getProcUnitObj()
1154 puObjIN = puConfINObj.getProcUnitObj()
1176
1155
1177 #Get current object
1156 #Get current object
1178 thisPUObj = thisPUConfObj.getProcUnitObj()
1157 thisPUObj = thisPUConfObj.getProcUnitObj()
1179
1158
1180 self.__connect(puObjIN, thisPUObj)
1159 self.__connect(puObjIN, thisPUObj)
1181
1160
1182 def __handleError(self, procUnitConfObj, send_email=True):
1161 def __handleError(self, procUnitConfObj, send_email=False):
1183
1162
1184 import socket
1163 import socket
1185
1164
1186 err = traceback.format_exception(sys.exc_info()[0],
1165 err = traceback.format_exception(sys.exc_info()[0],
1187 sys.exc_info()[1],
1166 sys.exc_info()[1],
1188 sys.exc_info()[2])
1167 sys.exc_info()[2])
1189
1168
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1191 print "***** %s" %err[-1]
1170 print '***** %s' %err[-1]
1192
1171
1193 message = "".join(err)
1172 message = ''.join(err)
1194
1173
1195 sys.stderr.write(message)
1174 sys.stderr.write(message)
1196
1175
1197 if not send_email:
1176 if not send_email:
1198 return
1177 return
1199
1178
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1201
1180
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1205 subtitle += "Configuration file: %s\n" %self.filename
1184 subtitle += 'Configuration file: %s\n' %self.filename
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1207
1186
1208 readUnitConfObj = self.getReadUnitObj()
1187 readUnitConfObj = self.getReadUnitObj()
1209 if readUnitConfObj:
1188 if readUnitConfObj:
1210 subtitle += "\nInput parameters:\n"
1189 subtitle += '\nInput parameters:\n'
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1217
1196
1218 adminObj = schainpy.admin.SchainNotify()
1197 adminObj = schainpy.admin.SchainNotify()
1219 adminObj.sendAlert(message=message,
1198 adminObj.sendAlert(message=message,
1220 subject=subject,
1199 subject=subject,
1221 subtitle=subtitle,
1200 subtitle=subtitle,
1222 filename=self.filename)
1201 filename=self.filename)
1223
1202
1224 def isPaused(self):
1203 def isPaused(self):
1225 return 0
1204 return 0
1226
1205
1227 def isStopped(self):
1206 def isStopped(self):
1228 return 0
1207 return 0
1229
1208
1230 def runController(self):
1209 def runController(self):
1231 """
1210 '''
1232 returns 0 when this process has been stopped, 1 otherwise
1211 returns 0 when this process has been stopped, 1 otherwise
1233 """
1212 '''
1234
1213
1235 if self.isPaused():
1214 if self.isPaused():
1236 print "Process suspended"
1215 print 'Process suspended'
1237
1216
1238 while True:
1217 while True:
1239 sleep(0.1)
1218 time.sleep(0.1)
1240
1219
1241 if not self.isPaused():
1220 if not self.isPaused():
1242 break
1221 break
1243
1222
1244 if self.isStopped():
1223 if self.isStopped():
1245 break
1224 break
1246
1225
1247 print "Process reinitialized"
1226 print 'Process reinitialized'
1248
1227
1249 if self.isStopped():
1228 if self.isStopped():
1250 print "Process stopped"
1229 print 'Process stopped'
1251 return 0
1230 return 0
1252
1231
1253 return 1
1232 return 1
1254
1233
1255 def setFilename(self, filename):
1234 def setFilename(self, filename):
1256
1235
1257 self.filename = filename
1236 self.filename = filename
1258
1237
1259 def setPlotterQueue(self, plotter_queue):
1238 def setPlotterQueue(self, plotter_queue):
1260
1239
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1262
1241
1263 def getPlotterQueue(self):
1242 def getPlotterQueue(self):
1264
1243
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1266
1245
1267 def useExternalPlotter(self):
1246 def useExternalPlotter(self):
1268
1247
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270
1249
1250 def run(self):
1271
1251
1272 def run(self, filename=None):
1252 log.success('Starting {}'.format(self.name))
1273
1253
1274 # self.writeXml(filename)
1275 self.createObjects()
1254 self.createObjects()
1276 self.connectObjects()
1255 self.connectObjects()
1277
1256
1278 print
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1282 print
1283
1284 keyList = self.procUnitConfObjDict.keys()
1257 keyList = self.procUnitConfObjDict.keys()
1285 keyList.sort()
1258 keyList.sort()
1286
1259
1287 while(True):
1260 while(True):
1288
1261
1289 is_ok = False
1262 is_ok = False
1290
1263
1291 for procKey in keyList:
1264 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293
1265
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295
1267
1296 try:
1268 try:
1297 sts = procUnitConfObj.run()
1269 sts = procUnitConfObj.run()
1298 is_ok = is_ok or sts
1270 is_ok = is_ok or sts
1299 except KeyboardInterrupt:
1271 except KeyboardInterrupt:
1300 is_ok = False
1272 is_ok = False
1301 break
1273 break
1302 except ValueError, e:
1274 except ValueError, e:
1303 sleep(0.5)
1275 time.sleep(0.5)
1304 self.__handleError(procUnitConfObj, send_email=True)
1276 self.__handleError(procUnitConfObj, send_email=True)
1305 is_ok = False
1277 is_ok = False
1306 break
1278 break
1307 except:
1279 except:
1308 sleep(0.5)
1280 time.sleep(0.5)
1309 self.__handleError(procUnitConfObj)
1281 self.__handleError(procUnitConfObj)
1310 is_ok = False
1282 is_ok = False
1311 break
1283 break
1312
1284
1313 #If every process unit finished so end process
1285 #If every process unit finished so end process
1314 if not(is_ok):
1286 if not(is_ok):
1315 # print "Every process unit have finished"
1316 break
1287 break
1317
1288
1318 if not self.runController():
1289 if not self.runController():
1319 break
1290 break
1320
1291
1321 #Closing every process
1292 #Closing every process
1322 for procKey in keyList:
1293 for procKey in keyList:
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 procUnitConfObj.close()
1295 procUnitConfObj.close()
1296
1297 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,692 +1,692
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4
4
5 import os
5 import os
6 import sys
6 import sys
7 import glob
7 import glob
8 import fnmatch
8 import fnmatch
9 import datetime
9 import datetime
10 import time
10 import time
11 import re
11 import re
12 import h5py
12 import h5py
13 import numpy
13 import numpy
14
14
15 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
15 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
16 from schainpy.model.data.jroamisr import AMISR
16 from schainpy.model.data.jroamisr import AMISR
17
17
18 try:
18 try:
19 from gevent import sleep
19 from gevent import sleep
20 except:
20 except:
21 from time import sleep
21 from time import sleep
22
22
23 class RadacHeader():
23 class RadacHeader():
24 def __init__(self, fp):
24 def __init__(self, fp):
25 header = 'Raw11/Data/RadacHeader'
25 header = 'Raw11/Data/RadacHeader'
26 self.beamCodeByPulse = fp.get(header+'/BeamCode')
26 self.beamCodeByPulse = fp.get(header+'/BeamCode')
27 self.beamCode = fp.get('Raw11/Data/Beamcodes')
27 self.beamCode = fp.get('Raw11/Data/Beamcodes')
28 self.code = fp.get(header+'/Code')
28 self.code = fp.get(header+'/Code')
29 self.frameCount = fp.get(header+'/FrameCount')
29 self.frameCount = fp.get(header+'/FrameCount')
30 self.modeGroup = fp.get(header+'/ModeGroup')
30 self.modeGroup = fp.get(header+'/ModeGroup')
31 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
31 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
32 self.pulseCount = fp.get(header+'/PulseCount')
32 self.pulseCount = fp.get(header+'/PulseCount')
33 self.radacTime = fp.get(header+'/RadacTime')
33 self.radacTime = fp.get(header+'/RadacTime')
34 self.timeCount = fp.get(header+'/TimeCount')
34 self.timeCount = fp.get(header+'/TimeCount')
35 self.timeStatus = fp.get(header+'/TimeStatus')
35 self.timeStatus = fp.get(header+'/TimeStatus')
36
36
37 self.nrecords = self.pulseCount.shape[0] #nblocks
37 self.nrecords = self.pulseCount.shape[0] #nblocks
38 self.npulses = self.pulseCount.shape[1] #nprofile
38 self.npulses = self.pulseCount.shape[1] #nprofile
39 self.nsamples = self.nsamplesPulse[0,0] #ngates
39 self.nsamples = self.nsamplesPulse[0,0] #ngates
40 self.nbeams = self.beamCode.shape[1]
40 self.nbeams = self.beamCode.shape[1]
41
41
42
42
43 def getIndexRangeToPulse(self, idrecord=0):
43 def getIndexRangeToPulse(self, idrecord=0):
44 #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
44 #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
45 #startPulseCountId = indexToZero[0][0]
45 #startPulseCountId = indexToZero[0][0]
46 #endPulseCountId = startPulseCountId - 1
46 #endPulseCountId = startPulseCountId - 1
47 #range1 = numpy.arange(startPulseCountId,self.npulses,1)
47 #range1 = numpy.arange(startPulseCountId,self.npulses,1)
48 #range2 = numpy.arange(0,startPulseCountId,1)
48 #range2 = numpy.arange(0,startPulseCountId,1)
49 #return range1, range2
49 #return range1, range2
50 zero = 0
50 zero = 0
51 npulse = max(self.pulseCount[0,:]+1)-1
51 npulse = max(self.pulseCount[0,:]+1)-1
52 looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
52 looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
53 getLastIndex = looking_index[-1]
53 getLastIndex = looking_index[-1]
54 index_data = numpy.arange(0,getLastIndex+1,1)
54 index_data = numpy.arange(0,getLastIndex+1,1)
55 index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
55 index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
56 return index_data, index_buffer
56 return index_data, index_buffer
57
57
58 class AMISRReader(ProcessingUnit):
58 class AMISRReader(ProcessingUnit):
59
59
60 path = None
60 path = None
61 startDate = None
61 startDate = None
62 endDate = None
62 endDate = None
63 startTime = None
63 startTime = None
64 endTime = None
64 endTime = None
65 walk = None
65 walk = None
66 isConfig = False
66 isConfig = False
67
67
68 def __init__(self):
68 def __init__(self):
69 self.set = None
69 self.set = None
70 self.subset = None
70 self.subset = None
71 self.extension_file = '.h5'
71 self.extension_file = '.h5'
72 self.dtc_str = 'dtc'
72 self.dtc_str = 'dtc'
73 self.dtc_id = 0
73 self.dtc_id = 0
74 self.status = True
74 self.status = True
75 self.isConfig = False
75 self.isConfig = False
76 self.dirnameList = []
76 self.dirnameList = []
77 self.filenameList = []
77 self.filenameList = []
78 self.fileIndex = None
78 self.fileIndex = None
79 self.flagNoMoreFiles = False
79 self.flagNoMoreFiles = False
80 self.flagIsNewFile = 0
80 self.flagIsNewFile = 0
81 self.filename = ''
81 self.filename = ''
82 self.amisrFilePointer = None
82 self.amisrFilePointer = None
83 self.radacHeaderObj = None
83 self.radacHeaderObj = None
84 self.dataOut = self.__createObjByDefault()
84 self.dataOut = self.__createObjByDefault()
85 self.datablock = None
85 self.datablock = None
86 self.rest_datablock = None
86 self.rest_datablock = None
87 self.range = None
87 self.range = None
88 self.idrecord_count = 0
88 self.idrecord_count = 0
89 self.profileIndex = 0
89 self.profileIndex = 0
90 self.index_amisr_sample = None
90 self.index_amisr_sample = None
91 self.index_amisr_buffer = None
91 self.index_amisr_buffer = None
92 self.beamCodeByFrame = None
92 self.beamCodeByFrame = None
93 self.radacTimeByFrame = None
93 self.radacTimeByFrame = None
94 #atributos originales tal y como esta en el archivo de datos
94 #atributos originales tal y como esta en el archivo de datos
95 self.beamCodesFromFile = None
95 self.beamCodesFromFile = None
96 self.radacTimeFromFile = None
96 self.radacTimeFromFile = None
97 self.rangeFromFile = None
97 self.rangeFromFile = None
98 self.dataByFrame = None
98 self.dataByFrame = None
99 self.dataset = None
99 self.dataset = None
100
100
101 self.beamCodeDict = {}
101 self.beamCodeDict = {}
102 self.beamRangeDict = {}
102 self.beamRangeDict = {}
103
103
104 #experiment cgf file
104 #experiment cgf file
105 self.npulsesint_fromfile = None
105 self.npulsesint_fromfile = None
106 self.recordsperfile_fromfile = None
106 self.recordsperfile_fromfile = None
107 self.nbeamcodes_fromfile = None
107 self.nbeamcodes_fromfile = None
108 self.ngates_fromfile = None
108 self.ngates_fromfile = None
109 self.ippSeconds_fromfile = None
109 self.ippSeconds_fromfile = None
110 self.frequency_h5file = None
110 self.frequency_h5file = None
111
111
112
112
113 self.__firstFile = True
113 self.__firstFile = True
114 self.buffer_radactime = None
114 self.buffer_radactime = None
115
115
116 self.index4_schain_datablock = None
116 self.index4_schain_datablock = None
117 self.index4_buffer = None
117 self.index4_buffer = None
118 self.schain_datablock = None
118 self.schain_datablock = None
119 self.buffer = None
119 self.buffer = None
120 self.linear_pulseCount = None
120 self.linear_pulseCount = None
121 self.npulseByFrame = None
121 self.npulseByFrame = None
122 self.profileIndex_offset = None
122 self.profileIndex_offset = None
123 self.timezone = 'ut'
123 self.timezone = 'ut'
124
124
125 self.__waitForNewFile = 20
125 self.__waitForNewFile = 20
126 self.__filename_online = None
126 self.__filename_online = None
127
127
128 def __createObjByDefault(self):
128 def __createObjByDefault(self):
129
129
130 dataObj = AMISR()
130 dataObj = AMISR()
131
131
132 return dataObj
132 return dataObj
133
133
134 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
134 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
135 self.path = path
135 self.path = path
136 self.startDate = startDate
136 self.startDate = startDate
137 self.endDate = endDate
137 self.endDate = endDate
138 self.startTime = startTime
138 self.startTime = startTime
139 self.endTime = endTime
139 self.endTime = endTime
140 self.walk = walk
140 self.walk = walk
141
141
142 def __checkPath(self):
142 def __checkPath(self):
143 if os.path.exists(self.path):
143 if os.path.exists(self.path):
144 self.status = 1
144 self.status = 1
145 else:
145 else:
146 self.status = 0
146 self.status = 0
147 print 'Path:%s does not exists'%self.path
147 print 'Path:%s does not exists'%self.path
148
148
149 return
149 return
150
150
151 def __selDates(self, amisr_dirname_format):
151 def __selDates(self, amisr_dirname_format):
152 try:
152 try:
153 year = int(amisr_dirname_format[0:4])
153 year = int(amisr_dirname_format[0:4])
154 month = int(amisr_dirname_format[4:6])
154 month = int(amisr_dirname_format[4:6])
155 dom = int(amisr_dirname_format[6:8])
155 dom = int(amisr_dirname_format[6:8])
156 thisDate = datetime.date(year,month,dom)
156 thisDate = datetime.date(year,month,dom)
157
157
158 if (thisDate>=self.startDate and thisDate <= self.endDate):
158 if (thisDate>=self.startDate and thisDate <= self.endDate):
159 return amisr_dirname_format
159 return amisr_dirname_format
160 except:
160 except:
161 return None
161 return None
162
162
163 def __findDataForDates(self,online=False):
163 def __findDataForDates(self,online=False):
164
164
165
165
166
166
167 if not(self.status):
167 if not(self.status):
168 return None
168 return None
169
169
170 pat = '\d+.\d+'
170 pat = '\d+.\d+'
171 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
171 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
172 dirnameList = filter(lambda x:x!=None,dirnameList)
172 dirnameList = filter(lambda x:x!=None,dirnameList)
173 dirnameList = [x.string for x in dirnameList]
173 dirnameList = [x.string for x in dirnameList]
174 if not(online):
174 if not(online):
175 dirnameList = [self.__selDates(x) for x in dirnameList]
175 dirnameList = [self.__selDates(x) for x in dirnameList]
176 dirnameList = filter(lambda x:x!=None,dirnameList)
176 dirnameList = filter(lambda x:x!=None,dirnameList)
177 if len(dirnameList)>0:
177 if len(dirnameList)>0:
178 self.status = 1
178 self.status = 1
179 self.dirnameList = dirnameList
179 self.dirnameList = dirnameList
180 self.dirnameList.sort()
180 self.dirnameList.sort()
181 else:
181 else:
182 self.status = 0
182 self.status = 0
183 return None
183 return None
184
184
185 def __getTimeFromData(self):
185 def __getTimeFromData(self):
186 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
186 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
187 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
187 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
188
188
189 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
189 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
190 print '........................................'
190 print '........................................'
191 filter_filenameList = []
191 filter_filenameList = []
192 self.filenameList.sort()
192 self.filenameList.sort()
193 for i in range(len(self.filenameList)-1):
193 for i in range(len(self.filenameList)-1):
194 filename = self.filenameList[i]
194 filename = self.filenameList[i]
195 fp = h5py.File(filename,'r')
195 fp = h5py.File(filename,'r')
196 time_str = fp.get('Time/RadacTimeString')
196 time_str = fp.get('Time/RadacTimeString')
197
197
198 startDateTimeStr_File = time_str[0][0].split('.')[0]
198 startDateTimeStr_File = time_str[0][0].split('.')[0]
199 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
199 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
200 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
200 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
201
201
202 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
202 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
203 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
203 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
204 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
204 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
205
205
206 fp.close()
206 fp.close()
207
207
208 if self.timezone == 'lt':
208 if self.timezone == 'lt':
209 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
209 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
210 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
210 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
211
211
212 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
212 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
213 #self.filenameList.remove(filename)
213 #self.filenameList.remove(filename)
214 filter_filenameList.append(filename)
214 filter_filenameList.append(filename)
215
215
216 filter_filenameList.sort()
216 filter_filenameList.sort()
217 self.filenameList = filter_filenameList
217 self.filenameList = filter_filenameList
218 return 1
218 return 1
219
219
220 def __filterByGlob1(self, dirName):
220 def __filterByGlob1(self, dirName):
221 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
221 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
222 filterDict = {}
222 filterDict = {}
223 filterDict.setdefault(dirName)
223 filterDict.setdefault(dirName)
224 filterDict[dirName] = filter_files
224 filterDict[dirName] = filter_files
225 return filterDict
225 return filterDict
226
226
227 def __getFilenameList(self, fileListInKeys, dirList):
227 def __getFilenameList(self, fileListInKeys, dirList):
228 for value in fileListInKeys:
228 for value in fileListInKeys:
229 dirName = value.keys()[0]
229 dirName = value.keys()[0]
230 for file in value[dirName]:
230 for file in value[dirName]:
231 filename = os.path.join(dirName, file)
231 filename = os.path.join(dirName, file)
232 self.filenameList.append(filename)
232 self.filenameList.append(filename)
233
233
234
234
235 def __selectDataForTimes(self, online=False):
235 def __selectDataForTimes(self, online=False):
236 #aun no esta implementado el filtro for tiempo
236 #aun no esta implementado el filtro for tiempo
237 if not(self.status):
237 if not(self.status):
238 return None
238 return None
239
239
240 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
240 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
241
241
242 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
242 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
243
243
244 self.__getFilenameList(fileListInKeys, dirList)
244 self.__getFilenameList(fileListInKeys, dirList)
245 if not(online):
245 if not(online):
246 #filtro por tiempo
246 #filtro por tiempo
247 if not(self.all):
247 if not(self.all):
248 self.__getTimeFromData()
248 self.__getTimeFromData()
249
249
250 if len(self.filenameList)>0:
250 if len(self.filenameList)>0:
251 self.status = 1
251 self.status = 1
252 self.filenameList.sort()
252 self.filenameList.sort()
253 else:
253 else:
254 self.status = 0
254 self.status = 0
255 return None
255 return None
256
256
257 else:
257 else:
258 #get the last file - 1
258 #get the last file - 1
259 self.filenameList = [self.filenameList[-2]]
259 self.filenameList = [self.filenameList[-2]]
260
260
261 new_dirnameList = []
261 new_dirnameList = []
262 for dirname in self.dirnameList:
262 for dirname in self.dirnameList:
263 junk = numpy.array([dirname in x for x in self.filenameList])
263 junk = numpy.array([dirname in x for x in self.filenameList])
264 junk_sum = junk.sum()
264 junk_sum = junk.sum()
265 if junk_sum > 0:
265 if junk_sum > 0:
266 new_dirnameList.append(dirname)
266 new_dirnameList.append(dirname)
267 self.dirnameList = new_dirnameList
267 self.dirnameList = new_dirnameList
268 return 1
268 return 1
269
269
270 def __searchFilesOnline(self,
270 def searchFilesOnLine(self,
271 path,
271 path,
272 walk=True):
272 walk=True):
273
273
274 startDate = datetime.datetime.utcnow().date()
274 startDate = datetime.datetime.utcnow().date()
275 endDate = datetime.datetime.utcnow().date()
275 endDate = datetime.datetime.utcnow().date()
276
276
277 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
277 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
278
278
279 self.__checkPath()
279 self.__checkPath()
280
280
281 self.__findDataForDates(online=True)
281 self.__findDataForDates(online=True)
282
282
283 self.dirnameList = [self.dirnameList[-1]]
283 self.dirnameList = [self.dirnameList[-1]]
284
284
285 self.__selectDataForTimes(online=True)
285 self.__selectDataForTimes(online=True)
286
286
287 return
287 return
288
288
289
289
290 def __searchFilesOffline(self,
290 def searchFilesOffLine(self,
291 path,
291 path,
292 startDate,
292 startDate,
293 endDate,
293 endDate,
294 startTime=datetime.time(0,0,0),
294 startTime=datetime.time(0,0,0),
295 endTime=datetime.time(23,59,59),
295 endTime=datetime.time(23,59,59),
296 walk=True):
296 walk=True):
297
297
298 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
298 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
299
299
300 self.__checkPath()
300 self.__checkPath()
301
301
302 self.__findDataForDates()
302 self.__findDataForDates()
303
303
304 self.__selectDataForTimes()
304 self.__selectDataForTimes()
305
305
306 for i in range(len(self.filenameList)):
306 for i in range(len(self.filenameList)):
307 print "%s" %(self.filenameList[i])
307 print "%s" %(self.filenameList[i])
308
308
309 return
309 return
310
310
311 def __setNextFileOffline(self):
311 def __setNextFileOffline(self):
312 idFile = self.fileIndex
312 idFile = self.fileIndex
313
313
314 while (True):
314 while (True):
315 idFile += 1
315 idFile += 1
316 if not(idFile < len(self.filenameList)):
316 if not(idFile < len(self.filenameList)):
317 self.flagNoMoreFiles = 1
317 self.flagNoMoreFiles = 1
318 print "No more Files"
318 print "No more Files"
319 return 0
319 return 0
320
320
321 filename = self.filenameList[idFile]
321 filename = self.filenameList[idFile]
322
322
323 amisrFilePointer = h5py.File(filename,'r')
323 amisrFilePointer = h5py.File(filename,'r')
324
324
325 break
325 break
326
326
327 self.flagIsNewFile = 1
327 self.flagIsNewFile = 1
328 self.fileIndex = idFile
328 self.fileIndex = idFile
329 self.filename = filename
329 self.filename = filename
330
330
331 self.amisrFilePointer = amisrFilePointer
331 self.amisrFilePointer = amisrFilePointer
332
332
333 print "Setting the file: %s"%self.filename
333 print "Setting the file: %s"%self.filename
334
334
335 return 1
335 return 1
336
336
337
337
338 def __setNextFileOnline(self):
338 def __setNextFileOnline(self):
339 filename = self.filenameList[0]
339 filename = self.filenameList[0]
340 if self.__filename_online != None:
340 if self.__filename_online != None:
341 self.__selectDataForTimes(online=True)
341 self.__selectDataForTimes(online=True)
342 filename = self.filenameList[0]
342 filename = self.filenameList[0]
343 while self.__filename_online == filename:
343 while self.__filename_online == filename:
344 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
344 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
345 sleep(self.__waitForNewFile)
345 sleep(self.__waitForNewFile)
346 self.__selectDataForTimes(online=True)
346 self.__selectDataForTimes(online=True)
347 filename = self.filenameList[0]
347 filename = self.filenameList[0]
348
348
349 self.__filename_online = filename
349 self.__filename_online = filename
350
350
351 self.amisrFilePointer = h5py.File(filename,'r')
351 self.amisrFilePointer = h5py.File(filename,'r')
352 self.flagIsNewFile = 1
352 self.flagIsNewFile = 1
353 self.filename = filename
353 self.filename = filename
354 print "Setting the file: %s"%self.filename
354 print "Setting the file: %s"%self.filename
355 return 1
355 return 1
356
356
357
357
358 def __readHeader(self):
358 def __readHeader(self):
359 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
359 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
360
360
361 #update values from experiment cfg file
361 #update values from experiment cfg file
362 if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
362 if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
363 self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
363 self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
364 self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
364 self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
365 self.radacHeaderObj.npulses = self.npulsesint_fromfile
365 self.radacHeaderObj.npulses = self.npulsesint_fromfile
366 self.radacHeaderObj.nsamples = self.ngates_fromfile
366 self.radacHeaderObj.nsamples = self.ngates_fromfile
367
367
368 #looking index list for data
368 #looking index list for data
369 start_index = self.radacHeaderObj.pulseCount[0,:][0]
369 start_index = self.radacHeaderObj.pulseCount[0,:][0]
370 end_index = self.radacHeaderObj.npulses
370 end_index = self.radacHeaderObj.npulses
371 range4data = range(start_index, end_index)
371 range4data = range(start_index, end_index)
372 self.index4_schain_datablock = numpy.array(range4data)
372 self.index4_schain_datablock = numpy.array(range4data)
373
373
374 buffer_start_index = 0
374 buffer_start_index = 0
375 buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
375 buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
376 range4buffer = range(buffer_start_index, buffer_end_index)
376 range4buffer = range(buffer_start_index, buffer_end_index)
377 self.index4_buffer = numpy.array(range4buffer)
377 self.index4_buffer = numpy.array(range4buffer)
378
378
379 self.linear_pulseCount = numpy.array(range4data + range4buffer)
379 self.linear_pulseCount = numpy.array(range4data + range4buffer)
380 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
380 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
381
381
382 #get tuning frequency
382 #get tuning frequency
383 frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
383 frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
384 self.frequency_h5file = frequency_h5file_dataset[0,0]
384 self.frequency_h5file = frequency_h5file_dataset[0,0]
385
385
386 self.flagIsNewFile = 1
386 self.flagIsNewFile = 1
387
387
388 def __getBeamCode(self):
388 def __getBeamCode(self):
389 self.beamCodeDict = {}
389 self.beamCodeDict = {}
390 self.beamRangeDict = {}
390 self.beamRangeDict = {}
391
391
392 beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
392 beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
393
393
394 for i in range(len(self.radacHeaderObj.beamCode[0,:])):
394 for i in range(len(self.radacHeaderObj.beamCode[0,:])):
395 self.beamCodeDict.setdefault(i)
395 self.beamCodeDict.setdefault(i)
396 self.beamRangeDict.setdefault(i)
396 self.beamRangeDict.setdefault(i)
397 beamcodeValue = self.radacHeaderObj.beamCode[0,i]
397 beamcodeValue = self.radacHeaderObj.beamCode[0,i]
398 beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
398 beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
399 x = beamCodeMap[beamcodeIndex][1]
399 x = beamCodeMap[beamcodeIndex][1]
400 y = beamCodeMap[beamcodeIndex][2]
400 y = beamCodeMap[beamcodeIndex][2]
401 z = beamCodeMap[beamcodeIndex][3]
401 z = beamCodeMap[beamcodeIndex][3]
402 self.beamCodeDict[i] = [beamcodeValue, x, y, z]
402 self.beamCodeDict[i] = [beamcodeValue, x, y, z]
403
403
404 just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
404 just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
405
405
406 for i in range(len(self.beamCodeDict.values())):
406 for i in range(len(self.beamCodeDict.values())):
407 xx = numpy.where(just4record0==self.beamCodeDict.values()[i][0])
407 xx = numpy.where(just4record0==self.beamCodeDict.values()[i][0])
408 indexPulseByBeam = self.linear_pulseCount[xx[0]]
408 indexPulseByBeam = self.linear_pulseCount[xx[0]]
409 self.beamRangeDict[i] = indexPulseByBeam
409 self.beamRangeDict[i] = indexPulseByBeam
410
410
411 def __getExpParameters(self):
411 def __getExpParameters(self):
412 if not(self.status):
412 if not(self.status):
413 return None
413 return None
414
414
415 experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
415 experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
416
416
417 expFinder = glob.glob1(experimentCfgPath,'*.exp')
417 expFinder = glob.glob1(experimentCfgPath,'*.exp')
418 if len(expFinder)== 0:
418 if len(expFinder)== 0:
419 self.status = 0
419 self.status = 0
420 return None
420 return None
421
421
422 experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
422 experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
423
423
424 f = open(experimentFilename)
424 f = open(experimentFilename)
425 lines = f.readlines()
425 lines = f.readlines()
426 f.close()
426 f.close()
427
427
428 parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
428 parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
429 filterList = [fnmatch.filter(lines, x) for x in parmsList]
429 filterList = [fnmatch.filter(lines, x) for x in parmsList]
430
430
431
431
432 values = [re.sub(r'\D',"",x[0]) for x in filterList]
432 values = [re.sub(r'\D',"",x[0]) for x in filterList]
433
433
434 self.npulsesint_fromfile = int(values[0])
434 self.npulsesint_fromfile = int(values[0])
435 self.recordsperfile_fromfile = int(values[1])
435 self.recordsperfile_fromfile = int(values[1])
436 self.nbeamcodes_fromfile = int(values[2])
436 self.nbeamcodes_fromfile = int(values[2])
437 self.ngates_fromfile = int(values[3])
437 self.ngates_fromfile = int(values[3])
438
438
439 tufileFinder = fnmatch.filter(lines, 'tufile=*')
439 tufileFinder = fnmatch.filter(lines, 'tufile=*')
440 tufile = tufileFinder[0].split('=')[1].split('\n')[0]
440 tufile = tufileFinder[0].split('=')[1].split('\n')[0]
441 tufile = tufile.split('\r')[0]
441 tufile = tufile.split('\r')[0]
442 tufilename = os.path.join(experimentCfgPath,tufile)
442 tufilename = os.path.join(experimentCfgPath,tufile)
443
443
444 f = open(tufilename)
444 f = open(tufilename)
445 lines = f.readlines()
445 lines = f.readlines()
446 f.close()
446 f.close()
447 self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
447 self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
448
448
449
449
450 self.status = 1
450 self.status = 1
451
451
452 def __setIdsAndArrays(self):
452 def __setIdsAndArrays(self):
453 self.dataByFrame = self.__setDataByFrame()
453 self.dataByFrame = self.__setDataByFrame()
454 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
454 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
455 self.readRanges()
455 self.readRanges()
456 self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
456 self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
457 self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
457 self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
458 if len(self.index_amisr_buffer) > 0:
458 if len(self.index_amisr_buffer) > 0:
459 self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
459 self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
460
460
461
461
462 def __setNextFile(self,online=False):
462 def __setNextFile(self,online=False):
463
463
464 if not(online):
464 if not(online):
465 newFile = self.__setNextFileOffline()
465 newFile = self.__setNextFileOffline()
466 else:
466 else:
467 newFile = self.__setNextFileOnline()
467 newFile = self.__setNextFileOnline()
468
468
469 if not(newFile):
469 if not(newFile):
470 return 0
470 return 0
471
471
472 self.__readHeader()
472 self.__readHeader()
473
473
474 if self.__firstFile:
474 if self.__firstFile:
475 self.__setIdsAndArrays()
475 self.__setIdsAndArrays()
476 self.__firstFile = False
476 self.__firstFile = False
477
477
478 self.__getBeamCode()
478 self.__getBeamCode()
479 self.readDataBlock()
479 self.readDataBlock()
480
480
481
481
482 def setup(self,path=None,
482 def setup(self,path=None,
483 startDate=None,
483 startDate=None,
484 endDate=None,
484 endDate=None,
485 startTime=datetime.time(0,0,0),
485 startTime=datetime.time(0,0,0),
486 endTime=datetime.time(23,59,59),
486 endTime=datetime.time(23,59,59),
487 walk=True,
487 walk=True,
488 timezone='ut',
488 timezone='ut',
489 all=0,
489 all=0,
490 online=False):
490 online=False):
491
491
492 self.timezone = timezone
492 self.timezone = timezone
493 self.all = all
493 self.all = all
494 self.online = online
494 self.online = online
495 if not(online):
495 if not(online):
496 #Busqueda de archivos offline
496 #Busqueda de archivos offline
497 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 else:
498 else:
499 self.__searchFilesOnline(path, walk)
499 self.searchFilesOnLine(path, walk)
500
500
501 if not(self.filenameList):
501 if not(self.filenameList):
502 print "There is no files into the folder: %s"%(path)
502 print "There is no files into the folder: %s"%(path)
503
503
504 sys.exit(-1)
504 sys.exit(-1)
505
505
506 self.__getExpParameters()
506 self.__getExpParameters()
507
507
508 self.fileIndex = -1
508 self.fileIndex = -1
509
509
510 self.__setNextFile(online)
510 self.__setNextFile(online)
511
511
512 # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
512 # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
513 # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
513 # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
514 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
514 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
515 self.profileIndex = self.profileIndex_offset
515 self.profileIndex = self.profileIndex_offset
516
516
517 def readRanges(self):
517 def readRanges(self):
518 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
518 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
519
519
520 self.rangeFromFile = numpy.reshape(dataset.value,(-1))
520 self.rangeFromFile = numpy.reshape(dataset.value,(-1))
521 return self.rangeFromFile
521 return self.rangeFromFile
522
522
523
523
524 def readRadacTime(self,idrecord, range1, range2):
524 def readRadacTime(self,idrecord, range1, range2):
525 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
525 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
526
526
527 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
527 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
528 #radacTimeByFrame = dataset[idrecord - 1,range1]
528 #radacTimeByFrame = dataset[idrecord - 1,range1]
529 #radacTimeByFrame = dataset[idrecord,range2]
529 #radacTimeByFrame = dataset[idrecord,range2]
530
530
531 return radacTimeByFrame
531 return radacTimeByFrame
532
532
533 def readBeamCode(self, idrecord, range1, range2):
533 def readBeamCode(self, idrecord, range1, range2):
534 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
534 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
535 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
535 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
536 self.beamCodesFromFile = dataset.value
536 self.beamCodesFromFile = dataset.value
537
537
538 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
538 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
539 #beamcodeByFrame[range2] = dataset[idrecord, range2]
539 #beamcodeByFrame[range2] = dataset[idrecord, range2]
540 beamcodeByFrame[range1] = dataset[idrecord, range1]
540 beamcodeByFrame[range1] = dataset[idrecord, range1]
541 beamcodeByFrame[range2] = dataset[idrecord, range2]
541 beamcodeByFrame[range2] = dataset[idrecord, range2]
542
542
543 return beamcodeByFrame
543 return beamcodeByFrame
544
544
545
545
546 def __setDataByFrame(self):
546 def __setDataByFrame(self):
547 ndata = 2 # porque es complejo
547 ndata = 2 # porque es complejo
548 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
548 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
549 return dataByFrame
549 return dataByFrame
550
550
551 def __readDataSet(self):
551 def __readDataSet(self):
552 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
552 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
553 return dataset
553 return dataset
554
554
555 def __setDataBlock(self,):
555 def __setDataBlock(self,):
556 real = self.dataByFrame[:,:,0] #asumo que 0 es real
556 real = self.dataByFrame[:,:,0] #asumo que 0 es real
557 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
557 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
558 datablock = real + imag*1j #armo el complejo
558 datablock = real + imag*1j #armo el complejo
559 return datablock
559 return datablock
560
560
561 def readSamples_version1(self,idrecord):
561 def readSamples_version1(self,idrecord):
562 #estas tres primeras lineas solo se deben ejecutar una vez
562 #estas tres primeras lineas solo se deben ejecutar una vez
563 if self.flagIsNewFile:
563 if self.flagIsNewFile:
564 #reading dataset
564 #reading dataset
565 self.dataset = self.__readDataSet()
565 self.dataset = self.__readDataSet()
566 self.flagIsNewFile = 0
566 self.flagIsNewFile = 0
567
567
568 if idrecord == 0:
568 if idrecord == 0:
569 self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
569 self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
570 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
570 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
571 datablock = self.__setDataBlock()
571 datablock = self.__setDataBlock()
572 if len(self.index_amisr_buffer) > 0:
572 if len(self.index_amisr_buffer) > 0:
573 self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
573 self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
574 self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
574 self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
575
575
576 return datablock
576 return datablock
577 if len(self.index_amisr_buffer) > 0:
577 if len(self.index_amisr_buffer) > 0:
578 self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
578 self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
579 self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
579 self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
580 self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
580 self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
581 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
581 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
582 datablock = self.__setDataBlock()
582 datablock = self.__setDataBlock()
583 if len(self.index_amisr_buffer) > 0:
583 if len(self.index_amisr_buffer) > 0:
584 self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
584 self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
585 self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
585 self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
586
586
587 return datablock
587 return datablock
588
588
589
589
590 def readSamples(self,idrecord):
590 def readSamples(self,idrecord):
591 if self.flagIsNewFile:
591 if self.flagIsNewFile:
592 self.dataByFrame = self.__setDataByFrame()
592 self.dataByFrame = self.__setDataByFrame()
593 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
593 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
594
594
595 #reading ranges
595 #reading ranges
596 self.readRanges()
596 self.readRanges()
597 #reading dataset
597 #reading dataset
598 self.dataset = self.__readDataSet()
598 self.dataset = self.__readDataSet()
599
599
600 self.flagIsNewFile = 0
600 self.flagIsNewFile = 0
601 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
601 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
602 self.dataByFrame = self.dataset[idrecord, :, :, :]
602 self.dataByFrame = self.dataset[idrecord, :, :, :]
603 datablock = self.__setDataBlock()
603 datablock = self.__setDataBlock()
604 return datablock
604 return datablock
605
605
606
606
607 def readDataBlock(self):
607 def readDataBlock(self):
608
608
609 self.datablock = self.readSamples_version1(self.idrecord_count)
609 self.datablock = self.readSamples_version1(self.idrecord_count)
610 #self.datablock = self.readSamples(self.idrecord_count)
610 #self.datablock = self.readSamples(self.idrecord_count)
611 #print 'record:', self.idrecord_count
611 #print 'record:', self.idrecord_count
612
612
613 self.idrecord_count += 1
613 self.idrecord_count += 1
614 self.profileIndex = 0
614 self.profileIndex = 0
615
615
616 if self.idrecord_count >= self.radacHeaderObj.nrecords:
616 if self.idrecord_count >= self.radacHeaderObj.nrecords:
617 self.idrecord_count = 0
617 self.idrecord_count = 0
618 self.flagIsNewFile = 1
618 self.flagIsNewFile = 1
619
619
620 def readNextBlock(self):
620 def readNextBlock(self):
621
621
622 self.readDataBlock()
622 self.readDataBlock()
623
623
624 if self.flagIsNewFile:
624 if self.flagIsNewFile:
625 self.__setNextFile(self.online)
625 self.__setNextFile(self.online)
626 pass
626 pass
627
627
628 def __hasNotDataInBuffer(self):
628 def __hasNotDataInBuffer(self):
629 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
629 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
630 if self.profileIndex >= self.radacHeaderObj.npulses:
630 if self.profileIndex >= self.radacHeaderObj.npulses:
631 return 1
631 return 1
632 return 0
632 return 0
633
633
634 def printUTC(self):
634 def printUTC(self):
635 print self.dataOut.utctime
635 print self.dataOut.utctime
636 print ''
636 print ''
637
637
638 def setObjProperties(self):
638 def setObjProperties(self):
639
639
640 self.dataOut.heightList = self.rangeFromFile/1000.0 #km
640 self.dataOut.heightList = self.rangeFromFile/1000.0 #km
641 self.dataOut.nProfiles = self.radacHeaderObj.npulses
641 self.dataOut.nProfiles = self.radacHeaderObj.npulses
642 self.dataOut.nRecords = self.radacHeaderObj.nrecords
642 self.dataOut.nRecords = self.radacHeaderObj.nrecords
643 self.dataOut.nBeams = self.radacHeaderObj.nbeams
643 self.dataOut.nBeams = self.radacHeaderObj.nbeams
644 self.dataOut.ippSeconds = self.ippSeconds_fromfile
644 self.dataOut.ippSeconds = self.ippSeconds_fromfile
645 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
645 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
646 self.dataOut.frequency = self.frequency_h5file
646 self.dataOut.frequency = self.frequency_h5file
647 self.dataOut.npulseByFrame = self.npulseByFrame
647 self.dataOut.npulseByFrame = self.npulseByFrame
648 self.dataOut.nBaud = None
648 self.dataOut.nBaud = None
649 self.dataOut.nCode = None
649 self.dataOut.nCode = None
650 self.dataOut.code = None
650 self.dataOut.code = None
651
651
652 self.dataOut.beamCodeDict = self.beamCodeDict
652 self.dataOut.beamCodeDict = self.beamCodeDict
653 self.dataOut.beamRangeDict = self.beamRangeDict
653 self.dataOut.beamRangeDict = self.beamRangeDict
654
654
655 if self.timezone == 'lt':
655 if self.timezone == 'lt':
656 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
656 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
657 else:
657 else:
658 self.dataOut.timeZone = 0 #by default time is UTC
658 self.dataOut.timeZone = 0 #by default time is UTC
659
659
660 def getData(self):
660 def getData(self):
661
661
662 if self.flagNoMoreFiles:
662 if self.flagNoMoreFiles:
663 self.dataOut.flagNoData = True
663 self.dataOut.flagNoData = True
664 print 'Process finished'
664 print 'Process finished'
665 return 0
665 return 0
666
666
667 if self.__hasNotDataInBuffer():
667 if self.__hasNotDataInBuffer():
668 self.readNextBlock()
668 self.readNextBlock()
669
669
670
670
671 if self.datablock is None: # setear esta condicion cuando no hayan datos por leers
671 if self.datablock is None: # setear esta condicion cuando no hayan datos por leers
672 self.dataOut.flagNoData = True
672 self.dataOut.flagNoData = True
673 return 0
673 return 0
674
674
675 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
675 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
676
676
677 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
677 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
678 self.dataOut.profileIndex = self.profileIndex
678 self.dataOut.profileIndex = self.profileIndex
679 self.dataOut.flagNoData = False
679 self.dataOut.flagNoData = False
680
680
681 self.profileIndex += 1
681 self.profileIndex += 1
682
682
683 return self.dataOut.data
683 return self.dataOut.data
684
684
685
685
686 def run(self, **kwargs):
686 def run(self, **kwargs):
687 if not(self.isConfig):
687 if not(self.isConfig):
688 self.setup(**kwargs)
688 self.setup(**kwargs)
689 self.setObjProperties()
689 self.setObjProperties()
690 self.isConfig = True
690 self.isConfig = True
691
691
692 self.getData()
692 self.getData()
Show file before | Show file after | Hide comments
@@ -1,1855 +1,1794
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time, datetime
13 import time, datetime
14 import traceback
14 import traceback
15 import zmq
15 import zmq
16
16
17 try:
17 try:
18 from gevent import sleep
18 from gevent import sleep
19 except:
19 except:
20 from time import sleep
20 from time import sleep
21
21
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24
24
25 LOCALTIME = True
25 LOCALTIME = True
26
26
27 def isNumber(cad):
27 def isNumber(cad):
28 """
28 """
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30
30
31 Excepciones:
31 Excepciones:
32 Si un determinado string no puede ser convertido a numero
32 Si un determinado string no puede ser convertido a numero
33 Input:
33 Input:
34 str, string al cual se le analiza para determinar si convertible a un numero o no
34 str, string al cual se le analiza para determinar si convertible a un numero o no
35
35
36 Return:
36 Return:
37 True : si el string es uno numerico
37 True : si el string es uno numerico
38 False : no es un string numerico
38 False : no es un string numerico
39 """
39 """
40 try:
40 try:
41 float( cad )
41 float( cad )
42 return True
42 return True
43 except:
43 except:
44 return False
44 return False
45
45
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 """
47 """
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49
49
50 Inputs:
50 Inputs:
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52
52
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 segundos contados desde 01/01/1970.
54 segundos contados desde 01/01/1970.
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 segundos contados desde 01/01/1970.
56 segundos contados desde 01/01/1970.
57
57
58 Return:
58 Return:
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 fecha especificado, de lo contrario retorna False.
60 fecha especificado, de lo contrario retorna False.
61
61
62 Excepciones:
62 Excepciones:
63 Si el archivo no existe o no puede ser abierto
63 Si el archivo no existe o no puede ser abierto
64 Si la cabecera no puede ser leida.
64 Si la cabecera no puede ser leida.
65
65
66 """
66 """
67 basicHeaderObj = BasicHeader(LOCALTIME)
67 basicHeaderObj = BasicHeader(LOCALTIME)
68
68
69 try:
69 try:
70 fp = open(filename,'rb')
70 fp = open(filename,'rb')
71 except IOError:
71 except IOError:
72 print "The file %s can't be opened" %(filename)
72 print "The file %s can't be opened" %(filename)
73 return 0
73 return 0
74
74
75 sts = basicHeaderObj.read(fp)
75 sts = basicHeaderObj.read(fp)
76 fp.close()
76 fp.close()
77
77
78 if not(sts):
78 if not(sts):
79 print "Skipping the file %s because it has not a valid header" %(filename)
79 print "Skipping the file %s because it has not a valid header" %(filename)
80 return 0
80 return 0
81
81
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 return 0
83 return 0
84
84
85 return 1
85 return 1
86
86
87 def isTimeInRange(thisTime, startTime, endTime):
87 def isTimeInRange(thisTime, startTime, endTime):
88
88
89 if endTime >= startTime:
89 if endTime >= startTime:
90 if (thisTime < startTime) or (thisTime > endTime):
90 if (thisTime < startTime) or (thisTime > endTime):
91 return 0
91 return 0
92
92
93 return 1
93 return 1
94 else:
94 else:
95 if (thisTime < startTime) and (thisTime > endTime):
95 if (thisTime < startTime) and (thisTime > endTime):
96 return 0
96 return 0
97
97
98 return 1
98 return 1
99
99
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 """
101 """
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103
103
104 Inputs:
104 Inputs:
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106
106
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108
108
109 endDate : fecha final del rango seleccionado en formato datetime.date
109 endDate : fecha final del rango seleccionado en formato datetime.date
110
110
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112
112
113 endTime : tiempo final del rango seleccionado en formato datetime.time
113 endTime : tiempo final del rango seleccionado en formato datetime.time
114
114
115 Return:
115 Return:
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 fecha especificado, de lo contrario retorna False.
117 fecha especificado, de lo contrario retorna False.
118
118
119 Excepciones:
119 Excepciones:
120 Si el archivo no existe o no puede ser abierto
120 Si el archivo no existe o no puede ser abierto
121 Si la cabecera no puede ser leida.
121 Si la cabecera no puede ser leida.
122
122
123 """
123 """
124
124
125
125
126 try:
126 try:
127 fp = open(filename,'rb')
127 fp = open(filename,'rb')
128 except IOError:
128 except IOError:
129 print "The file %s can't be opened" %(filename)
129 print "The file %s can't be opened" %(filename)
130 return None
130 return None
131
131
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 systemHeaderObj = SystemHeader()
133 systemHeaderObj = SystemHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
135 processingHeaderObj = ProcessingHeader()
135 processingHeaderObj = ProcessingHeader()
136
136
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138
138
139 sts = firstBasicHeaderObj.read(fp)
139 sts = firstBasicHeaderObj.read(fp)
140
140
141 if not(sts):
141 if not(sts):
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 return None
143 return None
144
144
145 if not systemHeaderObj.read(fp):
145 if not systemHeaderObj.read(fp):
146 return None
146 return None
147
147
148 if not radarControllerHeaderObj.read(fp):
148 if not radarControllerHeaderObj.read(fp):
149 return None
149 return None
150
150
151 if not processingHeaderObj.read(fp):
151 if not processingHeaderObj.read(fp):
152 return None
152 return None
153
153
154 filesize = os.path.getsize(filename)
154 filesize = os.path.getsize(filename)
155
155
156 offset = processingHeaderObj.blockSize + 24 #header size
156 offset = processingHeaderObj.blockSize + 24 #header size
157
157
158 if filesize <= offset:
158 if filesize <= offset:
159 print "[Reading] %s: This file has not enough data" %filename
159 print "[Reading] %s: This file has not enough data" %filename
160 return None
160 return None
161
161
162 fp.seek(-offset, 2)
162 fp.seek(-offset, 2)
163
163
164 sts = lastBasicHeaderObj.read(fp)
164 sts = lastBasicHeaderObj.read(fp)
165
165
166 fp.close()
166 fp.close()
167
167
168 thisDatetime = lastBasicHeaderObj.datatime
168 thisDatetime = lastBasicHeaderObj.datatime
169 thisTime_last_block = thisDatetime.time()
169 thisTime_last_block = thisDatetime.time()
170
170
171 thisDatetime = firstBasicHeaderObj.datatime
171 thisDatetime = firstBasicHeaderObj.datatime
172 thisDate = thisDatetime.date()
172 thisDate = thisDatetime.date()
173 thisTime_first_block = thisDatetime.time()
173 thisTime_first_block = thisDatetime.time()
174
174
175 #General case
175 #General case
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 #-----------o----------------------------o-----------
177 #-----------o----------------------------o-----------
178 # startTime endTime
178 # startTime endTime
179
179
180 if endTime >= startTime:
180 if endTime >= startTime:
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 return None
182 return None
183
183
184 return thisDatetime
184 return thisDatetime
185
185
186 #If endTime < startTime then endTime belongs to the next day
186 #If endTime < startTime then endTime belongs to the next day
187
187
188
188
189 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 #-----------o----------------------------o-----------
190 #-----------o----------------------------o-----------
191 # endTime startTime
191 # endTime startTime
192
192
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 return None
194 return None
195
195
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 return None
197 return None
198
198
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 return None
200 return None
201
201
202 return thisDatetime
202 return thisDatetime
203
203
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 """
205 """
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207
207
208 Inputs:
208 Inputs:
209 folder : nombre completo del directorio.
209 folder : nombre completo del directorio.
210 Su formato deberia ser "/path_root/?YYYYDDD"
210 Su formato deberia ser "/path_root/?YYYYDDD"
211
211
212 siendo:
212 siendo:
213 YYYY : Anio (ejemplo 2015)
213 YYYY : Anio (ejemplo 2015)
214 DDD : Dia del anio (ejemplo 305)
214 DDD : Dia del anio (ejemplo 305)
215
215
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217
217
218 endDate : fecha final del rango seleccionado en formato datetime.date
218 endDate : fecha final del rango seleccionado en formato datetime.date
219
219
220 Return:
220 Return:
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 fecha especificado, de lo contrario retorna False.
222 fecha especificado, de lo contrario retorna False.
223 Excepciones:
223 Excepciones:
224 Si el directorio no tiene el formato adecuado
224 Si el directorio no tiene el formato adecuado
225 """
225 """
226
226
227 basename = os.path.basename(folder)
227 basename = os.path.basename(folder)
228
228
229 if not isRadarFolder(basename):
229 if not isRadarFolder(basename):
230 print "The folder %s has not the rigth format" %folder
230 print "The folder %s has not the rigth format" %folder
231 return 0
231 return 0
232
232
233 if startDate and endDate:
233 if startDate and endDate:
234 thisDate = getDateFromRadarFolder(basename)
234 thisDate = getDateFromRadarFolder(basename)
235
235
236 if thisDate < startDate:
236 if thisDate < startDate:
237 return 0
237 return 0
238
238
239 if thisDate > endDate:
239 if thisDate > endDate:
240 return 0
240 return 0
241
241
242 return 1
242 return 1
243
243
244 def isFileInDateRange(filename, startDate=None, endDate=None):
244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 """
245 """
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247
247
248 Inputs:
248 Inputs:
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250
250
251 Su formato deberia ser "?YYYYDDDsss"
251 Su formato deberia ser "?YYYYDDDsss"
252
252
253 siendo:
253 siendo:
254 YYYY : Anio (ejemplo 2015)
254 YYYY : Anio (ejemplo 2015)
255 DDD : Dia del anio (ejemplo 305)
255 DDD : Dia del anio (ejemplo 305)
256 sss : set
256 sss : set
257
257
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259
259
260 endDate : fecha final del rango seleccionado en formato datetime.date
260 endDate : fecha final del rango seleccionado en formato datetime.date
261
261
262 Return:
262 Return:
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 fecha especificado, de lo contrario retorna False.
264 fecha especificado, de lo contrario retorna False.
265 Excepciones:
265 Excepciones:
266 Si el archivo no tiene el formato adecuado
266 Si el archivo no tiene el formato adecuado
267 """
267 """
268
268
269 basename = os.path.basename(filename)
269 basename = os.path.basename(filename)
270
270
271 if not isRadarFile(basename):
271 if not isRadarFile(basename):
272 print "The filename %s has not the rigth format" %filename
272 print "The filename %s has not the rigth format" %filename
273 return 0
273 return 0
274
274
275 if startDate and endDate:
275 if startDate and endDate:
276 thisDate = getDateFromRadarFile(basename)
276 thisDate = getDateFromRadarFile(basename)
277
277
278 if thisDate < startDate:
278 if thisDate < startDate:
279 return 0
279 return 0
280
280
281 if thisDate > endDate:
281 if thisDate > endDate:
282 return 0
282 return 0
283
283
284 return 1
284 return 1
285
285
286 def getFileFromSet(path, ext, set):
286 def getFileFromSet(path, ext, set):
287 validFilelist = []
287 validFilelist = []
288 fileList = os.listdir(path)
288 fileList = os.listdir(path)
289
289
290 # 0 1234 567 89A BCDE
290 # 0 1234 567 89A BCDE
291 # H YYYY DDD SSS .ext
291 # H YYYY DDD SSS .ext
292
292
293 for thisFile in fileList:
293 for thisFile in fileList:
294 try:
294 try:
295 year = int(thisFile[1:5])
295 year = int(thisFile[1:5])
296 doy = int(thisFile[5:8])
296 doy = int(thisFile[5:8])
297 except:
297 except:
298 continue
298 continue
299
299
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 continue
301 continue
302
302
303 validFilelist.append(thisFile)
303 validFilelist.append(thisFile)
304
304
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306
306
307 if len(myfile)!= 0:
307 if len(myfile)!= 0:
308 return myfile[0]
308 return myfile[0]
309 else:
309 else:
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 print 'the filename %s does not exist'%filename
311 print 'the filename %s does not exist'%filename
312 print '...going to the last file: '
312 print '...going to the last file: '
313
313
314 if validFilelist:
314 if validFilelist:
315 validFilelist = sorted( validFilelist, key=str.lower )
315 validFilelist = sorted( validFilelist, key=str.lower )
316 return validFilelist[-1]
316 return validFilelist[-1]
317
317
318 return None
318 return None
319
319
320 def getlastFileFromPath(path, ext):
320 def getlastFileFromPath(path, ext):
321 """
321 """
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324
324
325 Input:
325 Input:
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 ext : extension de los files contenidos en una carpeta
327 ext : extension de los files contenidos en una carpeta
328
328
329 Return:
329 Return:
330 El ultimo file de una determinada carpeta, no se considera el path.
330 El ultimo file de una determinada carpeta, no se considera el path.
331 """
331 """
332 validFilelist = []
332 validFilelist = []
333 fileList = os.listdir(path)
333 fileList = os.listdir(path)
334
334
335 # 0 1234 567 89A BCDE
335 # 0 1234 567 89A BCDE
336 # H YYYY DDD SSS .ext
336 # H YYYY DDD SSS .ext
337
337
338 for thisFile in fileList:
338 for thisFile in fileList:
339
339
340 year = thisFile[1:5]
340 year = thisFile[1:5]
341 if not isNumber(year):
341 if not isNumber(year):
342 continue
342 continue
343
343
344 doy = thisFile[5:8]
344 doy = thisFile[5:8]
345 if not isNumber(doy):
345 if not isNumber(doy):
346 continue
346 continue
347
347
348 year = int(year)
348 year = int(year)
349 doy = int(doy)
349 doy = int(doy)
350
350
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 continue
352 continue
353
353
354 validFilelist.append(thisFile)
354 validFilelist.append(thisFile)
355
355
356 if validFilelist:
356 if validFilelist:
357 validFilelist = sorted( validFilelist, key=str.lower )
357 validFilelist = sorted( validFilelist, key=str.lower )
358 return validFilelist[-1]
358 return validFilelist[-1]
359
359
360 return None
360 return None
361
361
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 """
363 """
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 el path exacto de un determinado file.
366 el path exacto de un determinado file.
367
367
368 Example :
368 Example :
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370
370
371 Entonces la funcion prueba con las siguientes combinaciones
371 Entonces la funcion prueba con las siguientes combinaciones
372 .../.../y2009307367.ext
372 .../.../y2009307367.ext
373 .../.../Y2009307367.ext
373 .../.../Y2009307367.ext
374 .../.../x2009307/y2009307367.ext
374 .../.../x2009307/y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
376 .../.../X2009307/y2009307367.ext
376 .../.../X2009307/y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379
379
380 Return:
380 Return:
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 para el filename
383 para el filename
384 """
384 """
385 fullfilename = None
385 fullfilename = None
386 find_flag = False
386 find_flag = False
387 filename = None
387 filename = None
388
388
389 prefixDirList = [None,'d','D']
389 prefixDirList = [None,'d','D']
390 if ext.lower() == ".r": #voltage
390 if ext.lower() == ".r": #voltage
391 prefixFileList = ['d','D']
391 prefixFileList = ['d','D']
392 elif ext.lower() == ".pdata": #spectra
392 elif ext.lower() == ".pdata": #spectra
393 prefixFileList = ['p','P']
393 prefixFileList = ['p','P']
394 else:
394 else:
395 return None, filename
395 return None, filename
396
396
397 #barrido por las combinaciones posibles
397 #barrido por las combinaciones posibles
398 for prefixDir in prefixDirList:
398 for prefixDir in prefixDirList:
399 thispath = path
399 thispath = path
400 if prefixDir != None:
400 if prefixDir != None:
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 if foldercounter == 0:
402 if foldercounter == 0:
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 else:
404 else:
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409
409
410 if os.path.exists( fullfilename ): #verifico que exista
410 if os.path.exists( fullfilename ): #verifico que exista
411 find_flag = True
411 find_flag = True
412 break
412 break
413 if find_flag:
413 if find_flag:
414 break
414 break
415
415
416 if not(find_flag):
416 if not(find_flag):
417 return None, filename
417 return None, filename
418
418
419 return fullfilename, filename
419 return fullfilename, filename
420
420
421 def isRadarFolder(folder):
421 def isRadarFolder(folder):
422 try:
422 try:
423 year = int(folder[1:5])
423 year = int(folder[1:5])
424 doy = int(folder[5:8])
424 doy = int(folder[5:8])
425 except:
425 except:
426 return 0
426 return 0
427
427
428 return 1
428 return 1
429
429
430 def isRadarFile(file):
430 def isRadarFile(file):
431 try:
431 try:
432 year = int(file[1:5])
432 year = int(file[1:5])
433 doy = int(file[5:8])
433 doy = int(file[5:8])
434 set = int(file[8:11])
434 set = int(file[8:11])
435 except:
435 except:
436 return 0
436 return 0
437
437
438 return 1
438 return 1
439
439
440 def getDateFromRadarFile(file):
440 def getDateFromRadarFile(file):
441 try:
441 try:
442 year = int(file[1:5])
442 year = int(file[1:5])
443 doy = int(file[5:8])
443 doy = int(file[5:8])
444 set = int(file[8:11])
444 set = int(file[8:11])
445 except:
445 except:
446 return None
446 return None
447
447
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 return thisDate
449 return thisDate
450
450
451 def getDateFromRadarFolder(folder):
451 def getDateFromRadarFolder(folder):
452 try:
452 try:
453 year = int(folder[1:5])
453 year = int(folder[1:5])
454 doy = int(folder[5:8])
454 doy = int(folder[5:8])
455 except:
455 except:
456 return None
456 return None
457
457
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 return thisDate
459 return thisDate
460
460
461 class JRODataIO:
461 class JRODataIO:
462
462
463 c = 3E8
463 c = 3E8
464
464
465 isConfig = False
465 isConfig = False
466
466
467 basicHeaderObj = None
467 basicHeaderObj = None
468
468
469 systemHeaderObj = None
469 systemHeaderObj = None
470
470
471 radarControllerHeaderObj = None
471 radarControllerHeaderObj = None
472
472
473 processingHeaderObj = None
473 processingHeaderObj = None
474
474
475 dtype = None
475 dtype = None
476
476
477 pathList = []
477 pathList = []
478
478
479 filenameList = []
479 filenameList = []
480
480
481 filename = None
481 filename = None
482
482
483 ext = None
483 ext = None
484
484
485 flagIsNewFile = 1
485 flagIsNewFile = 1
486
486
487 flagDiscontinuousBlock = 0
487 flagDiscontinuousBlock = 0
488
488
489 flagIsNewBlock = 0
489 flagIsNewBlock = 0
490
490
491 fp = None
491 fp = None
492
492
493 firstHeaderSize = 0
493 firstHeaderSize = 0
494
494
495 basicHeaderSize = 24
495 basicHeaderSize = 24
496
496
497 versionFile = 1103
497 versionFile = 1103
498
498
499 fileSize = None
499 fileSize = None
500
500
501 # ippSeconds = None
501 # ippSeconds = None
502
502
503 fileSizeByHeader = None
503 fileSizeByHeader = None
504
504
505 fileIndex = None
505 fileIndex = None
506
506
507 profileIndex = None
507 profileIndex = None
508
508
509 blockIndex = None
509 blockIndex = None
510
510
511 nTotalBlocks = None
511 nTotalBlocks = None
512
512
513 maxTimeStep = 30
513 maxTimeStep = 30
514
514
515 lastUTTime = None
515 lastUTTime = None
516
516
517 datablock = None
517 datablock = None
518
518
519 dataOut = None
519 dataOut = None
520
520
521 blocksize = None
521 blocksize = None
522
522
523 getByBlock = False
523 getByBlock = False
524
524
525 def __init__(self):
525 def __init__(self):
526
526
527 raise NotImplementedError
527 raise NotImplementedError
528
528
529 def run(self):
529 def run(self):
530
530
531 raise NotImplementedError
531 raise NotImplementedError
532
532
533 def getDtypeWidth(self):
533 def getDtypeWidth(self):
534
534
535 dtype_index = get_dtype_index(self.dtype)
535 dtype_index = get_dtype_index(self.dtype)
536 dtype_width = get_dtype_width(dtype_index)
536 dtype_width = get_dtype_width(dtype_index)
537
537
538 return dtype_width
538 return dtype_width
539
539
540 def getAllowedArgs(self):
540 def getAllowedArgs(self):
541 return inspect.getargspec(self.run).args
541 return inspect.getargspec(self.run).args
542
542
543 class JRODataReader(JRODataIO):
543 class JRODataReader(JRODataIO):
544
544
545 firstTime = True
546 online = 0
545 online = 0
547
546
548 realtime = 0
547 realtime = 0
549
548
550 nReadBlocks = 0
549 nReadBlocks = 0
551
550
552 delay = 10 #number of seconds waiting a new file
551 delay = 10 #number of seconds waiting a new file
553
552
554 nTries = 3 #quantity tries
553 nTries = 3 #quantity tries
555
554
556 nFiles = 3 #number of files for searching
555 nFiles = 3 #number of files for searching
557
556
558 path = None
557 path = None
559
558
560 foldercounter = 0
559 foldercounter = 0
561
560
562 flagNoMoreFiles = 0
561 flagNoMoreFiles = 0
563
562
564 datetimeList = []
563 datetimeList = []
565
564
566 __isFirstTimeOnline = 1
565 __isFirstTimeOnline = 1
567
566
568 __printInfo = True
567 __printInfo = True
569
568
570 profileIndex = None
569 profileIndex = None
571
570
572 nTxs = 1
571 nTxs = 1
573
572
574 txIndex = None
573 txIndex = None
575
574
576 #Added--------------------
575 #Added--------------------
577
576
578 selBlocksize = None
577 selBlocksize = None
579
578
580 selBlocktime = None
579 selBlocktime = None
581
580
582 onlineWithDate = False
583 def __init__(self):
581 def __init__(self):
584
582
585 """
583 """
586 This class is used to find data files
584 This class is used to find data files
587
585
588 Example:
586 Example:
589 reader = JRODataReader()
587 reader = JRODataReader()
590 fileList = reader.findDataFiles()
588 fileList = reader.findDataFiles()
591
589
592 """
590 """
593 pass
591 pass
594
592
595
593
596 def createObjByDefault(self):
594 def createObjByDefault(self):
597 """
595 """
598
596
599 """
597 """
600 raise NotImplementedError
598 raise NotImplementedError
601
599
602 def getBlockDimension(self):
600 def getBlockDimension(self):
603
601
604 raise NotImplementedError
602 raise NotImplementedError
605
603
606 def __searchFilesOffLine(self,
604 def searchFilesOffLine(self,
607 path,
605 path,
608 startDate=None,
606 startDate=None,
609 endDate=None,
607 endDate=None,
610 startTime=datetime.time(0,0,0),
608 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
609 endTime=datetime.time(23,59,59),
612 set=None,
610 set=None,
613 expLabel='',
611 expLabel='',
614 ext='.r',
612 ext='.r',
615 queue=None,
616 cursor=None,
613 cursor=None,
617 skip=None,
614 skip=None,
618 walk=True):
615 walk=True):
616
619 self.filenameList = []
617 self.filenameList = []
620 self.datetimeList = []
618 self.datetimeList = []
621
619
622 pathList = []
620 pathList = []
623
621
624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625
623
626 if dateList == []:
624 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
625 return [], []
628 return None, None
629
626
630 if len(dateList) > 1:
627 if len(dateList) > 1:
631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 else:
629 else:
633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
630 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634
631
635 filenameList = []
632 filenameList = []
636 datetimeList = []
633 datetimeList = []
637
634
638 for thisPath in pathList:
635 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
640
636
641 fileList = glob.glob1(thisPath, "*%s" %ext)
637 fileList = glob.glob1(thisPath, "*%s" %ext)
642 fileList.sort()
638 fileList.sort()
643
639
644 skippedFileList = []
640 skippedFileList = []
645
641
646 if cursor is not None and skip is not None:
642 if cursor is not None and skip is not None:
647 # if cursor*skip > len(fileList):
643
648 if skip == 0:
644 if skip == 0:
649 if queue is not None:
650 queue.put(len(fileList))
651 skippedFileList = []
645 skippedFileList = []
652 else:
646 else:
653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654
648
655 else:
649 else:
656 skippedFileList = fileList
650 skippedFileList = fileList
657
651
658 for file in skippedFileList:
652 for file in skippedFileList:
659
653
660 filename = os.path.join(thisPath,file)
654 filename = os.path.join(thisPath,file)
661
655
662 if not isFileInDateRange(filename, startDate, endDate):
656 if not isFileInDateRange(filename, startDate, endDate):
663 continue
657 continue
664
658
665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
659 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666
660
667 if not(thisDatetime):
661 if not(thisDatetime):
668 continue
662 continue
669
663
670 filenameList.append(filename)
664 filenameList.append(filename)
671 datetimeList.append(thisDatetime)
665 datetimeList.append(thisDatetime)
672
666
673 if not(filenameList):
667 if not(filenameList):
674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
669 return [], []
676
670
677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 print
672 print
679
673
680 for i in range(len(filenameList)):
674 # for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682
676
683 self.filenameList = filenameList
677 self.filenameList = filenameList
684 self.datetimeList = datetimeList
678 self.datetimeList = datetimeList
679
685 return pathList, filenameList
680 return pathList, filenameList
686
681
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
688
683
689 """
684 """
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
686 devuelve el archivo encontrado ademas de otros datos.
692
687
693 Input:
688 Input:
694 path : carpeta donde estan contenidos los files que contiene data
689 path : carpeta donde estan contenidos los files que contiene data
695
690
696 expLabel : Nombre del subexperimento (subfolder)
691 expLabel : Nombre del subexperimento (subfolder)
697
692
698 ext : extension de los files
693 ext : extension de los files
699
694
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
701
696
702 Return:
697 Return:
703 directory : eL directorio donde esta el file encontrado
698 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
699 filename : el ultimo file de una determinada carpeta
705 year : el anho
700 year : el anho
706 doy : el numero de dia del anho
701 doy : el numero de dia del anho
707 set : el set del archivo
702 set : el set del archivo
708
703
709
704
710 """
705 """
711 pathList = None
712 filenameList = None
713 if not os.path.isdir(path):
706 if not os.path.isdir(path):
714 return None, None, None, None, None, None
707 return None, None, None, None, None, None
715
708
716 dirList = []
709 dirList = []
717
710
718 if not walk:
711 if not walk:
719 fullpath = path
712 fullpath = path
720 foldercounter = 0
713 foldercounter = 0
721 else:
714 else:
722 # Filtra solo los directorios
715 #Filtra solo los directorios
723 for thisPath in os.listdir(path):
716 for thisPath in os.listdir(path):
724 if not os.path.isdir(os.path.join(path,thisPath)):
717 if not os.path.isdir(os.path.join(path,thisPath)):
725 continue
718 continue
726 if not isRadarFolder(thisPath):
719 if not isRadarFolder(thisPath):
727 continue
720 continue
728
721
729 dirList.append(thisPath)
722 dirList.append(thisPath)
730
723
731 if not(dirList):
724 if not(dirList):
732 return None, None, None, None, None, None
725 return None, None, None, None, None, None
733
726
734 dirList = sorted( dirList, key=str.lower )
727 dirList = sorted( dirList, key=str.lower )
735
728
736 doypath = dirList[-1]
729 doypath = dirList[-1]
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
730 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 fullpath = os.path.join(path, doypath, expLabel)
731 fullpath = os.path.join(path, doypath, expLabel)
739
732
740
733
741 print "[Reading] %s folder was found: " %(fullpath )
734 print "[Reading] %s folder was found: " %(fullpath )
742
735
743 if set == None:
736 if set == None:
744 filename = getlastFileFromPath(fullpath, ext)
737 filename = getlastFileFromPath(fullpath, ext)
745 else:
738 else:
746 filename = getFileFromSet(fullpath, ext, set)
739 filename = getFileFromSet(fullpath, ext, set)
747
740
748 if not(filename):
741 if not(filename):
749 return None, None, None, None, None, None
742 return None, None, None, None, None, None
750
743
751 print "[Reading] %s file was found" %(filename)
744 print "[Reading] %s file was found" %(filename)
752
745
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
746 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 return None, None, None, None, None, None
747 return None, None, None, None, None, None
755
748
756 year = int( filename[1:5] )
749 year = int( filename[1:5] )
757 doy = int( filename[5:8] )
750 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
751 set = int( filename[8:11] )
759
752
760 return fullpath, foldercounter, filename, year, doy, set
753 return fullpath, foldercounter, filename, year, doy, set
761
754
762 def __setNextFileOffline(self):
755 def __setNextFileOffline(self):
763
756
764 idFile = self.fileIndex
757 idFile = self.fileIndex
765
758
766 while (True):
759 while (True):
767 idFile += 1
760 idFile += 1
768 if not(idFile < len(self.filenameList)):
761 if not(idFile < len(self.filenameList)):
769 self.flagNoMoreFiles = 1
762 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
763 # print "[Reading] No more Files"
771 return 0
764 return 0
772
765
773 filename = self.filenameList[idFile]
766 filename = self.filenameList[idFile]
774
767
775 if not(self.__verifyFile(filename)):
768 if not(self.__verifyFile(filename)):
776 continue
769 continue
777
770
778 fileSize = os.path.getsize(filename)
771 fileSize = os.path.getsize(filename)
779 fp = open(filename,'rb')
772 fp = open(filename,'rb')
780 break
773 break
781
774
782 self.flagIsNewFile = 1
775 self.flagIsNewFile = 1
783 self.fileIndex = idFile
776 self.fileIndex = idFile
784 self.filename = filename
777 self.filename = filename
785 self.fileSize = fileSize
778 self.fileSize = fileSize
786 self.fp = fp
779 self.fp = fp
787
780
788 #print "[Reading] Setting the file: %s"%self.filename
781 # print "[Reading] Setting the file: %s"%self.filename
789
782
790 return 1
783 return 1
791
784
792 def __setNextFileOnline(self):
785 def __setNextFileOnline(self):
793 """
786 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
787 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
788 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
789 siguientes.
797
790
798 Affected:
791 Affected:
799 self.flagIsNewFile
792 self.flagIsNewFile
800 self.filename
793 self.filename
801 self.fileSize
794 self.fileSize
802 self.fp
795 self.fp
803 self.set
796 self.set
804 self.flagNoMoreFiles
797 self.flagNoMoreFiles
805
798
806 Return:
799 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
800 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
801 1 : si el file fue abierto con exito y esta listo a ser leido
809
802
810 Excepciones:
803 Excepciones:
811 Si un determinado file no puede ser abierto
804 Si un determinado file no puede ser abierto
812 """
805 """
813
814 nFiles = 0
806 nFiles = 0
815 fileOk_flag = False
807 fileOk_flag = False
816 firstTime_flag = True
808 firstTime_flag = True
817
809
818 self.set += 1
810 self.set += 1
819
811
820 if self.set > 999:
812 if self.set > 999:
821 self.set = 0
813 self.set = 0
822 self.foldercounter += 1
814 self.foldercounter += 1
823
815
824 #busca el 1er file disponible
816 #busca el 1er file disponible
825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
817 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
826 if fullfilename:
818 if fullfilename:
827 if self.__verifyFile(fullfilename, False):
819 if self.__verifyFile(fullfilename, False):
828 fileOk_flag = True
820 fileOk_flag = True
829
821
830 #si no encuentra un file entonces espera y vuelve a buscar
822 #si no encuentra un file entonces espera y vuelve a buscar
831 if not(fileOk_flag):
823 if not(fileOk_flag):
832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
824 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
833
825
834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
826 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
835 tries = self.nTries
827 tries = self.nTries
836 else:
828 else:
837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
829 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
838
830
839 for nTries in range( tries ):
831 for nTries in range( tries ):
840 if firstTime_flag:
832 if firstTime_flag:
841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
833 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
842 sleep( self.delay )
834 sleep( self.delay )
843 else:
835 else:
844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
836 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
845
837
846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
838 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
847 if fullfilename:
839 if fullfilename:
848 if self.__verifyFile(fullfilename):
840 if self.__verifyFile(fullfilename):
849 fileOk_flag = True
841 fileOk_flag = True
850 break
842 break
851
843
852 if fileOk_flag:
844 if fileOk_flag:
853 break
845 break
854
846
855 firstTime_flag = False
847 firstTime_flag = False
856
848
857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
849 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
858 self.set += 1
850 self.set += 1
859
851
860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
852 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
861 self.set = 0
853 self.set = 0
862 self.doy += 1
854 self.doy += 1
863 self.foldercounter = 0
855 self.foldercounter = 0
864
856
865 if fileOk_flag:
857 if fileOk_flag:
866 self.fileSize = os.path.getsize( fullfilename )
858 self.fileSize = os.path.getsize( fullfilename )
867 self.filename = fullfilename
859 self.filename = fullfilename
868 self.flagIsNewFile = 1
860 self.flagIsNewFile = 1
869 if self.fp != None: self.fp.close()
861 if self.fp != None: self.fp.close()
870 self.fp = open(fullfilename, 'rb')
862 self.fp = open(fullfilename, 'rb')
871 self.flagNoMoreFiles = 0
863 self.flagNoMoreFiles = 0
872 # print '[Reading] Setting the file: %s' % fullfilename
864 # print '[Reading] Setting the file: %s' % fullfilename
873 else:
865 else:
874 self.fileSize = 0
866 self.fileSize = 0
875 self.filename = None
867 self.filename = None
876 self.flagIsNewFile = 0
868 self.flagIsNewFile = 0
877 self.fp = None
869 self.fp = None
878 self.flagNoMoreFiles = 1
870 self.flagNoMoreFiles = 1
879 # print '[Reading] No more files to read'
871 # print '[Reading] No more files to read'
880
872
881 return fileOk_flag
873 return fileOk_flag
882
874
883 def setNextFile(self):
875 def setNextFile(self):
884 if self.fp != None:
876 if self.fp != None:
885 self.fp.close()
877 self.fp.close()
878
886 if self.online:
879 if self.online:
887 newFile = self.__setNextFileOnline()
880 newFile = self.__setNextFileOnline()
888 else:
881 else:
889 newFile = self.__setNextFileOffline()
882 newFile = self.__setNextFileOffline()
883
890 if not(newFile):
884 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
914 print '[Reading] No more files to read'
885 print '[Reading] No more files to read'
915 return 0
886 return 0
916
887
917 if self.verbose:
888 if self.verbose:
918 print '[Reading] Setting the file: %s' % self.filename
889 print '[Reading] Setting the file: %s' % self.filename
919
890
920 self.__readFirstHeader()
891 self.__readFirstHeader()
921 self.nReadBlocks = 0
892 self.nReadBlocks = 0
922 return 1
893 return 1
923
894
924 def __waitNewBlock(self):
895 def __waitNewBlock(self):
925 """
896 """
926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
897 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
927
898
928 Si el modo de lectura es OffLine siempre retorn 0
899 Si el modo de lectura es OffLine siempre retorn 0
929 """
900 """
930 if not self.online:
901 if not self.online:
931 return 0
902 return 0
932
903
933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
904 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
934 return 0
905 return 0
935
906
936 currentPointer = self.fp.tell()
907 currentPointer = self.fp.tell()
937
908
938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
909 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939
910
940 for nTries in range( self.nTries ):
911 for nTries in range( self.nTries ):
941
912
942 self.fp.close()
913 self.fp.close()
943 self.fp = open( self.filename, 'rb' )
914 self.fp = open( self.filename, 'rb' )
944 self.fp.seek( currentPointer )
915 self.fp.seek( currentPointer )
945
916
946 self.fileSize = os.path.getsize( self.filename )
917 self.fileSize = os.path.getsize( self.filename )
947 currentSize = self.fileSize - currentPointer
918 currentSize = self.fileSize - currentPointer
948
919
949 if ( currentSize >= neededSize ):
920 if ( currentSize >= neededSize ):
950 self.basicHeaderObj.read(self.fp)
921 self.basicHeaderObj.read(self.fp)
951 return 1
922 return 1
952
923
953 if self.fileSize == self.fileSizeByHeader:
924 if self.fileSize == self.fileSizeByHeader:
954 # self.flagEoF = True
925 # self.flagEoF = True
955 return 0
926 return 0
956
927
957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
928 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 sleep( self.delay )
929 sleep( self.delay )
959
930
960
931
961 return 0
932 return 0
962
933
963 def waitDataBlock(self,pointer_location):
934 def waitDataBlock(self,pointer_location):
964
935
965 currentPointer = pointer_location
936 currentPointer = pointer_location
966
937
967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
938 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
968
939
969 for nTries in range( self.nTries ):
940 for nTries in range( self.nTries ):
970 self.fp.close()
941 self.fp.close()
971 self.fp = open( self.filename, 'rb' )
942 self.fp = open( self.filename, 'rb' )
972 self.fp.seek( currentPointer )
943 self.fp.seek( currentPointer )
973
944
974 self.fileSize = os.path.getsize( self.filename )
945 self.fileSize = os.path.getsize( self.filename )
975 currentSize = self.fileSize - currentPointer
946 currentSize = self.fileSize - currentPointer
976
947
977 if ( currentSize >= neededSize ):
948 if ( currentSize >= neededSize ):
978 return 1
949 return 1
979
950
980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
951 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
981 sleep( self.delay )
952 sleep( self.delay )
982
953
983 return 0
954 return 0
984
955
985 def __jumpToLastBlock(self):
956 def __jumpToLastBlock(self):
986
957
987 if not(self.__isFirstTimeOnline):
958 if not(self.__isFirstTimeOnline):
988 return
959 return
989
960
990 csize = self.fileSize - self.fp.tell()
961 csize = self.fileSize - self.fp.tell()
991 blocksize = self.processingHeaderObj.blockSize
962 blocksize = self.processingHeaderObj.blockSize
992
963
993 #salta el primer bloque de datos
964 #salta el primer bloque de datos
994 if csize > self.processingHeaderObj.blockSize:
965 if csize > self.processingHeaderObj.blockSize:
995 self.fp.seek(self.fp.tell() + blocksize)
966 self.fp.seek(self.fp.tell() + blocksize)
996 else:
967 else:
997 return
968 return
998
969
999 csize = self.fileSize - self.fp.tell()
970 csize = self.fileSize - self.fp.tell()
1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
971 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 while True:
972 while True:
1002
973
1003 if self.fp.tell()<self.fileSize:
974 if self.fp.tell()<self.fileSize:
1004 self.fp.seek(self.fp.tell() + neededsize)
975 self.fp.seek(self.fp.tell() + neededsize)
1005 else:
976 else:
1006 self.fp.seek(self.fp.tell() - neededsize)
977 self.fp.seek(self.fp.tell() - neededsize)
1007 break
978 break
1008
979
1009 # csize = self.fileSize - self.fp.tell()
980 # csize = self.fileSize - self.fp.tell()
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
981 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 # factor = int(csize/neededsize)
982 # factor = int(csize/neededsize)
1012 # if factor > 0:
983 # if factor > 0:
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
984 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014
985
1015 self.flagIsNewFile = 0
986 self.flagIsNewFile = 0
1016 self.__isFirstTimeOnline = 0
987 self.__isFirstTimeOnline = 0
1017
988
1018 def __setNewBlock(self):
989 def __setNewBlock(self):
1019 #if self.server is None:
990 #if self.server is None:
1020 if self.fp == None:
991 if self.fp == None:
1021 return 0
992 return 0
1022
993
1023 # if self.online:
994 # if self.online:
1024 # self.__jumpToLastBlock()
995 # self.__jumpToLastBlock()
1025
996
1026 if self.flagIsNewFile:
997 if self.flagIsNewFile:
1027 self.lastUTTime = self.basicHeaderObj.utc
998 self.lastUTTime = self.basicHeaderObj.utc
1028 return 1
999 return 1
1029
1000
1030 if self.realtime:
1001 if self.realtime:
1031 self.flagDiscontinuousBlock = 1
1002 self.flagDiscontinuousBlock = 1
1032 if not(self.setNextFile()):
1003 if not(self.setNextFile()):
1033 return 0
1004 return 0
1034 else:
1005 else:
1035 return 1
1006 return 1
1036 #if self.server is None:
1007 #if self.server is None:
1037 currentSize = self.fileSize - self.fp.tell()
1008 currentSize = self.fileSize - self.fp.tell()
1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1009 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1039 if (currentSize >= neededSize):
1010 if (currentSize >= neededSize):
1040 self.basicHeaderObj.read(self.fp)
1011 self.basicHeaderObj.read(self.fp)
1041 self.lastUTTime = self.basicHeaderObj.utc
1012 self.lastUTTime = self.basicHeaderObj.utc
1042 return 1
1013 return 1
1043 # else:
1014 # else:
1044 # self.basicHeaderObj.read(self.zHeader)
1015 # self.basicHeaderObj.read(self.zHeader)
1045 # self.lastUTTime = self.basicHeaderObj.utc
1016 # self.lastUTTime = self.basicHeaderObj.utc
1046 # return 1
1017 # return 1
1047 if self.__waitNewBlock():
1018 if self.__waitNewBlock():
1048 self.lastUTTime = self.basicHeaderObj.utc
1019 self.lastUTTime = self.basicHeaderObj.utc
1049 return 1
1020 return 1
1050 #if self.server is None:
1021 #if self.server is None:
1051 if not(self.setNextFile()):
1022 if not(self.setNextFile()):
1052 return 0
1023 return 0
1053
1024
1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1025 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1055 self.lastUTTime = self.basicHeaderObj.utc
1026 self.lastUTTime = self.basicHeaderObj.utc
1056
1027
1057 self.flagDiscontinuousBlock = 0
1028 self.flagDiscontinuousBlock = 0
1058
1029
1059 if deltaTime > self.maxTimeStep:
1030 if deltaTime > self.maxTimeStep:
1060 self.flagDiscontinuousBlock = 1
1031 self.flagDiscontinuousBlock = 1
1061
1032
1062 return 1
1033 return 1
1063
1034
1064 def readNextBlock(self):
1035 def readNextBlock(self):
1065
1036
1066 #Skip block out of startTime and endTime
1037 #Skip block out of startTime and endTime
1067 while True:
1038 while True:
1068 if not(self.__setNewBlock()):
1039 if not(self.__setNewBlock()):
1069 print 'returning'
1070 return 0
1040 return 0
1041
1071 if not(self.readBlock()):
1042 if not(self.readBlock()):
1072 return 0
1043 return 0
1044
1073 self.getBasicHeader()
1045 self.getBasicHeader()
1046
1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075
1048
1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1077 self.processingHeaderObj.dataBlocksPerFile,
1050 self.processingHeaderObj.dataBlocksPerFile,
1078 self.dataOut.datatime.ctime())
1051 self.dataOut.datatime.ctime())
1079 continue
1052 continue
1080
1053
1081 break
1054 break
1082
1055
1083 if self.verbose:
1056 if self.verbose:
1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1057 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 self.processingHeaderObj.dataBlocksPerFile,
1058 self.processingHeaderObj.dataBlocksPerFile,
1086 self.dataOut.datatime.ctime())
1059 self.dataOut.datatime.ctime())
1087 return 1
1060 return 1
1088
1061
1089 def __readFirstHeader(self):
1062 def __readFirstHeader(self):
1090
1063
1091 self.basicHeaderObj.read(self.fp)
1064 self.basicHeaderObj.read(self.fp)
1092 self.systemHeaderObj.read(self.fp)
1065 self.systemHeaderObj.read(self.fp)
1093 self.radarControllerHeaderObj.read(self.fp)
1066 self.radarControllerHeaderObj.read(self.fp)
1094 self.processingHeaderObj.read(self.fp)
1067 self.processingHeaderObj.read(self.fp)
1095
1068
1096 self.firstHeaderSize = self.basicHeaderObj.size
1069 self.firstHeaderSize = self.basicHeaderObj.size
1097
1070
1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1071 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1099 if datatype == 0:
1072 if datatype == 0:
1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1073 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1101 elif datatype == 1:
1074 elif datatype == 1:
1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1075 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1103 elif datatype == 2:
1076 elif datatype == 2:
1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1077 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1105 elif datatype == 3:
1078 elif datatype == 3:
1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1079 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1107 elif datatype == 4:
1080 elif datatype == 4:
1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1081 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1109 elif datatype == 5:
1082 elif datatype == 5:
1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1083 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1111 else:
1084 else:
1112 raise ValueError, 'Data type was not defined'
1085 raise ValueError, 'Data type was not defined'
1113
1086
1114 self.dtype = datatype_str
1087 self.dtype = datatype_str
1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1088 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1089 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1090 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1091 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 self.getBlockDimension()
1092 self.getBlockDimension()
1120
1093
1121 def __verifyFile(self, filename, msgFlag=True):
1094 def __verifyFile(self, filename, msgFlag=True):
1122
1095
1123 msg = None
1096 msg = None
1124
1097
1125 try:
1098 try:
1126 fp = open(filename, 'rb')
1099 fp = open(filename, 'rb')
1127 except IOError:
1100 except IOError:
1128
1101
1129 if msgFlag:
1102 if msgFlag:
1130 print "[Reading] File %s can't be opened" % (filename)
1103 print "[Reading] File %s can't be opened" % (filename)
1131
1104
1132 return False
1105 return False
1133
1106
1134 currentPosition = fp.tell()
1107 currentPosition = fp.tell()
1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1108 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1136
1109
1137 if neededSize == 0:
1110 if neededSize == 0:
1138 basicHeaderObj = BasicHeader(LOCALTIME)
1111 basicHeaderObj = BasicHeader(LOCALTIME)
1139 systemHeaderObj = SystemHeader()
1112 systemHeaderObj = SystemHeader()
1140 radarControllerHeaderObj = RadarControllerHeader()
1113 radarControllerHeaderObj = RadarControllerHeader()
1141 processingHeaderObj = ProcessingHeader()
1114 processingHeaderObj = ProcessingHeader()
1142
1115
1143 if not( basicHeaderObj.read(fp) ):
1116 if not( basicHeaderObj.read(fp) ):
1144 fp.close()
1117 fp.close()
1145 return False
1118 return False
1146
1119
1147 if not( systemHeaderObj.read(fp) ):
1120 if not( systemHeaderObj.read(fp) ):
1148 fp.close()
1121 fp.close()
1149 return False
1122 return False
1150
1123
1151 if not( radarControllerHeaderObj.read(fp) ):
1124 if not( radarControllerHeaderObj.read(fp) ):
1152 fp.close()
1125 fp.close()
1153 return False
1126 return False
1154
1127
1155 if not( processingHeaderObj.read(fp) ):
1128 if not( processingHeaderObj.read(fp) ):
1156 fp.close()
1129 fp.close()
1157 return False
1130 return False
1158
1131
1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1132 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1160 else:
1133 else:
1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1134 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1162
1135
1163 fp.close()
1136 fp.close()
1164
1137
1165 fileSize = os.path.getsize(filename)
1138 fileSize = os.path.getsize(filename)
1166 currentSize = fileSize - currentPosition
1139 currentSize = fileSize - currentPosition
1167
1140
1168 if currentSize < neededSize:
1141 if currentSize < neededSize:
1169 if msgFlag and (msg != None):
1142 if msgFlag and (msg != None):
1170 print msg
1143 print msg
1171 return False
1144 return False
1172
1145
1173 return True
1146 return True
1174
1147
1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1148 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1176
1149
1177 path_empty = True
1150 path_empty = True
1178
1151
1179 dateList = []
1152 dateList = []
1180 pathList = []
1153 pathList = []
1181
1154
1182 multi_path = path.split(',')
1155 multi_path = path.split(',')
1183
1156
1184 if not walk:
1157 if not walk:
1185
1158
1186 for single_path in multi_path:
1159 for single_path in multi_path:
1187
1160
1188 if not os.path.isdir(single_path):
1161 if not os.path.isdir(single_path):
1189 continue
1162 continue
1190
1163
1191 fileList = glob.glob1(single_path, "*"+ext)
1164 fileList = glob.glob1(single_path, "*"+ext)
1192
1165
1193 if not fileList:
1166 if not fileList:
1194 continue
1167 continue
1195
1168
1196 path_empty = False
1169 path_empty = False
1197
1170
1198 fileList.sort()
1171 fileList.sort()
1199
1172
1200 for thisFile in fileList:
1173 for thisFile in fileList:
1201
1174
1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1175 if not os.path.isfile(os.path.join(single_path, thisFile)):
1203 continue
1176 continue
1204
1177
1205 if not isRadarFile(thisFile):
1178 if not isRadarFile(thisFile):
1206 continue
1179 continue
1207
1180
1208 if not isFileInDateRange(thisFile, startDate, endDate):
1181 if not isFileInDateRange(thisFile, startDate, endDate):
1209 continue
1182 continue
1210
1183
1211 thisDate = getDateFromRadarFile(thisFile)
1184 thisDate = getDateFromRadarFile(thisFile)
1212
1185
1213 if thisDate in dateList:
1186 if thisDate in dateList:
1214 continue
1187 continue
1215
1188
1216 dateList.append(thisDate)
1189 dateList.append(thisDate)
1217 pathList.append(single_path)
1190 pathList.append(single_path)
1218
1191
1219 else:
1192 else:
1220 for single_path in multi_path:
1193 for single_path in multi_path:
1221
1194
1222 if not os.path.isdir(single_path):
1195 if not os.path.isdir(single_path):
1223 continue
1196 continue
1224
1197
1225 dirList = []
1198 dirList = []
1226
1199
1227 for thisPath in os.listdir(single_path):
1200 for thisPath in os.listdir(single_path):
1228
1201
1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1202 if not os.path.isdir(os.path.join(single_path,thisPath)):
1230 continue
1203 continue
1231
1204
1232 if not isRadarFolder(thisPath):
1205 if not isRadarFolder(thisPath):
1233 continue
1206 continue
1234
1207
1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1208 if not isFolderInDateRange(thisPath, startDate, endDate):
1236 continue
1209 continue
1237
1210
1238 dirList.append(thisPath)
1211 dirList.append(thisPath)
1239
1212
1240 if not dirList:
1213 if not dirList:
1241 continue
1214 continue
1242
1215
1243 dirList.sort()
1216 dirList.sort()
1244
1217
1245 for thisDir in dirList:
1218 for thisDir in dirList:
1246
1219
1247 datapath = os.path.join(single_path, thisDir, expLabel)
1220 datapath = os.path.join(single_path, thisDir, expLabel)
1248 fileList = glob.glob1(datapath, "*"+ext)
1221 fileList = glob.glob1(datapath, "*"+ext)
1249
1222
1250 if not fileList:
1223 if not fileList:
1251 continue
1224 continue
1252
1225
1253 path_empty = False
1226 path_empty = False
1254
1227
1255 thisDate = getDateFromRadarFolder(thisDir)
1228 thisDate = getDateFromRadarFolder(thisDir)
1256
1229
1257 pathList.append(datapath)
1230 pathList.append(datapath)
1258 dateList.append(thisDate)
1231 dateList.append(thisDate)
1259
1232
1260 dateList.sort()
1233 dateList.sort()
1261
1234
1262 if walk:
1235 if walk:
1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1236 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1264 else:
1237 else:
1265 pattern_path = multi_path[0]
1238 pattern_path = multi_path[0]
1266
1239
1267 if path_empty:
1240 if path_empty:
1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1241 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1269 else:
1242 else:
1270 if not dateList:
1243 if not dateList:
1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1244 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1272
1245
1273 if include_path:
1246 if include_path:
1274 return dateList, pathList
1247 return dateList, pathList
1275
1248
1276 return dateList
1249 return dateList
1277
1250
1278 def setup(self,
1251 def setup(self,
1279 path=None,
1252 path=None,
1280 startDate=None,
1253 startDate=None,
1281 endDate=None,
1254 endDate=None,
1282 startTime=datetime.time(0,0,0),
1255 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1256 endTime=datetime.time(23,59,59),
1284 set=None,
1257 set=None,
1285 expLabel = "",
1258 expLabel = "",
1286 ext = None,
1259 ext = None,
1287 online = False,
1260 online = False,
1288 delay = 60,
1261 delay = 60,
1289 walk = True,
1262 walk = True,
1290 getblock = False,
1263 getblock = False,
1291 nTxs = 1,
1264 nTxs = 1,
1292 realtime=False,
1265 realtime=False,
1293 blocksize=None,
1266 blocksize=None,
1294 blocktime=None,
1267 blocktime=None,
1268 skip=None,
1269 cursor=None,
1270 warnings=True,
1295 verbose=True,
1271 verbose=True,
1296 **kwargs):
1272 server=None):
1297
1273 if server is not None:
1274 if 'tcp://' in server:
1275 address = server
1276 else:
1277 address = 'ipc:///tmp/%s' % server
1278 self.server = address
1279 self.context = zmq.Context()
1280 self.receiver = self.context.socket(zmq.PULL)
1281 self.receiver.connect(self.server)
1282 time.sleep(0.5)
1283 print '[Starting] ReceiverData from {}'.format(self.server)
1284 else:
1285 self.server = None
1298 if path == None:
1286 if path == None:
1299 raise ValueError, "[Reading] The path is not valid"
1287 raise ValueError, "[Reading] The path is not valid"
1300
1288
1301
1302 if ext == None:
1289 if ext == None:
1303 ext = self.ext
1290 ext = self.ext
1304
1291
1305 self.verbose=verbose
1306 self.path = path
1307 self.startDate = startDate
1308 self.endDate = endDate
1309 self.startTime = startTime
1310 self.endTime = endTime
1311 self.set = set
1312 self.expLabel = expLabel
1313 self.ext = ext
1314 self.online = online
1315 self.delay = delay
1316 self.walk = walk
1317 self.getblock = getblock
1318 self.nTxs = nTxs
1319 self.realtime = realtime
1320 self.blocksize = blocksize
1321 self.blocktime = blocktime
1322
1323
1324 if self.firstTime is True:
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 startTime=startTime, endTime=endTime,
1327 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk)
1329 if filenameList is not None: filenameList = filenameList[:-1]
1330
1331 if pathList is not None and filenameList is not None and online:
1332 self.onlineWithDate = True
1333 online = False
1334 self.fileIndex = -1
1335 self.pathList = pathList
1336 self.filenameList = filenameList
1337 file_name = os.path.basename(filenameList[-1])
1338 basename, ext = os.path.splitext(file_name)
1339 last_set = int(basename[-3:])
1340
1341 if online:
1292 if online:
1342 print "[Reading] Searching files in online mode..."
1293 print "[Reading] Searching files in online mode..."
1343
1294
1344 for nTries in range(self.nTries):
1295 for nTries in range( self.nTries ):
1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1296 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1346 expLabel=expLabel,
1347 ext=ext,
1348 walk=walk,
1349 startDate=startDate,
1350 startTime=startTime,
1351 set=set)
1352
1297
1353 if fullpath:
1298 if fullpath:
1354 break
1299 break
1300
1355 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1301 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 sleep( self.delay )
1302 sleep( self.delay )
1357
1303
1358 if not(fullpath):
1304 if not(fullpath):
1359 print "[Reading] There 'isn't any valid file in %s" % path
1305 print "[Reading] There 'isn't any valid file in %s" % path
1360 return
1306 return
1361
1307
1362 self.year = year
1308 self.year = year
1363 self.doy = doy
1309 self.doy = doy
1364 self.set = set - 1
1310 self.set = set - 1
1365 self.path = path
1311 self.path = path
1366 self.foldercounter = foldercounter
1312 self.foldercounter = foldercounter
1367 last_set = None
1313 last_set = None
1368 else:
1314 else:
1369 print "[Reading] Searching files in offline mode ..."
1315 print "[Reading] Searching files in offline mode ..."
1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1316 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 startTime=startTime, endTime=endTime,
1317 startTime=startTime, endTime=endTime,
1372 set=set, expLabel=expLabel, ext=ext,
1318 set=set, expLabel=expLabel, ext=ext,
1373 walk=walk)
1319 walk=walk, cursor=cursor,
1320 skip=skip)
1374
1321
1375 if not(pathList):
1322 if not(pathList):
1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379
1380 # sys.exit(-1)
1381
1382 self.fileIndex = -1
1323 self.fileIndex = -1
1383 self.pathList = []
1324 self.pathList = []
1384 self.filenameList = []
1325 self.filenameList = []
1385 return
1326 return
1386
1327
1387 self.fileIndex = -1
1328 self.fileIndex = -1
1388 self.pathList = pathList
1329 self.pathList = pathList
1389 self.filenameList = filenameList
1330 self.filenameList = filenameList
1390 file_name = os.path.basename(filenameList[-1])
1331 file_name = os.path.basename(filenameList[-1])
1391 basename, ext = os.path.splitext(file_name)
1332 basename, ext = os.path.splitext(file_name)
1392 last_set = int(basename[-3:])
1333 last_set = int(basename[-3:])
1393
1334
1394
1395 self.online = online
1335 self.online = online
1396 self.realtime = realtime
1336 self.realtime = realtime
1397 self.delay = delay
1337 self.delay = delay
1398 ext = ext.lower()
1338 ext = ext.lower()
1399 self.ext = ext
1339 self.ext = ext
1400 self.getByBlock = getblock
1340 self.getByBlock = getblock
1401 self.nTxs = nTxs
1341 self.nTxs = nTxs
1402 self.startTime = startTime
1342 self.startTime = startTime
1403 self.endTime = endTime
1343 self.endTime = endTime
1404
1344
1405
1406 #Added-----------------
1345 #Added-----------------
1407 self.selBlocksize = blocksize
1346 self.selBlocksize = blocksize
1408 self.selBlocktime = blocktime
1347 self.selBlocktime = blocktime
1409
1348
1349 # Verbose-----------
1350 self.verbose = verbose
1351 self.warnings = warnings
1410
1352
1411 if not(self.setNextFile()):
1353 if not(self.setNextFile()):
1412 if (startDate!=None) and (endDate!=None):
1354 if (startDate!=None) and (endDate!=None):
1413 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1355 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1414 elif startDate != None:
1356 elif startDate != None:
1415 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1357 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1416 else:
1358 else:
1417 print "[Reading] No files"
1359 print "[Reading] No files"
1418
1360
1419 self.fileIndex = -1
1361 self.fileIndex = -1
1420 self.pathList = []
1362 self.pathList = []
1421 self.filenameList = []
1363 self.filenameList = []
1422 return
1364 return
1423
1365
1424 # self.getBasicHeader()
1366 # self.getBasicHeader()
1425
1367
1426 if last_set != None:
1368 if last_set != None:
1427 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1369 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1428 return
1370 return
1429
1371
1430 def getBasicHeader(self):
1372 def getBasicHeader(self):
1431
1373
1432 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1374 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1433
1375
1434 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1376 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1435
1377
1436 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1378 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1437
1379
1438 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1380 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1439
1381
1440 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1382 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1441
1383
1442 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1384 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1443
1385
1444 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1386 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1445
1387
1446 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1388 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1447
1389
1448
1390
1449 def getFirstHeader(self):
1391 def getFirstHeader(self):
1450
1392
1451 raise NotImplementedError
1393 raise NotImplementedError
1452
1394
1453 def getData(self):
1395 def getData(self):
1454
1396
1455 raise NotImplementedError
1397 raise NotImplementedError
1456
1398
1457 def hasNotDataInBuffer(self):
1399 def hasNotDataInBuffer(self):
1458
1400
1459 raise NotImplementedError
1401 raise NotImplementedError
1460
1402
1461 def readBlock(self):
1403 def readBlock(self):
1462
1404
1463 raise NotImplementedError
1405 raise NotImplementedError
1464
1406
1465 def isEndProcess(self):
1407 def isEndProcess(self):
1466
1408
1467 return self.flagNoMoreFiles
1409 return self.flagNoMoreFiles
1468
1410
1469 def printReadBlocks(self):
1411 def printReadBlocks(self):
1470
1412
1471 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1413 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1472
1414
1473 def printTotalBlocks(self):
1415 def printTotalBlocks(self):
1474
1416
1475 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1417 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1476
1418
1477 def printNumberOfBlock(self):
1419 def printNumberOfBlock(self):
1478
1420
1479 if self.flagIsNewBlock:
1421 if self.flagIsNewBlock:
1480 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1422 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1481 self.processingHeaderObj.dataBlocksPerFile,
1423 self.processingHeaderObj.dataBlocksPerFile,
1482 self.dataOut.datatime.ctime())
1424 self.dataOut.datatime.ctime())
1483
1425
1484 def printInfo(self):
1426 def printInfo(self):
1485
1427
1486 if self.__printInfo == False:
1428 if self.__printInfo == False:
1487 return
1429 return
1488
1430
1489 self.basicHeaderObj.printInfo()
1431 self.basicHeaderObj.printInfo()
1490 self.systemHeaderObj.printInfo()
1432 self.systemHeaderObj.printInfo()
1491 self.radarControllerHeaderObj.printInfo()
1433 self.radarControllerHeaderObj.printInfo()
1492 self.processingHeaderObj.printInfo()
1434 self.processingHeaderObj.printInfo()
1493
1435
1494 self.__printInfo = False
1436 self.__printInfo = False
1495
1437
1496 def run(self,
1438 def run(self,
1497 path=None,
1439 path=None,
1498 startDate=None,
1440 startDate=None,
1499 endDate=None,
1441 endDate=None,
1500 startTime=datetime.time(0,0,0),
1442 startTime=datetime.time(0,0,0),
1501 endTime=datetime.time(23,59,59),
1443 endTime=datetime.time(23,59,59),
1502 set=None,
1444 set=None,
1503 expLabel = "",
1445 expLabel = "",
1504 ext = None,
1446 ext = None,
1505 online = False,
1447 online = False,
1506 delay = 60,
1448 delay = 60,
1507 walk = True,
1449 walk = True,
1508 getblock = False,
1450 getblock = False,
1509 nTxs = 1,
1451 nTxs = 1,
1510 realtime=False,
1452 realtime=False,
1511 blocksize=None,
1453 blocksize=None,
1512 blocktime=None,
1454 blocktime=None,
1513 queue=None,
1514 skip=None,
1455 skip=None,
1515 cursor=None,
1456 cursor=None,
1516 warnings=True,
1457 warnings=True,
1517 server=None,
1458 server=None,
1518 verbose=True, **kwargs):
1459 verbose=True, **kwargs):
1519
1460
1520 if not(self.isConfig):
1461 if not(self.isConfig):
1521 # self.dataOut = dataOut
1522 self.setup( path=path,
1462 self.setup(path=path,
1523 startDate=startDate,
1463 startDate=startDate,
1524 endDate=endDate,
1464 endDate=endDate,
1525 startTime=startTime,
1465 startTime=startTime,
1526 endTime=endTime,
1466 endTime=endTime,
1527 set=set,
1467 set=set,
1528 expLabel=expLabel,
1468 expLabel=expLabel,
1529 ext=ext,
1469 ext=ext,
1530 online=online,
1470 online=online,
1531 delay=delay,
1471 delay=delay,
1532 walk=walk,
1472 walk=walk,
1533 getblock=getblock,
1473 getblock=getblock,
1534 nTxs=nTxs,
1474 nTxs=nTxs,
1535 realtime=realtime,
1475 realtime=realtime,
1536 blocksize=blocksize,
1476 blocksize=blocksize,
1537 blocktime=blocktime,
1477 blocktime=blocktime,
1538 queue=queue,
1539 skip=skip,
1478 skip=skip,
1540 cursor=cursor,
1479 cursor=cursor,
1541 warnings=warnings,
1480 warnings=warnings,
1542 server=server,
1481 server=server,
1543 verbose=verbose, **kwargs)
1482 verbose=verbose)
1544 self.isConfig = True
1483 self.isConfig = True
1545 if server is None:
1484 if server is None:
1546 self.getData()
1485 self.getData()
1547 else:
1486 else:
1548 self.getFromServer()
1487 self.getFromServer()
1549
1488
1550 class JRODataWriter(JRODataIO):
1489 class JRODataWriter(JRODataIO):
1551
1490
1552 """
1491 """
1553 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1492 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1554 de los datos siempre se realiza por bloques.
1493 de los datos siempre se realiza por bloques.
1555 """
1494 """
1556
1495
1557 blockIndex = 0
1496 blockIndex = 0
1558
1497
1559 path = None
1498 path = None
1560
1499
1561 setFile = None
1500 setFile = None
1562
1501
1563 profilesPerBlock = None
1502 profilesPerBlock = None
1564
1503
1565 blocksPerFile = None
1504 blocksPerFile = None
1566
1505
1567 nWriteBlocks = 0
1506 nWriteBlocks = 0
1568
1507
1569 fileDate = None
1508 fileDate = None
1570
1509
1571 def __init__(self, dataOut=None):
1510 def __init__(self, dataOut=None):
1572 raise NotImplementedError
1511 raise NotImplementedError
1573
1512
1574
1513
1575 def hasAllDataInBuffer(self):
1514 def hasAllDataInBuffer(self):
1576 raise NotImplementedError
1515 raise NotImplementedError
1577
1516
1578
1517
1579 def setBlockDimension(self):
1518 def setBlockDimension(self):
1580 raise NotImplementedError
1519 raise NotImplementedError
1581
1520
1582
1521
1583 def writeBlock(self):
1522 def writeBlock(self):
1584 raise NotImplementedError
1523 raise NotImplementedError
1585
1524
1586
1525
1587 def putData(self):
1526 def putData(self):
1588 raise NotImplementedError
1527 raise NotImplementedError
1589
1528
1590
1529
1591 def getProcessFlags(self):
1530 def getProcessFlags(self):
1592
1531
1593 processFlags = 0
1532 processFlags = 0
1594
1533
1595 dtype_index = get_dtype_index(self.dtype)
1534 dtype_index = get_dtype_index(self.dtype)
1596 procflag_dtype = get_procflag_dtype(dtype_index)
1535 procflag_dtype = get_procflag_dtype(dtype_index)
1597
1536
1598 processFlags += procflag_dtype
1537 processFlags += procflag_dtype
1599
1538
1600 if self.dataOut.flagDecodeData:
1539 if self.dataOut.flagDecodeData:
1601 processFlags += PROCFLAG.DECODE_DATA
1540 processFlags += PROCFLAG.DECODE_DATA
1602
1541
1603 if self.dataOut.flagDeflipData:
1542 if self.dataOut.flagDeflipData:
1604 processFlags += PROCFLAG.DEFLIP_DATA
1543 processFlags += PROCFLAG.DEFLIP_DATA
1605
1544
1606 if self.dataOut.code is not None:
1545 if self.dataOut.code is not None:
1607 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1546 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1608
1547
1609 if self.dataOut.nCohInt > 1:
1548 if self.dataOut.nCohInt > 1:
1610 processFlags += PROCFLAG.COHERENT_INTEGRATION
1549 processFlags += PROCFLAG.COHERENT_INTEGRATION
1611
1550
1612 if self.dataOut.type == "Spectra":
1551 if self.dataOut.type == "Spectra":
1613 if self.dataOut.nIncohInt > 1:
1552 if self.dataOut.nIncohInt > 1:
1614 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1553 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1615
1554
1616 if self.dataOut.data_dc is not None:
1555 if self.dataOut.data_dc is not None:
1617 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1556 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1618
1557
1619 if self.dataOut.flagShiftFFT:
1558 if self.dataOut.flagShiftFFT:
1620 processFlags += PROCFLAG.SHIFT_FFT_DATA
1559 processFlags += PROCFLAG.SHIFT_FFT_DATA
1621
1560
1622 return processFlags
1561 return processFlags
1623
1562
1624 def setBasicHeader(self):
1563 def setBasicHeader(self):
1625
1564
1626 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1565 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1627 self.basicHeaderObj.version = self.versionFile
1566 self.basicHeaderObj.version = self.versionFile
1628 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1567 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1629
1568
1630 utc = numpy.floor(self.dataOut.utctime)
1569 utc = numpy.floor(self.dataOut.utctime)
1631 milisecond = (self.dataOut.utctime - utc)* 1000.0
1570 milisecond = (self.dataOut.utctime - utc)* 1000.0
1632
1571
1633 self.basicHeaderObj.utc = utc
1572 self.basicHeaderObj.utc = utc
1634 self.basicHeaderObj.miliSecond = milisecond
1573 self.basicHeaderObj.miliSecond = milisecond
1635 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1574 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1636 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1575 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1637 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1576 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1638
1577
1639 def setFirstHeader(self):
1578 def setFirstHeader(self):
1640 """
1579 """
1641 Obtiene una copia del First Header
1580 Obtiene una copia del First Header
1642
1581
1643 Affected:
1582 Affected:
1644
1583
1645 self.basicHeaderObj
1584 self.basicHeaderObj
1646 self.systemHeaderObj
1585 self.systemHeaderObj
1647 self.radarControllerHeaderObj
1586 self.radarControllerHeaderObj
1648 self.processingHeaderObj self.
1587 self.processingHeaderObj self.
1649
1588
1650 Return:
1589 Return:
1651 None
1590 None
1652 """
1591 """
1653
1592
1654 raise NotImplementedError
1593 raise NotImplementedError
1655
1594
1656 def __writeFirstHeader(self):
1595 def __writeFirstHeader(self):
1657 """
1596 """
1658 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1597 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1659
1598
1660 Affected:
1599 Affected:
1661 __dataType
1600 __dataType
1662
1601
1663 Return:
1602 Return:
1664 None
1603 None
1665 """
1604 """
1666
1605
1667 # CALCULAR PARAMETROS
1606 # CALCULAR PARAMETROS
1668
1607
1669 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1608 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1670 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1609 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1671
1610
1672 self.basicHeaderObj.write(self.fp)
1611 self.basicHeaderObj.write(self.fp)
1673 self.systemHeaderObj.write(self.fp)
1612 self.systemHeaderObj.write(self.fp)
1674 self.radarControllerHeaderObj.write(self.fp)
1613 self.radarControllerHeaderObj.write(self.fp)
1675 self.processingHeaderObj.write(self.fp)
1614 self.processingHeaderObj.write(self.fp)
1676
1615
1677 def __setNewBlock(self):
1616 def __setNewBlock(self):
1678 """
1617 """
1679 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1618 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1680
1619
1681 Return:
1620 Return:
1682 0 : si no pudo escribir nada
1621 0 : si no pudo escribir nada
1683 1 : Si escribio el Basic el First Header
1622 1 : Si escribio el Basic el First Header
1684 """
1623 """
1685 if self.fp == None:
1624 if self.fp == None:
1686 self.setNextFile()
1625 self.setNextFile()
1687
1626
1688 if self.flagIsNewFile:
1627 if self.flagIsNewFile:
1689 return 1
1628 return 1
1690
1629
1691 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1630 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1692 self.basicHeaderObj.write(self.fp)
1631 self.basicHeaderObj.write(self.fp)
1693 return 1
1632 return 1
1694
1633
1695 if not( self.setNextFile() ):
1634 if not( self.setNextFile() ):
1696 return 0
1635 return 0
1697
1636
1698 return 1
1637 return 1
1699
1638
1700
1639
1701 def writeNextBlock(self):
1640 def writeNextBlock(self):
1702 """
1641 """
1703 Selecciona el bloque siguiente de datos y los escribe en un file
1642 Selecciona el bloque siguiente de datos y los escribe en un file
1704
1643
1705 Return:
1644 Return:
1706 0 : Si no hizo pudo escribir el bloque de datos
1645 0 : Si no hizo pudo escribir el bloque de datos
1707 1 : Si no pudo escribir el bloque de datos
1646 1 : Si no pudo escribir el bloque de datos
1708 """
1647 """
1709 if not( self.__setNewBlock() ):
1648 if not( self.__setNewBlock() ):
1710 return 0
1649 return 0
1711
1650
1712 self.writeBlock()
1651 self.writeBlock()
1713
1652
1714 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1653 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1715 self.processingHeaderObj.dataBlocksPerFile)
1654 self.processingHeaderObj.dataBlocksPerFile)
1716
1655
1717 return 1
1656 return 1
1718
1657
1719 def setNextFile(self):
1658 def setNextFile(self):
1720 """
1659 """
1721 Determina el siguiente file que sera escrito
1660 Determina el siguiente file que sera escrito
1722
1661
1723 Affected:
1662 Affected:
1724 self.filename
1663 self.filename
1725 self.subfolder
1664 self.subfolder
1726 self.fp
1665 self.fp
1727 self.setFile
1666 self.setFile
1728 self.flagIsNewFile
1667 self.flagIsNewFile
1729
1668
1730 Return:
1669 Return:
1731 0 : Si el archivo no puede ser escrito
1670 0 : Si el archivo no puede ser escrito
1732 1 : Si el archivo esta listo para ser escrito
1671 1 : Si el archivo esta listo para ser escrito
1733 """
1672 """
1734 ext = self.ext
1673 ext = self.ext
1735 path = self.path
1674 path = self.path
1736
1675
1737 if self.fp != None:
1676 if self.fp != None:
1738 self.fp.close()
1677 self.fp.close()
1739
1678
1740 timeTuple = time.localtime( self.dataOut.utctime)
1679 timeTuple = time.localtime( self.dataOut.utctime)
1741 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1680 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1742
1681
1743 fullpath = os.path.join( path, subfolder )
1682 fullpath = os.path.join( path, subfolder )
1744 setFile = self.setFile
1683 setFile = self.setFile
1745
1684
1746 if not( os.path.exists(fullpath) ):
1685 if not( os.path.exists(fullpath) ):
1747 os.mkdir(fullpath)
1686 os.mkdir(fullpath)
1748 setFile = -1 #inicializo mi contador de seteo
1687 setFile = -1 #inicializo mi contador de seteo
1749 else:
1688 else:
1750 filesList = os.listdir( fullpath )
1689 filesList = os.listdir( fullpath )
1751 if len( filesList ) > 0:
1690 if len( filesList ) > 0:
1752 filesList = sorted( filesList, key=str.lower )
1691 filesList = sorted( filesList, key=str.lower )
1753 filen = filesList[-1]
1692 filen = filesList[-1]
1754 # el filename debera tener el siguiente formato
1693 # el filename debera tener el siguiente formato
1755 # 0 1234 567 89A BCDE (hex)
1694 # 0 1234 567 89A BCDE (hex)
1756 # x YYYY DDD SSS .ext
1695 # x YYYY DDD SSS .ext
1757 if isNumber( filen[8:11] ):
1696 if isNumber( filen[8:11] ):
1758 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1697 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1759 else:
1698 else:
1760 setFile = -1
1699 setFile = -1
1761 else:
1700 else:
1762 setFile = -1 #inicializo mi contador de seteo
1701 setFile = -1 #inicializo mi contador de seteo
1763
1702
1764 setFile += 1
1703 setFile += 1
1765
1704
1766 #If this is a new day it resets some values
1705 #If this is a new day it resets some values
1767 if self.dataOut.datatime.date() > self.fileDate:
1706 if self.dataOut.datatime.date() > self.fileDate:
1768 setFile = 0
1707 setFile = 0
1769 self.nTotalBlocks = 0
1708 self.nTotalBlocks = 0
1770
1709
1771 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1710 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1772
1711
1773 filename = os.path.join( path, subfolder, filen )
1712 filename = os.path.join( path, subfolder, filen )
1774
1713
1775 fp = open( filename,'wb' )
1714 fp = open( filename,'wb' )
1776
1715
1777 self.blockIndex = 0
1716 self.blockIndex = 0
1778
1717
1779 #guardando atributos
1718 #guardando atributos
1780 self.filename = filename
1719 self.filename = filename
1781 self.subfolder = subfolder
1720 self.subfolder = subfolder
1782 self.fp = fp
1721 self.fp = fp
1783 self.setFile = setFile
1722 self.setFile = setFile
1784 self.flagIsNewFile = 1
1723 self.flagIsNewFile = 1
1785 self.fileDate = self.dataOut.datatime.date()
1724 self.fileDate = self.dataOut.datatime.date()
1786
1725
1787 self.setFirstHeader()
1726 self.setFirstHeader()
1788
1727
1789 print '[Writing] Opening file: %s'%self.filename
1728 print '[Writing] Opening file: %s'%self.filename
1790
1729
1791 self.__writeFirstHeader()
1730 self.__writeFirstHeader()
1792
1731
1793 return 1
1732 return 1
1794
1733
1795 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1734 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1796 """
1735 """
1797 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1736 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1798
1737
1799 Inputs:
1738 Inputs:
1800 path : directory where data will be saved
1739 path : directory where data will be saved
1801 profilesPerBlock : number of profiles per block
1740 profilesPerBlock : number of profiles per block
1802 set : initial file set
1741 set : initial file set
1803 datatype : An integer number that defines data type:
1742 datatype : An integer number that defines data type:
1804 0 : int8 (1 byte)
1743 0 : int8 (1 byte)
1805 1 : int16 (2 bytes)
1744 1 : int16 (2 bytes)
1806 2 : int32 (4 bytes)
1745 2 : int32 (4 bytes)
1807 3 : int64 (8 bytes)
1746 3 : int64 (8 bytes)
1808 4 : float32 (4 bytes)
1747 4 : float32 (4 bytes)
1809 5 : double64 (8 bytes)
1748 5 : double64 (8 bytes)
1810
1749
1811 Return:
1750 Return:
1812 0 : Si no realizo un buen seteo
1751 0 : Si no realizo un buen seteo
1813 1 : Si realizo un buen seteo
1752 1 : Si realizo un buen seteo
1814 """
1753 """
1815
1754
1816 if ext == None:
1755 if ext == None:
1817 ext = self.ext
1756 ext = self.ext
1818
1757
1819 self.ext = ext.lower()
1758 self.ext = ext.lower()
1820
1759
1821 self.path = path
1760 self.path = path
1822
1761
1823 if set is None:
1762 if set is None:
1824 self.setFile = -1
1763 self.setFile = -1
1825 else:
1764 else:
1826 self.setFile = set - 1
1765 self.setFile = set - 1
1827
1766
1828 self.blocksPerFile = blocksPerFile
1767 self.blocksPerFile = blocksPerFile
1829
1768
1830 self.profilesPerBlock = profilesPerBlock
1769 self.profilesPerBlock = profilesPerBlock
1831
1770
1832 self.dataOut = dataOut
1771 self.dataOut = dataOut
1833 self.fileDate = self.dataOut.datatime.date()
1772 self.fileDate = self.dataOut.datatime.date()
1834 #By default
1773 #By default
1835 self.dtype = self.dataOut.dtype
1774 self.dtype = self.dataOut.dtype
1836
1775
1837 if datatype is not None:
1776 if datatype is not None:
1838 self.dtype = get_numpy_dtype(datatype)
1777 self.dtype = get_numpy_dtype(datatype)
1839
1778
1840 if not(self.setNextFile()):
1779 if not(self.setNextFile()):
1841 print "[Writing] There isn't a next file"
1780 print "[Writing] There isn't a next file"
1842 return 0
1781 return 0
1843
1782
1844 self.setBlockDimension()
1783 self.setBlockDimension()
1845
1784
1846 return 1
1785 return 1
1847
1786
1848 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1787 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1849
1788
1850 if not(self.isConfig):
1789 if not(self.isConfig):
1851
1790
1852 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1791 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1853 self.isConfig = True
1792 self.isConfig = True
1854
1793
1855 self.putData()
1794 self.putData()
Show file before | Show file after | Hide comments
@@ -1,848 +1,848
1 '''
1 '''
2 Created on Jul 3, 2014
2 Created on Jul 3, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import os, sys
7 import os, sys
8 import time, datetime
8 import time, datetime
9 import numpy
9 import numpy
10 import fnmatch
10 import fnmatch
11 import glob
11 import glob
12 from time import sleep
12 from time import sleep
13
13
14 try:
14 try:
15 import pyfits
15 import pyfits
16 except ImportError, e:
16 except ImportError, e:
17 print "Fits data cannot be used. Install pyfits module"
17 print "Fits data cannot be used. Install pyfits module"
18
18
19 from xml.etree.ElementTree import ElementTree
19 from xml.etree.ElementTree import ElementTree
20
20
21 from jroIO_base import isRadarFolder, isNumber
21 from jroIO_base import isRadarFolder, isNumber
22 from schainpy.model.data.jrodata import Fits
22 from schainpy.model.data.jrodata import Fits
23 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
23 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
24
24
25 class PyFits(object):
25 class PyFits(object):
26 name=None
26 name=None
27 format=None
27 format=None
28 array =None
28 array =None
29 data =None
29 data =None
30 thdulist=None
30 thdulist=None
31 prihdr=None
31 prihdr=None
32 hdu=None
32 hdu=None
33
33
34 def __init__(self):
34 def __init__(self):
35
35
36 pass
36 pass
37
37
38 def setColF(self,name,format,array):
38 def setColF(self,name,format,array):
39 self.name=name
39 self.name=name
40 self.format=format
40 self.format=format
41 self.array=array
41 self.array=array
42 a1=numpy.array([self.array],dtype=numpy.float32)
42 a1=numpy.array([self.array],dtype=numpy.float32)
43 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
43 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
44 return self.col1
44 return self.col1
45
45
46 # def setColP(self,name,format,data):
46 # def setColP(self,name,format,data):
47 # self.name=name
47 # self.name=name
48 # self.format=format
48 # self.format=format
49 # self.data=data
49 # self.data=data
50 # a2=numpy.array([self.data],dtype=numpy.float32)
50 # a2=numpy.array([self.data],dtype=numpy.float32)
51 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
51 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
52 # return self.col2
52 # return self.col2
53
53
54
54
55 def writeData(self,name,format,data):
55 def writeData(self,name,format,data):
56 self.name=name
56 self.name=name
57 self.format=format
57 self.format=format
58 self.data=data
58 self.data=data
59 a2=numpy.array([self.data],dtype=numpy.float32)
59 a2=numpy.array([self.data],dtype=numpy.float32)
60 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
60 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
61 return self.col2
61 return self.col2
62
62
63 def cFImage(self,idblock,year,month,day,hour,minute,second):
63 def cFImage(self,idblock,year,month,day,hour,minute,second):
64 self.hdu= pyfits.PrimaryHDU(idblock)
64 self.hdu= pyfits.PrimaryHDU(idblock)
65 self.hdu.header.set("Year",year)
65 self.hdu.header.set("Year",year)
66 self.hdu.header.set("Month",month)
66 self.hdu.header.set("Month",month)
67 self.hdu.header.set("Day",day)
67 self.hdu.header.set("Day",day)
68 self.hdu.header.set("Hour",hour)
68 self.hdu.header.set("Hour",hour)
69 self.hdu.header.set("Minute",minute)
69 self.hdu.header.set("Minute",minute)
70 self.hdu.header.set("Second",second)
70 self.hdu.header.set("Second",second)
71 return self.hdu
71 return self.hdu
72
72
73
73
74 def Ctable(self,colList):
74 def Ctable(self,colList):
75 self.cols=pyfits.ColDefs(colList)
75 self.cols=pyfits.ColDefs(colList)
76 self.tbhdu = pyfits.new_table(self.cols)
76 self.tbhdu = pyfits.new_table(self.cols)
77 return self.tbhdu
77 return self.tbhdu
78
78
79
79
80 def CFile(self,hdu,tbhdu):
80 def CFile(self,hdu,tbhdu):
81 self.thdulist=pyfits.HDUList([hdu,tbhdu])
81 self.thdulist=pyfits.HDUList([hdu,tbhdu])
82
82
83 def wFile(self,filename):
83 def wFile(self,filename):
84 if os.path.isfile(filename):
84 if os.path.isfile(filename):
85 os.remove(filename)
85 os.remove(filename)
86 self.thdulist.writeto(filename)
86 self.thdulist.writeto(filename)
87
87
88
88
89 class ParameterConf:
89 class ParameterConf:
90 ELEMENTNAME = 'Parameter'
90 ELEMENTNAME = 'Parameter'
91 def __init__(self):
91 def __init__(self):
92 self.name = ''
92 self.name = ''
93 self.value = ''
93 self.value = ''
94
94
95 def readXml(self, parmElement):
95 def readXml(self, parmElement):
96 self.name = parmElement.get('name')
96 self.name = parmElement.get('name')
97 self.value = parmElement.get('value')
97 self.value = parmElement.get('value')
98
98
99 def getElementName(self):
99 def getElementName(self):
100 return self.ELEMENTNAME
100 return self.ELEMENTNAME
101
101
102 class Metadata(object):
102 class Metadata(object):
103
103
104 def __init__(self, filename):
104 def __init__(self, filename):
105 self.parmConfObjList = []
105 self.parmConfObjList = []
106 self.readXml(filename)
106 self.readXml(filename)
107
107
108 def readXml(self, filename):
108 def readXml(self, filename):
109 self.projectElement = None
109 self.projectElement = None
110 self.procUnitConfObjDict = {}
110 self.procUnitConfObjDict = {}
111 self.projectElement = ElementTree().parse(filename)
111 self.projectElement = ElementTree().parse(filename)
112 self.project = self.projectElement.tag
112 self.project = self.projectElement.tag
113
113
114 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
114 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
115
115
116 for parmElement in parmElementList:
116 for parmElement in parmElementList:
117 parmConfObj = ParameterConf()
117 parmConfObj = ParameterConf()
118 parmConfObj.readXml(parmElement)
118 parmConfObj.readXml(parmElement)
119 self.parmConfObjList.append(parmConfObj)
119 self.parmConfObjList.append(parmConfObj)
120
120
121 class FitsWriter(Operation):
121 class FitsWriter(Operation):
122 def __init__(self, **kwargs):
122 def __init__(self, **kwargs):
123 Operation.__init__(self, **kwargs)
123 Operation.__init__(self, **kwargs)
124 self.isConfig = False
124 self.isConfig = False
125 self.dataBlocksPerFile = None
125 self.dataBlocksPerFile = None
126 self.blockIndex = 0
126 self.blockIndex = 0
127 self.flagIsNewFile = 1
127 self.flagIsNewFile = 1
128 self.fitsObj = None
128 self.fitsObj = None
129 self.optchar = 'P'
129 self.optchar = 'P'
130 self.ext = '.fits'
130 self.ext = '.fits'
131 self.setFile = 0
131 self.setFile = 0
132
132
133 def setFitsHeader(self, dataOut, metadatafile=None):
133 def setFitsHeader(self, dataOut, metadatafile=None):
134
134
135 header_data = pyfits.PrimaryHDU()
135 header_data = pyfits.PrimaryHDU()
136
136
137 header_data.header['EXPNAME'] = "RADAR DATA"
137 header_data.header['EXPNAME'] = "RADAR DATA"
138 header_data.header['DATATYPE'] = "SPECTRA"
138 header_data.header['DATATYPE'] = "SPECTRA"
139 header_data.header['COMMENT'] = ""
139 header_data.header['COMMENT'] = ""
140
140
141 if metadatafile:
141 if metadatafile:
142
142
143 metadata4fits = Metadata(metadatafile)
143 metadata4fits = Metadata(metadatafile)
144
144
145 for parameter in metadata4fits.parmConfObjList:
145 for parameter in metadata4fits.parmConfObjList:
146 parm_name = parameter.name
146 parm_name = parameter.name
147 parm_value = parameter.value
147 parm_value = parameter.value
148
148
149 header_data.header[parm_name] = parm_value
149 header_data.header[parm_name] = parm_value
150
150
151 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
151 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
152 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
152 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
153 header_data.header['NCHANNELS'] = dataOut.nChannels
153 header_data.header['NCHANNELS'] = dataOut.nChannels
154 #header_data.header['HEIGHTS'] = dataOut.heightList
154 #header_data.header['HEIGHTS'] = dataOut.heightList
155 header_data.header['NHEIGHTS'] = dataOut.nHeights
155 header_data.header['NHEIGHTS'] = dataOut.nHeights
156
156
157 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
157 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
158 header_data.header['NCOHINT'] = dataOut.nCohInt
158 header_data.header['NCOHINT'] = dataOut.nCohInt
159 header_data.header['NINCOHINT'] = dataOut.nIncohInt
159 header_data.header['NINCOHINT'] = dataOut.nIncohInt
160 header_data.header['TIMEZONE'] = dataOut.timeZone
160 header_data.header['TIMEZONE'] = dataOut.timeZone
161 header_data.header['NBLOCK'] = self.blockIndex
161 header_data.header['NBLOCK'] = self.blockIndex
162
162
163 header_data.writeto(self.filename)
163 header_data.writeto(self.filename)
164
164
165 self.addExtension(dataOut.heightList,'HEIGHTLIST')
165 self.addExtension(dataOut.heightList,'HEIGHTLIST')
166
166
167
167
168 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
168 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
169
169
170 self.path = path
170 self.path = path
171 self.dataOut = dataOut
171 self.dataOut = dataOut
172 self.metadatafile = metadatafile
172 self.metadatafile = metadatafile
173 self.dataBlocksPerFile = dataBlocksPerFile
173 self.dataBlocksPerFile = dataBlocksPerFile
174
174
175 def open(self):
175 def open(self):
176 self.fitsObj = pyfits.open(self.filename, mode='update')
176 self.fitsObj = pyfits.open(self.filename, mode='update')
177
177
178
178
179 def addExtension(self, data, tagname):
179 def addExtension(self, data, tagname):
180 self.open()
180 self.open()
181 extension = pyfits.ImageHDU(data=data, name=tagname)
181 extension = pyfits.ImageHDU(data=data, name=tagname)
182 #extension.header['TAG'] = tagname
182 #extension.header['TAG'] = tagname
183 self.fitsObj.append(extension)
183 self.fitsObj.append(extension)
184 self.write()
184 self.write()
185
185
186 def addData(self, data):
186 def addData(self, data):
187 self.open()
187 self.open()
188 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
188 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
189 extension.header['UTCTIME'] = self.dataOut.utctime
189 extension.header['UTCTIME'] = self.dataOut.utctime
190 self.fitsObj.append(extension)
190 self.fitsObj.append(extension)
191 self.blockIndex += 1
191 self.blockIndex += 1
192 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
192 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
193
193
194 self.write()
194 self.write()
195
195
196 def write(self):
196 def write(self):
197
197
198 self.fitsObj.flush(verbose=True)
198 self.fitsObj.flush(verbose=True)
199 self.fitsObj.close()
199 self.fitsObj.close()
200
200
201
201
202 def setNextFile(self):
202 def setNextFile(self):
203
203
204 ext = self.ext
204 ext = self.ext
205 path = self.path
205 path = self.path
206
206
207 timeTuple = time.localtime( self.dataOut.utctime)
207 timeTuple = time.localtime( self.dataOut.utctime)
208 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
208 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
209
209
210 fullpath = os.path.join( path, subfolder )
210 fullpath = os.path.join( path, subfolder )
211 if not( os.path.exists(fullpath) ):
211 if not( os.path.exists(fullpath) ):
212 os.mkdir(fullpath)
212 os.mkdir(fullpath)
213 self.setFile = -1 #inicializo mi contador de seteo
213 self.setFile = -1 #inicializo mi contador de seteo
214 else:
214 else:
215 filesList = os.listdir( fullpath )
215 filesList = os.listdir( fullpath )
216 if len( filesList ) > 0:
216 if len( filesList ) > 0:
217 filesList = sorted( filesList, key=str.lower )
217 filesList = sorted( filesList, key=str.lower )
218 filen = filesList[-1]
218 filen = filesList[-1]
219
219
220 if isNumber( filen[8:11] ):
220 if isNumber( filen[8:11] ):
221 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
221 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
222 else:
222 else:
223 self.setFile = -1
223 self.setFile = -1
224 else:
224 else:
225 self.setFile = -1 #inicializo mi contador de seteo
225 self.setFile = -1 #inicializo mi contador de seteo
226
226
227 setFile = self.setFile
227 setFile = self.setFile
228 setFile += 1
228 setFile += 1
229
229
230 thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
230 thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
231 timeTuple.tm_year,
231 timeTuple.tm_year,
232 timeTuple.tm_yday,
232 timeTuple.tm_yday,
233 setFile,
233 setFile,
234 ext )
234 ext )
235
235
236 filename = os.path.join( path, subfolder, thisFile )
236 filename = os.path.join( path, subfolder, thisFile )
237
237
238 self.blockIndex = 0
238 self.blockIndex = 0
239 self.filename = filename
239 self.filename = filename
240 self.setFile = setFile
240 self.setFile = setFile
241 self.flagIsNewFile = 1
241 self.flagIsNewFile = 1
242
242
243 print 'Writing the file: %s'%self.filename
243 print 'Writing the file: %s'%self.filename
244
244
245 self.setFitsHeader(self.dataOut, self.metadatafile)
245 self.setFitsHeader(self.dataOut, self.metadatafile)
246
246
247 return 1
247 return 1
248
248
249 def writeBlock(self):
249 def writeBlock(self):
250 self.addData(self.dataOut.data_spc)
250 self.addData(self.dataOut.data_spc)
251 self.flagIsNewFile = 0
251 self.flagIsNewFile = 0
252
252
253
253
254 def __setNewBlock(self):
254 def __setNewBlock(self):
255
255
256 if self.flagIsNewFile:
256 if self.flagIsNewFile:
257 return 1
257 return 1
258
258
259 if self.blockIndex < self.dataBlocksPerFile:
259 if self.blockIndex < self.dataBlocksPerFile:
260 return 1
260 return 1
261
261
262 if not( self.setNextFile() ):
262 if not( self.setNextFile() ):
263 return 0
263 return 0
264
264
265 return 1
265 return 1
266
266
267 def writeNextBlock(self):
267 def writeNextBlock(self):
268 if not( self.__setNewBlock() ):
268 if not( self.__setNewBlock() ):
269 return 0
269 return 0
270 self.writeBlock()
270 self.writeBlock()
271 return 1
271 return 1
272
272
273 def putData(self):
273 def putData(self):
274 if self.flagIsNewFile:
274 if self.flagIsNewFile:
275 self.setNextFile()
275 self.setNextFile()
276 self.writeNextBlock()
276 self.writeNextBlock()
277
277
278 def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs):
278 def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs):
279 if not(self.isConfig):
279 if not(self.isConfig):
280 self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs)
280 self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs)
281 self.isConfig = True
281 self.isConfig = True
282 self.putData()
282 self.putData()
283
283
284
284
285 class FitsReader(ProcessingUnit):
285 class FitsReader(ProcessingUnit):
286
286
287 # __TIMEZONE = time.timezone
287 # __TIMEZONE = time.timezone
288
288
289 expName = None
289 expName = None
290 datetimestr = None
290 datetimestr = None
291 utc = None
291 utc = None
292 nChannels = None
292 nChannels = None
293 nSamples = None
293 nSamples = None
294 dataBlocksPerFile = None
294 dataBlocksPerFile = None
295 comments = None
295 comments = None
296 lastUTTime = None
296 lastUTTime = None
297 header_dict = None
297 header_dict = None
298 data = None
298 data = None
299 data_header_dict = None
299 data_header_dict = None
300
300
301 def __init__(self, **kwargs):
301 def __init__(self, **kwargs):
302 ProcessingUnit.__init__(self, **kwargs)
302 ProcessingUnit.__init__(self, **kwargs)
303 self.isConfig = False
303 self.isConfig = False
304 self.ext = '.fits'
304 self.ext = '.fits'
305 self.setFile = 0
305 self.setFile = 0
306 self.flagNoMoreFiles = 0
306 self.flagNoMoreFiles = 0
307 self.flagIsNewFile = 1
307 self.flagIsNewFile = 1
308 self.flagDiscontinuousBlock = None
308 self.flagDiscontinuousBlock = None
309 self.fileIndex = None
309 self.fileIndex = None
310 self.filename = None
310 self.filename = None
311 self.fileSize = None
311 self.fileSize = None
312 self.fitsObj = None
312 self.fitsObj = None
313 self.timeZone = None
313 self.timeZone = None
314 self.nReadBlocks = 0
314 self.nReadBlocks = 0
315 self.nTotalBlocks = 0
315 self.nTotalBlocks = 0
316 self.dataOut = self.createObjByDefault()
316 self.dataOut = self.createObjByDefault()
317 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
317 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
318 self.blockIndex = 1
318 self.blockIndex = 1
319
319
320 def createObjByDefault(self):
320 def createObjByDefault(self):
321
321
322 dataObj = Fits()
322 dataObj = Fits()
323
323
324 return dataObj
324 return dataObj
325
325
326 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
326 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
327 try:
327 try:
328 fitsObj = pyfits.open(filename,'readonly')
328 fitsObj = pyfits.open(filename,'readonly')
329 except:
329 except:
330 print "File %s can't be opened" %(filename)
330 print "File %s can't be opened" %(filename)
331 return None
331 return None
332
332
333 header = fitsObj[0].header
333 header = fitsObj[0].header
334 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
334 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
335 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
335 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
336
336
337 ltc = utc
337 ltc = utc
338 if useLocalTime:
338 if useLocalTime:
339 ltc -= time.timezone
339 ltc -= time.timezone
340 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
340 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
341 thisTime = thisDatetime.time()
341 thisTime = thisDatetime.time()
342
342
343 if not ((startTime <= thisTime) and (endTime > thisTime)):
343 if not ((startTime <= thisTime) and (endTime > thisTime)):
344 return None
344 return None
345
345
346 return thisDatetime
346 return thisDatetime
347
347
348 def __setNextFileOnline(self):
348 def __setNextFileOnline(self):
349 raise NotImplementedError
349 raise NotImplementedError
350
350
351 def __setNextFileOffline(self):
351 def __setNextFileOffline(self):
352 idFile = self.fileIndex
352 idFile = self.fileIndex
353
353
354 while (True):
354 while (True):
355 idFile += 1
355 idFile += 1
356 if not(idFile < len(self.filenameList)):
356 if not(idFile < len(self.filenameList)):
357 self.flagNoMoreFiles = 1
357 self.flagNoMoreFiles = 1
358 print "No more Files"
358 print "No more Files"
359 return 0
359 return 0
360
360
361 filename = self.filenameList[idFile]
361 filename = self.filenameList[idFile]
362
362
363 # if not(self.__verifyFile(filename)):
363 # if not(self.__verifyFile(filename)):
364 # continue
364 # continue
365
365
366 fileSize = os.path.getsize(filename)
366 fileSize = os.path.getsize(filename)
367 fitsObj = pyfits.open(filename,'readonly')
367 fitsObj = pyfits.open(filename,'readonly')
368 break
368 break
369
369
370 self.flagIsNewFile = 1
370 self.flagIsNewFile = 1
371 self.fileIndex = idFile
371 self.fileIndex = idFile
372 self.filename = filename
372 self.filename = filename
373 self.fileSize = fileSize
373 self.fileSize = fileSize
374 self.fitsObj = fitsObj
374 self.fitsObj = fitsObj
375 self.blockIndex = 0
375 self.blockIndex = 0
376 print "Setting the file: %s"%self.filename
376 print "Setting the file: %s"%self.filename
377
377
378 return 1
378 return 1
379
379
380 def __setValuesFromHeader(self):
380 def __setValuesFromHeader(self):
381
381
382 self.dataOut.header = self.header_dict
382 self.dataOut.header = self.header_dict
383 self.dataOut.expName = self.expName
383 self.dataOut.expName = self.expName
384
384
385 self.dataOut.timeZone = self.timeZone
385 self.dataOut.timeZone = self.timeZone
386 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
386 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
387 self.dataOut.comments = self.comments
387 self.dataOut.comments = self.comments
388 # self.dataOut.timeInterval = self.timeInterval
388 # self.dataOut.timeInterval = self.timeInterval
389 self.dataOut.channelList = self.channelList
389 self.dataOut.channelList = self.channelList
390 self.dataOut.heightList = self.heightList
390 self.dataOut.heightList = self.heightList
391
391
392 self.dataOut.nCohInt = self.nCohInt
392 self.dataOut.nCohInt = self.nCohInt
393 self.dataOut.nIncohInt = self.nIncohInt
393 self.dataOut.nIncohInt = self.nIncohInt
394
394
395 self.dataOut.ippSeconds = self.ippSeconds
395 self.dataOut.ippSeconds = self.ippSeconds
396
396
397 def readHeader(self):
397 def readHeader(self):
398 headerObj = self.fitsObj[0]
398 headerObj = self.fitsObj[0]
399
399
400 self.header_dict = headerObj.header
400 self.header_dict = headerObj.header
401 if 'EXPNAME' in headerObj.header.keys():
401 if 'EXPNAME' in headerObj.header.keys():
402 self.expName = headerObj.header['EXPNAME']
402 self.expName = headerObj.header['EXPNAME']
403
403
404 if 'DATATYPE' in headerObj.header.keys():
404 if 'DATATYPE' in headerObj.header.keys():
405 self.dataType = headerObj.header['DATATYPE']
405 self.dataType = headerObj.header['DATATYPE']
406
406
407 self.datetimestr = headerObj.header['DATETIME']
407 self.datetimestr = headerObj.header['DATETIME']
408 channelList = headerObj.header['CHANNELLIST']
408 channelList = headerObj.header['CHANNELLIST']
409 channelList = channelList.split('[')
409 channelList = channelList.split('[')
410 channelList = channelList[1].split(']')
410 channelList = channelList[1].split(']')
411 channelList = channelList[0].split(',')
411 channelList = channelList[0].split(',')
412 channelList = [int(ch) for ch in channelList]
412 channelList = [int(ch) for ch in channelList]
413 self.channelList = channelList
413 self.channelList = channelList
414 self.nChannels = headerObj.header['NCHANNELS']
414 self.nChannels = headerObj.header['NCHANNELS']
415 self.nHeights = headerObj.header['NHEIGHTS']
415 self.nHeights = headerObj.header['NHEIGHTS']
416 self.ippSeconds = headerObj.header['IPPSECONDS']
416 self.ippSeconds = headerObj.header['IPPSECONDS']
417 self.nCohInt = headerObj.header['NCOHINT']
417 self.nCohInt = headerObj.header['NCOHINT']
418 self.nIncohInt = headerObj.header['NINCOHINT']
418 self.nIncohInt = headerObj.header['NINCOHINT']
419 self.dataBlocksPerFile = headerObj.header['NBLOCK']
419 self.dataBlocksPerFile = headerObj.header['NBLOCK']
420 self.timeZone = headerObj.header['TIMEZONE']
420 self.timeZone = headerObj.header['TIMEZONE']
421
421
422 # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
422 # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
423
423
424 if 'COMMENT' in headerObj.header.keys():
424 if 'COMMENT' in headerObj.header.keys():
425 self.comments = headerObj.header['COMMENT']
425 self.comments = headerObj.header['COMMENT']
426
426
427 self.readHeightList()
427 self.readHeightList()
428
428
429 def readHeightList(self):
429 def readHeightList(self):
430 self.blockIndex = self.blockIndex + 1
430 self.blockIndex = self.blockIndex + 1
431 obj = self.fitsObj[self.blockIndex]
431 obj = self.fitsObj[self.blockIndex]
432 self.heightList = obj.data
432 self.heightList = obj.data
433 self.blockIndex = self.blockIndex + 1
433 self.blockIndex = self.blockIndex + 1
434
434
435 def readExtension(self):
435 def readExtension(self):
436 obj = self.fitsObj[self.blockIndex]
436 obj = self.fitsObj[self.blockIndex]
437 self.heightList = obj.data
437 self.heightList = obj.data
438 self.blockIndex = self.blockIndex + 1
438 self.blockIndex = self.blockIndex + 1
439
439
440 def setNextFile(self):
440 def setNextFile(self):
441
441
442 if self.online:
442 if self.online:
443 newFile = self.__setNextFileOnline()
443 newFile = self.__setNextFileOnline()
444 else:
444 else:
445 newFile = self.__setNextFileOffline()
445 newFile = self.__setNextFileOffline()
446
446
447 if not(newFile):
447 if not(newFile):
448 return 0
448 return 0
449
449
450 self.readHeader()
450 self.readHeader()
451 self.__setValuesFromHeader()
451 self.__setValuesFromHeader()
452 self.nReadBlocks = 0
452 self.nReadBlocks = 0
453 # self.blockIndex = 1
453 # self.blockIndex = 1
454 return 1
454 return 1
455
455
456 def __searchFilesOffLine(self,
456 def searchFilesOffLine(self,
457 path,
457 path,
458 startDate,
458 startDate,
459 endDate,
459 endDate,
460 startTime=datetime.time(0,0,0),
460 startTime=datetime.time(0,0,0),
461 endTime=datetime.time(23,59,59),
461 endTime=datetime.time(23,59,59),
462 set=None,
462 set=None,
463 expLabel='',
463 expLabel='',
464 ext='.fits',
464 ext='.fits',
465 walk=True):
465 walk=True):
466
466
467 pathList = []
467 pathList = []
468
468
469 if not walk:
469 if not walk:
470 pathList.append(path)
470 pathList.append(path)
471
471
472 else:
472 else:
473 dirList = []
473 dirList = []
474 for thisPath in os.listdir(path):
474 for thisPath in os.listdir(path):
475 if not os.path.isdir(os.path.join(path,thisPath)):
475 if not os.path.isdir(os.path.join(path,thisPath)):
476 continue
476 continue
477 if not isRadarFolder(thisPath):
477 if not isRadarFolder(thisPath):
478 continue
478 continue
479
479
480 dirList.append(thisPath)
480 dirList.append(thisPath)
481
481
482 if not(dirList):
482 if not(dirList):
483 return None, None
483 return None, None
484
484
485 thisDate = startDate
485 thisDate = startDate
486
486
487 while(thisDate <= endDate):
487 while(thisDate <= endDate):
488 year = thisDate.timetuple().tm_year
488 year = thisDate.timetuple().tm_year
489 doy = thisDate.timetuple().tm_yday
489 doy = thisDate.timetuple().tm_yday
490
490
491 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
491 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
492 if len(matchlist) == 0:
492 if len(matchlist) == 0:
493 thisDate += datetime.timedelta(1)
493 thisDate += datetime.timedelta(1)
494 continue
494 continue
495 for match in matchlist:
495 for match in matchlist:
496 pathList.append(os.path.join(path,match,expLabel))
496 pathList.append(os.path.join(path,match,expLabel))
497
497
498 thisDate += datetime.timedelta(1)
498 thisDate += datetime.timedelta(1)
499
499
500 if pathList == []:
500 if pathList == []:
501 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
501 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
502 return None, None
502 return None, None
503
503
504 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
504 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
505
505
506 filenameList = []
506 filenameList = []
507 datetimeList = []
507 datetimeList = []
508
508
509 for i in range(len(pathList)):
509 for i in range(len(pathList)):
510
510
511 thisPath = pathList[i]
511 thisPath = pathList[i]
512
512
513 fileList = glob.glob1(thisPath, "*%s" %ext)
513 fileList = glob.glob1(thisPath, "*%s" %ext)
514 fileList.sort()
514 fileList.sort()
515
515
516 for thisFile in fileList:
516 for thisFile in fileList:
517
517
518 filename = os.path.join(thisPath,thisFile)
518 filename = os.path.join(thisPath,thisFile)
519 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
519 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
520
520
521 if not(thisDatetime):
521 if not(thisDatetime):
522 continue
522 continue
523
523
524 filenameList.append(filename)
524 filenameList.append(filename)
525 datetimeList.append(thisDatetime)
525 datetimeList.append(thisDatetime)
526
526
527 if not(filenameList):
527 if not(filenameList):
528 print "Any file was found for the time range %s - %s" %(startTime, endTime)
528 print "Any file was found for the time range %s - %s" %(startTime, endTime)
529 return None, None
529 return None, None
530
530
531 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
531 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
532 print
532 print
533
533
534 for i in range(len(filenameList)):
534 for i in range(len(filenameList)):
535 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
535 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
536
536
537 self.filenameList = filenameList
537 self.filenameList = filenameList
538 self.datetimeList = datetimeList
538 self.datetimeList = datetimeList
539
539
540 return pathList, filenameList
540 return pathList, filenameList
541
541
542 def setup(self, path=None,
542 def setup(self, path=None,
543 startDate=None,
543 startDate=None,
544 endDate=None,
544 endDate=None,
545 startTime=datetime.time(0,0,0),
545 startTime=datetime.time(0,0,0),
546 endTime=datetime.time(23,59,59),
546 endTime=datetime.time(23,59,59),
547 set=0,
547 set=0,
548 expLabel = "",
548 expLabel = "",
549 ext = None,
549 ext = None,
550 online = False,
550 online = False,
551 delay = 60,
551 delay = 60,
552 walk = True):
552 walk = True):
553
553
554 if path == None:
554 if path == None:
555 raise ValueError, "The path is not valid"
555 raise ValueError, "The path is not valid"
556
556
557 if ext == None:
557 if ext == None:
558 ext = self.ext
558 ext = self.ext
559
559
560 if not(online):
560 if not(online):
561 print "Searching files in offline mode ..."
561 print "Searching files in offline mode ..."
562 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
562 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 startTime=startTime, endTime=endTime,
563 startTime=startTime, endTime=endTime,
564 set=set, expLabel=expLabel, ext=ext,
564 set=set, expLabel=expLabel, ext=ext,
565 walk=walk)
565 walk=walk)
566
566
567 if not(pathList):
567 if not(pathList):
568 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
568 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
569 datetime.datetime.combine(startDate,startTime).ctime(),
569 datetime.datetime.combine(startDate,startTime).ctime(),
570 datetime.datetime.combine(endDate,endTime).ctime())
570 datetime.datetime.combine(endDate,endTime).ctime())
571
571
572 sys.exit(-1)
572 sys.exit(-1)
573
573
574 self.fileIndex = -1
574 self.fileIndex = -1
575 self.pathList = pathList
575 self.pathList = pathList
576 self.filenameList = filenameList
576 self.filenameList = filenameList
577
577
578 self.online = online
578 self.online = online
579 self.delay = delay
579 self.delay = delay
580 ext = ext.lower()
580 ext = ext.lower()
581 self.ext = ext
581 self.ext = ext
582
582
583 if not(self.setNextFile()):
583 if not(self.setNextFile()):
584 if (startDate!=None) and (endDate!=None):
584 if (startDate!=None) and (endDate!=None):
585 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
585 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
586 elif startDate != None:
586 elif startDate != None:
587 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
587 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
588 else:
588 else:
589 print "No files"
589 print "No files"
590
590
591 sys.exit(-1)
591 sys.exit(-1)
592
592
593
593
594
594
595 def readBlock(self):
595 def readBlock(self):
596 dataObj = self.fitsObj[self.blockIndex]
596 dataObj = self.fitsObj[self.blockIndex]
597
597
598 self.data = dataObj.data
598 self.data = dataObj.data
599 self.data_header_dict = dataObj.header
599 self.data_header_dict = dataObj.header
600 self.utc = self.data_header_dict['UTCTIME']
600 self.utc = self.data_header_dict['UTCTIME']
601
601
602 self.flagIsNewFile = 0
602 self.flagIsNewFile = 0
603 self.blockIndex += 1
603 self.blockIndex += 1
604 self.nTotalBlocks += 1
604 self.nTotalBlocks += 1
605 self.nReadBlocks += 1
605 self.nReadBlocks += 1
606
606
607 return 1
607 return 1
608
608
609 def __jumpToLastBlock(self):
609 def __jumpToLastBlock(self):
610 raise NotImplementedError
610 raise NotImplementedError
611
611
612 def __waitNewBlock(self):
612 def __waitNewBlock(self):
613 """
613 """
614 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
614 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
615
615
616 Si el modo de lectura es OffLine siempre retorn 0
616 Si el modo de lectura es OffLine siempre retorn 0
617 """
617 """
618 if not self.online:
618 if not self.online:
619 return 0
619 return 0
620
620
621 if (self.nReadBlocks >= self.dataBlocksPerFile):
621 if (self.nReadBlocks >= self.dataBlocksPerFile):
622 return 0
622 return 0
623
623
624 currentPointer = self.fp.tell()
624 currentPointer = self.fp.tell()
625
625
626 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
626 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
627
627
628 for nTries in range( self.nTries ):
628 for nTries in range( self.nTries ):
629
629
630 self.fp.close()
630 self.fp.close()
631 self.fp = open( self.filename, 'rb' )
631 self.fp = open( self.filename, 'rb' )
632 self.fp.seek( currentPointer )
632 self.fp.seek( currentPointer )
633
633
634 self.fileSize = os.path.getsize( self.filename )
634 self.fileSize = os.path.getsize( self.filename )
635 currentSize = self.fileSize - currentPointer
635 currentSize = self.fileSize - currentPointer
636
636
637 if ( currentSize >= neededSize ):
637 if ( currentSize >= neededSize ):
638 self.__rdBasicHeader()
638 self.__rdBasicHeader()
639 return 1
639 return 1
640
640
641 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
641 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
642 sleep( self.delay )
642 sleep( self.delay )
643
643
644
644
645 return 0
645 return 0
646
646
647 def __setNewBlock(self):
647 def __setNewBlock(self):
648
648
649 if self.online:
649 if self.online:
650 self.__jumpToLastBlock()
650 self.__jumpToLastBlock()
651
651
652 if self.flagIsNewFile:
652 if self.flagIsNewFile:
653 return 1
653 return 1
654
654
655 self.lastUTTime = self.utc
655 self.lastUTTime = self.utc
656
656
657 if self.online:
657 if self.online:
658 if self.__waitNewBlock():
658 if self.__waitNewBlock():
659 return 1
659 return 1
660
660
661 if self.nReadBlocks < self.dataBlocksPerFile:
661 if self.nReadBlocks < self.dataBlocksPerFile:
662 return 1
662 return 1
663
663
664 if not(self.setNextFile()):
664 if not(self.setNextFile()):
665 return 0
665 return 0
666
666
667 deltaTime = self.utc - self.lastUTTime
667 deltaTime = self.utc - self.lastUTTime
668
668
669 self.flagDiscontinuousBlock = 0
669 self.flagDiscontinuousBlock = 0
670
670
671 if deltaTime > self.maxTimeStep:
671 if deltaTime > self.maxTimeStep:
672 self.flagDiscontinuousBlock = 1
672 self.flagDiscontinuousBlock = 1
673
673
674 return 1
674 return 1
675
675
676
676
677 def readNextBlock(self):
677 def readNextBlock(self):
678 if not(self.__setNewBlock()):
678 if not(self.__setNewBlock()):
679 return 0
679 return 0
680
680
681 if not(self.readBlock()):
681 if not(self.readBlock()):
682 return 0
682 return 0
683
683
684 return 1
684 return 1
685
685
686 def printInfo(self):
686 def printInfo(self):
687
687
688 pass
688 pass
689
689
690 def getData(self):
690 def getData(self):
691
691
692 if self.flagNoMoreFiles:
692 if self.flagNoMoreFiles:
693 self.dataOut.flagNoData = True
693 self.dataOut.flagNoData = True
694 print 'Process finished'
694 print 'Process finished'
695 return 0
695 return 0
696
696
697 self.flagDiscontinuousBlock = 0
697 self.flagDiscontinuousBlock = 0
698 self.flagIsNewBlock = 0
698 self.flagIsNewBlock = 0
699
699
700 if not(self.readNextBlock()):
700 if not(self.readNextBlock()):
701 return 0
701 return 0
702
702
703 if self.data is None:
703 if self.data is None:
704 self.dataOut.flagNoData = True
704 self.dataOut.flagNoData = True
705 return 0
705 return 0
706
706
707 self.dataOut.data = self.data
707 self.dataOut.data = self.data
708 self.dataOut.data_header = self.data_header_dict
708 self.dataOut.data_header = self.data_header_dict
709 self.dataOut.utctime = self.utc
709 self.dataOut.utctime = self.utc
710
710
711 # self.dataOut.header = self.header_dict
711 # self.dataOut.header = self.header_dict
712 # self.dataOut.expName = self.expName
712 # self.dataOut.expName = self.expName
713 # self.dataOut.nChannels = self.nChannels
713 # self.dataOut.nChannels = self.nChannels
714 # self.dataOut.timeZone = self.timeZone
714 # self.dataOut.timeZone = self.timeZone
715 # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
715 # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
716 # self.dataOut.comments = self.comments
716 # self.dataOut.comments = self.comments
717 # # self.dataOut.timeInterval = self.timeInterval
717 # # self.dataOut.timeInterval = self.timeInterval
718 # self.dataOut.channelList = self.channelList
718 # self.dataOut.channelList = self.channelList
719 # self.dataOut.heightList = self.heightList
719 # self.dataOut.heightList = self.heightList
720 self.dataOut.flagNoData = False
720 self.dataOut.flagNoData = False
721
721
722 return self.dataOut.data
722 return self.dataOut.data
723
723
724 def run(self, **kwargs):
724 def run(self, **kwargs):
725
725
726 if not(self.isConfig):
726 if not(self.isConfig):
727 self.setup(**kwargs)
727 self.setup(**kwargs)
728 self.isConfig = True
728 self.isConfig = True
729
729
730 self.getData()
730 self.getData()
731
731
732 class SpectraHeisWriter(Operation):
732 class SpectraHeisWriter(Operation):
733 # set = None
733 # set = None
734 setFile = None
734 setFile = None
735 idblock = None
735 idblock = None
736 doypath = None
736 doypath = None
737 subfolder = None
737 subfolder = None
738
738
739 def __init__(self, **kwargs):
739 def __init__(self, **kwargs):
740 Operation.__init__(self, **kwargs)
740 Operation.__init__(self, **kwargs)
741 self.wrObj = PyFits()
741 self.wrObj = PyFits()
742 # self.dataOut = dataOut
742 # self.dataOut = dataOut
743 self.nTotalBlocks=0
743 self.nTotalBlocks=0
744 # self.set = None
744 # self.set = None
745 self.setFile = None
745 self.setFile = None
746 self.idblock = 0
746 self.idblock = 0
747 self.wrpath = None
747 self.wrpath = None
748 self.doypath = None
748 self.doypath = None
749 self.subfolder = None
749 self.subfolder = None
750 self.isConfig = False
750 self.isConfig = False
751
751
752 def isNumber(str):
752 def isNumber(str):
753 """
753 """
754 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
754 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
755
755
756 Excepciones:
756 Excepciones:
757 Si un determinado string no puede ser convertido a numero
757 Si un determinado string no puede ser convertido a numero
758 Input:
758 Input:
759 str, string al cual se le analiza para determinar si convertible a un numero o no
759 str, string al cual se le analiza para determinar si convertible a un numero o no
760
760
761 Return:
761 Return:
762 True : si el string es uno numerico
762 True : si el string es uno numerico
763 False : no es un string numerico
763 False : no es un string numerico
764 """
764 """
765 try:
765 try:
766 float( str )
766 float( str )
767 return True
767 return True
768 except:
768 except:
769 return False
769 return False
770
770
771 def setup(self, dataOut, wrpath):
771 def setup(self, dataOut, wrpath):
772
772
773 if not(os.path.exists(wrpath)):
773 if not(os.path.exists(wrpath)):
774 os.mkdir(wrpath)
774 os.mkdir(wrpath)
775
775
776 self.wrpath = wrpath
776 self.wrpath = wrpath
777 # self.setFile = 0
777 # self.setFile = 0
778 self.dataOut = dataOut
778 self.dataOut = dataOut
779
779
780 def putData(self):
780 def putData(self):
781 name= time.localtime( self.dataOut.utctime)
781 name= time.localtime( self.dataOut.utctime)
782 ext=".fits"
782 ext=".fits"
783
783
784 if self.doypath == None:
784 if self.doypath == None:
785 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
785 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
786 self.doypath = os.path.join( self.wrpath, self.subfolder )
786 self.doypath = os.path.join( self.wrpath, self.subfolder )
787 os.mkdir(self.doypath)
787 os.mkdir(self.doypath)
788
788
789 if self.setFile == None:
789 if self.setFile == None:
790 # self.set = self.dataOut.set
790 # self.set = self.dataOut.set
791 self.setFile = 0
791 self.setFile = 0
792 # if self.set != self.dataOut.set:
792 # if self.set != self.dataOut.set:
793 ## self.set = self.dataOut.set
793 ## self.set = self.dataOut.set
794 # self.setFile = 0
794 # self.setFile = 0
795
795
796 #make the filename
796 #make the filename
797 thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
797 thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
798
798
799 filename = os.path.join(self.wrpath,self.subfolder, thisFile)
799 filename = os.path.join(self.wrpath,self.subfolder, thisFile)
800
800
801 idblock = numpy.array([self.idblock],dtype="int64")
801 idblock = numpy.array([self.idblock],dtype="int64")
802 header=self.wrObj.cFImage(idblock=idblock,
802 header=self.wrObj.cFImage(idblock=idblock,
803 year=time.gmtime(self.dataOut.utctime).tm_year,
803 year=time.gmtime(self.dataOut.utctime).tm_year,
804 month=time.gmtime(self.dataOut.utctime).tm_mon,
804 month=time.gmtime(self.dataOut.utctime).tm_mon,
805 day=time.gmtime(self.dataOut.utctime).tm_mday,
805 day=time.gmtime(self.dataOut.utctime).tm_mday,
806 hour=time.gmtime(self.dataOut.utctime).tm_hour,
806 hour=time.gmtime(self.dataOut.utctime).tm_hour,
807 minute=time.gmtime(self.dataOut.utctime).tm_min,
807 minute=time.gmtime(self.dataOut.utctime).tm_min,
808 second=time.gmtime(self.dataOut.utctime).tm_sec)
808 second=time.gmtime(self.dataOut.utctime).tm_sec)
809
809
810 c=3E8
810 c=3E8
811 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
811 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
812 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
812 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
813
813
814 colList = []
814 colList = []
815
815
816 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
816 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
817
817
818 colList.append(colFreq)
818 colList.append(colFreq)
819
819
820 nchannel=self.dataOut.nChannels
820 nchannel=self.dataOut.nChannels
821
821
822 for i in range(nchannel):
822 for i in range(nchannel):
823 col = self.wrObj.writeData(name="PCh"+str(i+1),
823 col = self.wrObj.writeData(name="PCh"+str(i+1),
824 format=str(self.dataOut.nFFTPoints)+'E',
824 format=str(self.dataOut.nFFTPoints)+'E',
825 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
825 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
826
826
827 colList.append(col)
827 colList.append(col)
828
828
829 data=self.wrObj.Ctable(colList=colList)
829 data=self.wrObj.Ctable(colList=colList)
830
830
831 self.wrObj.CFile(header,data)
831 self.wrObj.CFile(header,data)
832
832
833 self.wrObj.wFile(filename)
833 self.wrObj.wFile(filename)
834
834
835 #update the setFile
835 #update the setFile
836 self.setFile += 1
836 self.setFile += 1
837 self.idblock += 1
837 self.idblock += 1
838
838
839 return 1
839 return 1
840
840
841 def run(self, dataOut, **kwargs):
841 def run(self, dataOut, **kwargs):
842
842
843 if not(self.isConfig):
843 if not(self.isConfig):
844
844
845 self.setup(dataOut, **kwargs)
845 self.setup(dataOut, **kwargs)
846 self.isConfig = True
846 self.isConfig = True
847
847
848 self.putData()
848 self.putData()
Show file before | Show file after | Hide comments
@@ -1,863 +1,863
1 '''
1 '''
2 Created on Jul 3, 2014
2 Created on Jul 3, 2014
3
3
4 @author: roj-com0419
4 @author: roj-com0419
5 '''
5 '''
6
6
7 import os,sys
7 import os,sys
8 import time,datetime
8 import time,datetime
9 import h5py
9 import h5py
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import re
12 import re
13
13
14 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
14 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 from schainpy.model.data.jrodata import Voltage
15 from schainpy.model.data.jrodata import Voltage
16 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
16 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
17
17
18
18
19 def isNumber(str):
19 def isNumber(str):
20 """
20 """
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22
22
23 Excepciones:
23 Excepciones:
24 Si un determinado string no puede ser convertido a numero
24 Si un determinado string no puede ser convertido a numero
25 Input:
25 Input:
26 str, string al cual se le analiza para determinar si convertible a un numero o no
26 str, string al cual se le analiza para determinar si convertible a un numero o no
27
27
28 Return:
28 Return:
29 True : si el string es uno numerico
29 True : si el string es uno numerico
30 False : no es un string numerico
30 False : no es un string numerico
31 """
31 """
32 try:
32 try:
33 float( str )
33 float( str )
34 return True
34 return True
35 except:
35 except:
36 return False
36 return False
37
37
38 def getFileFromSet(path, ext, set=None):
38 def getFileFromSet(path, ext, set=None):
39 validFilelist = []
39 validFilelist = []
40 fileList = os.listdir(path)
40 fileList = os.listdir(path)
41
41
42
42
43 if len(fileList) < 1:
43 if len(fileList) < 1:
44 return None
44 return None
45
45
46 # 0 1234 567 89A BCDE
46 # 0 1234 567 89A BCDE
47 # H YYYY DDD SSS .ext
47 # H YYYY DDD SSS .ext
48
48
49 for thisFile in fileList:
49 for thisFile in fileList:
50 try:
50 try:
51 number= int(thisFile[6:16])
51 number= int(thisFile[6:16])
52
52
53 # year = int(thisFile[1:5])
53 # year = int(thisFile[1:5])
54 # doy = int(thisFile[5:8])
54 # doy = int(thisFile[5:8])
55 except:
55 except:
56 continue
56 continue
57
57
58 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
58 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
59 continue
59 continue
60
60
61 validFilelist.append(thisFile)
61 validFilelist.append(thisFile)
62
62
63 if len(validFilelist) < 1:
63 if len(validFilelist) < 1:
64 return None
64 return None
65
65
66 validFilelist = sorted( validFilelist, key=str.lower )
66 validFilelist = sorted( validFilelist, key=str.lower )
67
67
68 if set == None:
68 if set == None:
69 return validFilelist[-1]
69 return validFilelist[-1]
70
70
71 print "set =" ,set
71 print "set =" ,set
72 for thisFile in validFilelist:
72 for thisFile in validFilelist:
73 if set <= int(thisFile[6:16]):
73 if set <= int(thisFile[6:16]):
74 print thisFile,int(thisFile[6:16])
74 print thisFile,int(thisFile[6:16])
75 return thisFile
75 return thisFile
76
76
77 return validFilelist[-1]
77 return validFilelist[-1]
78
78
79 myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
79 myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
80 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
80 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
81
81
82 if len(myfile)!= 0:
82 if len(myfile)!= 0:
83 return myfile[0]
83 return myfile[0]
84 else:
84 else:
85 filename = '*%10.10d%s'%(set,ext.lower())
85 filename = '*%10.10d%s'%(set,ext.lower())
86 print 'the filename %s does not exist'%filename
86 print 'the filename %s does not exist'%filename
87 print '...going to the last file: '
87 print '...going to the last file: '
88
88
89 if validFilelist:
89 if validFilelist:
90 validFilelist = sorted( validFilelist, key=str.lower )
90 validFilelist = sorted( validFilelist, key=str.lower )
91 return validFilelist[-1]
91 return validFilelist[-1]
92
92
93 return None
93 return None
94
94
95 def getlastFileFromPath(path, ext):
95 def getlastFileFromPath(path, ext):
96 """
96 """
97 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
97 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
98 al final de la depuracion devuelve el ultimo file de la lista que quedo.
98 al final de la depuracion devuelve el ultimo file de la lista que quedo.
99
99
100 Input:
100 Input:
101 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
101 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
102 ext : extension de los files contenidos en una carpeta
102 ext : extension de los files contenidos en una carpeta
103
103
104 Return:
104 Return:
105 El ultimo file de una determinada carpeta, no se considera el path.
105 El ultimo file de una determinada carpeta, no se considera el path.
106 """
106 """
107 validFilelist = []
107 validFilelist = []
108 fileList = os.listdir(path)
108 fileList = os.listdir(path)
109
109
110 # 0 1234 567 89A BCDE
110 # 0 1234 567 89A BCDE
111 # H YYYY DDD SSS .ext
111 # H YYYY DDD SSS .ext
112
112
113 for thisFile in fileList:
113 for thisFile in fileList:
114
114
115 try:
115 try:
116 number= int(thisFile[6:16])
116 number= int(thisFile[6:16])
117 except:
117 except:
118 print "There is a file or folder with different format"
118 print "There is a file or folder with different format"
119 if not isNumber(number):
119 if not isNumber(number):
120 continue
120 continue
121
121
122 # year = thisFile[1:5]
122 # year = thisFile[1:5]
123 # if not isNumber(year):
123 # if not isNumber(year):
124 # continue
124 # continue
125
125
126 # doy = thisFile[5:8]
126 # doy = thisFile[5:8]
127 # if not isNumber(doy):
127 # if not isNumber(doy):
128 # continue
128 # continue
129
129
130 number= int(number)
130 number= int(number)
131 # year = int(year)
131 # year = int(year)
132 # doy = int(doy)
132 # doy = int(doy)
133
133
134 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
134 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
135 continue
135 continue
136
136
137
137
138 validFilelist.append(thisFile)
138 validFilelist.append(thisFile)
139
139
140
140
141 if validFilelist:
141 if validFilelist:
142 validFilelist = sorted( validFilelist, key=str.lower )
142 validFilelist = sorted( validFilelist, key=str.lower )
143 return validFilelist[-1]
143 return validFilelist[-1]
144
144
145 return None
145 return None
146
146
147
147
148
148
149 class HFReader(ProcessingUnit):
149 class HFReader(ProcessingUnit):
150 '''
150 '''
151 classdocs
151 classdocs
152 '''
152 '''
153 path = None
153 path = None
154 startDate= None
154 startDate= None
155 endDate = None
155 endDate = None
156 startTime= None
156 startTime= None
157 endTime = None
157 endTime = None
158 walk = None
158 walk = None
159 isConfig = False
159 isConfig = False
160 dataOut=None
160 dataOut=None
161 nTries = 3
161 nTries = 3
162 ext = ".hdf5"
162 ext = ".hdf5"
163
163
164 def __init__(self, **kwargs):
164 def __init__(self, **kwargs):
165 '''
165 '''
166 Constructor
166 Constructor
167 '''
167 '''
168 ProcessingUnit.__init__(self, **kwargs)
168 ProcessingUnit.__init__(self, **kwargs)
169
169
170 self.isConfig =False
170 self.isConfig =False
171
171
172 self.datablock = None
172 self.datablock = None
173
173
174 self.filename_current=None
174 self.filename_current=None
175
175
176 self.utc = 0
176 self.utc = 0
177
177
178 self.ext='.hdf5'
178 self.ext='.hdf5'
179
179
180 self.flagIsNewFile = 1
180 self.flagIsNewFile = 1
181
181
182 #-------------------------------------------------
182 #-------------------------------------------------
183 self.fileIndex=None
183 self.fileIndex=None
184
184
185 self.profileIndex_offset=None
185 self.profileIndex_offset=None
186
186
187 self.filenameList=[]
187 self.filenameList=[]
188
188
189 self.hfFilePointer= None
189 self.hfFilePointer= None
190
190
191 self.filename_online = None
191 self.filename_online = None
192
192
193 self.status=True
193 self.status=True
194
194
195 self.flagNoMoreFiles= False
195 self.flagNoMoreFiles= False
196
196
197 self.__waitForNewFile = 20
197 self.__waitForNewFile = 20
198
198
199
199
200 #--------------------------------------------------
200 #--------------------------------------------------
201
201
202 self.dataOut = self.createObjByDefault()
202 self.dataOut = self.createObjByDefault()
203
203
204
204
205 def createObjByDefault(self):
205 def createObjByDefault(self):
206
206
207 dataObj = Voltage()
207 dataObj = Voltage()
208
208
209 return dataObj
209 return dataObj
210
210
211 def setObjProperties(self):
211 def setObjProperties(self):
212
212
213 pass
213 pass
214
214
215 def getBlockDimension(self):
215 def getBlockDimension(self):
216 """
216 """
217 Obtiene la cantidad de puntos a leer por cada bloque de datos
217 Obtiene la cantidad de puntos a leer por cada bloque de datos
218
218
219 Affected:
219 Affected:
220 self.blocksize
220 self.blocksize
221
221
222 Return:
222 Return:
223 None
223 None
224 """
224 """
225 pts2read =self.nChannels*self.nHeights*self.nProfiles
225 pts2read =self.nChannels*self.nHeights*self.nProfiles
226 self.blocksize = pts2read
226 self.blocksize = pts2read
227
227
228 def __readHeader(self):
228 def __readHeader(self):
229
229
230 self.nProfiles = 100
230 self.nProfiles = 100
231 self.nHeights = 1000
231 self.nHeights = 1000
232 self.nChannels = 2
232 self.nChannels = 2
233 self.__firstHeigth=0
233 self.__firstHeigth=0
234 self.__nSamples=1000
234 self.__nSamples=1000
235 self.__deltaHeigth=1.5
235 self.__deltaHeigth=1.5
236 self.__sample_rate=1e5
236 self.__sample_rate=1e5
237 #self.__frequency=2.72e6
237 #self.__frequency=2.72e6
238 #self.__frequency=3.64e6
238 #self.__frequency=3.64e6
239 self.__frequency=None
239 self.__frequency=None
240 self.__online = False
240 self.__online = False
241 self.filename_next_set=None
241 self.filename_next_set=None
242
242
243 #print "Frequency of Operation:", self.__frequency
243 #print "Frequency of Operation:", self.__frequency
244
244
245
245
246 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
246 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
247 self.path = path
247 self.path = path
248 self.startDate = startDate
248 self.startDate = startDate
249 self.endDate = endDate
249 self.endDate = endDate
250 self.startTime = startTime
250 self.startTime = startTime
251 self.endTime = endTime
251 self.endTime = endTime
252 self.walk = walk
252 self.walk = walk
253
253
254 def __checkPath(self):
254 def __checkPath(self):
255 if os.path.exists(self.path):
255 if os.path.exists(self.path):
256 self.status=1
256 self.status=1
257 else:
257 else:
258 self.status=0
258 self.status=0
259 print 'Path %s does not exits'%self.path
259 print 'Path %s does not exits'%self.path
260 return
260 return
261 return
261 return
262
262
263 def __selDates(self, hf_dirname_format):
263 def __selDates(self, hf_dirname_format):
264 try:
264 try:
265 dir_hf_filename= self.path+"/"+hf_dirname_format
265 dir_hf_filename= self.path+"/"+hf_dirname_format
266 fp= h5py.File(dir_hf_filename,'r')
266 fp= h5py.File(dir_hf_filename,'r')
267 hipoc=fp['t'].value
267 hipoc=fp['t'].value
268 fp.close()
268 fp.close()
269 date_time=datetime.datetime.utcfromtimestamp(hipoc)
269 date_time=datetime.datetime.utcfromtimestamp(hipoc)
270 year =int(date_time[0:4])
270 year =int(date_time[0:4])
271 month=int(date_time[5:7])
271 month=int(date_time[5:7])
272 dom =int(date_time[8:10])
272 dom =int(date_time[8:10])
273 thisDate= datetime.date(year,month,dom)
273 thisDate= datetime.date(year,month,dom)
274 if (thisDate>=self.startDate and thisDate <= self.endDate):
274 if (thisDate>=self.startDate and thisDate <= self.endDate):
275 return hf_dirname_format
275 return hf_dirname_format
276 except:
276 except:
277 return None
277 return None
278
278
279 def __findDataForDates(self,online=False):
279 def __findDataForDates(self,online=False):
280 if not(self.status):
280 if not(self.status):
281 return None
281 return None
282
282
283 pat = '\d+.\d+'
283 pat = '\d+.\d+'
284 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
284 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
285 dirnameList = filter(lambda x:x!=None,dirnameList)
285 dirnameList = filter(lambda x:x!=None,dirnameList)
286 dirnameList = [x.string for x in dirnameList]
286 dirnameList = [x.string for x in dirnameList]
287 if not(online):
287 if not(online):
288
288
289 dirnameList = [self.__selDates(x) for x in dirnameList]
289 dirnameList = [self.__selDates(x) for x in dirnameList]
290 dirnameList = filter(lambda x:x!=None,dirnameList)
290 dirnameList = filter(lambda x:x!=None,dirnameList)
291
291
292 if len(dirnameList)>0:
292 if len(dirnameList)>0:
293 self.status = 1
293 self.status = 1
294 self.dirnameList = dirnameList
294 self.dirnameList = dirnameList
295 self.dirnameList.sort()
295 self.dirnameList.sort()
296
296
297 else:
297 else:
298 self.status = 0
298 self.status = 0
299 return None
299 return None
300
300
301 def __getTimeFromData(self):
301 def __getTimeFromData(self):
302 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
302 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
303 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
303 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
304 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
304 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
305 print '........................................'
305 print '........................................'
306 filter_filenameList=[]
306 filter_filenameList=[]
307 self.filenameList.sort()
307 self.filenameList.sort()
308 for i in range(len(self.filenameList)-1):
308 for i in range(len(self.filenameList)-1):
309 filename=self.filenameList[i]
309 filename=self.filenameList[i]
310 dir_hf_filename= filename
310 dir_hf_filename= filename
311 fp= h5py.File(dir_hf_filename,'r')
311 fp= h5py.File(dir_hf_filename,'r')
312 hipoc=fp['t'].value
312 hipoc=fp['t'].value
313 hipoc=hipoc+self.timezone
313 hipoc=hipoc+self.timezone
314 date_time=datetime.datetime.utcfromtimestamp(hipoc)
314 date_time=datetime.datetime.utcfromtimestamp(hipoc)
315 fp.close()
315 fp.close()
316 year =int(date_time[0:4])
316 year =int(date_time[0:4])
317 month=int(date_time[5:7])
317 month=int(date_time[5:7])
318 dom =int(date_time[8:10])
318 dom =int(date_time[8:10])
319 hour =int(date_time[11:13])
319 hour =int(date_time[11:13])
320 min =int(date_time[14:16])
320 min =int(date_time[14:16])
321 sec =int(date_time[17:19])
321 sec =int(date_time[17:19])
322 this_time=datetime.datetime(year,month,dom,hour,min,sec)
322 this_time=datetime.datetime(year,month,dom,hour,min,sec)
323 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
323 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
324 filter_filenameList.append(filename)
324 filter_filenameList.append(filename)
325 filter_filenameList.sort()
325 filter_filenameList.sort()
326 self.filenameList = filter_filenameList
326 self.filenameList = filter_filenameList
327 return 1
327 return 1
328
328
329 def __getFilenameList(self):
329 def __getFilenameList(self):
330 #print "hola"
330 #print "hola"
331 #print self.dirnameList
331 #print self.dirnameList
332 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
332 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
333 self.filenameList= dirList
333 self.filenameList= dirList
334 #print self.filenameList
334 #print self.filenameList
335 #print "pase",len(self.filenameList)
335 #print "pase",len(self.filenameList)
336
336
337 def __selectDataForTimes(self, online=False):
337 def __selectDataForTimes(self, online=False):
338
338
339 if not(self.status):
339 if not(self.status):
340 return None
340 return None
341 #----------------
341 #----------------
342 self.__getFilenameList()
342 self.__getFilenameList()
343 #----------------
343 #----------------
344 if not(online):
344 if not(online):
345 if not(self.all):
345 if not(self.all):
346 self.__getTimeFromData()
346 self.__getTimeFromData()
347 if len(self.filenameList)>0:
347 if len(self.filenameList)>0:
348 self.status=1
348 self.status=1
349 self.filenameList.sort()
349 self.filenameList.sort()
350 else:
350 else:
351 self.status=0
351 self.status=0
352 return None
352 return None
353 else:
353 else:
354 if self.set != None:
354 if self.set != None:
355
355
356 filename=getFileFromSet(self.path,self.ext,self.set)
356 filename=getFileFromSet(self.path,self.ext,self.set)
357
357
358 if self.flag_nextfile==True:
358 if self.flag_nextfile==True:
359 self.dirnameList=[filename]
359 self.dirnameList=[filename]
360 fullfilename=self.path+"/"+filename
360 fullfilename=self.path+"/"+filename
361 self.filenameList=[fullfilename]
361 self.filenameList=[fullfilename]
362 self.filename_next_set=int(filename[6:16])+10
362 self.filename_next_set=int(filename[6:16])+10
363
363
364 self.flag_nextfile=False
364 self.flag_nextfile=False
365 else:
365 else:
366 print filename
366 print filename
367 print "PRIMERA CONDICION"
367 print "PRIMERA CONDICION"
368 #if self.filename_next_set== int(filename[6:16]):
368 #if self.filename_next_set== int(filename[6:16]):
369 print "TODO BIEN"
369 print "TODO BIEN"
370
370
371 if filename == None:
371 if filename == None:
372 raise ValueError, "corregir"
372 raise ValueError, "corregir"
373
373
374 self.dirnameList=[filename]
374 self.dirnameList=[filename]
375 fullfilename=self.path+"/"+filename
375 fullfilename=self.path+"/"+filename
376 self.filenameList=[fullfilename]
376 self.filenameList=[fullfilename]
377 self.filename_next_set=int(filename[6:16])+10
377 self.filename_next_set=int(filename[6:16])+10
378 print "Setting next file",self.filename_next_set
378 print "Setting next file",self.filename_next_set
379 self.set=int(filename[6:16])
379 self.set=int(filename[6:16])
380 if True:
380 if True:
381 pass
381 pass
382 else:
382 else:
383 print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
383 print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
384
384
385 else:
385 else:
386 filename =getlastFileFromPath(self.path,self.ext)
386 filename =getlastFileFromPath(self.path,self.ext)
387
387
388 if self.flag_nextfile==True:
388 if self.flag_nextfile==True:
389 self.dirnameList=[filename]
389 self.dirnameList=[filename]
390 fullfilename=self.path+"/"+filename
390 fullfilename=self.path+"/"+filename
391 self.filenameList=[self.filenameList[-1]]
391 self.filenameList=[self.filenameList[-1]]
392 self.filename_next_set=int(filename[6:16])+10
392 self.filename_next_set=int(filename[6:16])+10
393
393
394 self.flag_nextfile=False
394 self.flag_nextfile=False
395 else:
395 else:
396 filename=getFileFromSet(self.path,self.ext,self.set)
396 filename=getFileFromSet(self.path,self.ext,self.set)
397 print filename
397 print filename
398 print "PRIMERA CONDICION"
398 print "PRIMERA CONDICION"
399 #if self.filename_next_set== int(filename[6:16]):
399 #if self.filename_next_set== int(filename[6:16]):
400 print "TODO BIEN"
400 print "TODO BIEN"
401
401
402 if filename == None:
402 if filename == None:
403 raise ValueError, "corregir"
403 raise ValueError, "corregir"
404
404
405 self.dirnameList=[filename]
405 self.dirnameList=[filename]
406 fullfilename=self.path+"/"+filename
406 fullfilename=self.path+"/"+filename
407 self.filenameList=[fullfilename]
407 self.filenameList=[fullfilename]
408 self.filename_next_set=int(filename[6:16])+10
408 self.filename_next_set=int(filename[6:16])+10
409 print "Setting next file",self.filename_next_set
409 print "Setting next file",self.filename_next_set
410 self.set=int(filename[6:16])
410 self.set=int(filename[6:16])
411 if True:
411 if True:
412 pass
412 pass
413 else:
413 else:
414 print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
414 print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
415
415
416
416
417
417
418 def __searchFilesOffline(self,
418 def searchFilesOffLine(self,
419 path,
419 path,
420 startDate,
420 startDate,
421 endDate,
421 endDate,
422 ext,
422 ext,
423 startTime=datetime.time(0,0,0),
423 startTime=datetime.time(0,0,0),
424 endTime=datetime.time(23,59,59),
424 endTime=datetime.time(23,59,59),
425 walk=True):
425 walk=True):
426
426
427 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
427 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
428
428
429 self.__checkPath()
429 self.__checkPath()
430
430
431 self.__findDataForDates()
431 self.__findDataForDates()
432 #print self.dirnameList
432 #print self.dirnameList
433
433
434 self.__selectDataForTimes()
434 self.__selectDataForTimes()
435
435
436 for i in range(len(self.filenameList)):
436 for i in range(len(self.filenameList)):
437 print "%s"% (self.filenameList[i])
437 print "%s"% (self.filenameList[i])
438
438
439 return
439 return
440
440
441 def __searchFilesOnline(self,
441 def searchFilesOnLine(self,
442 path,
442 path,
443 expLabel= "",
443 expLabel= "",
444 ext=None,
444 ext=None,
445 startDate=None,
445 startDate=None,
446 endDate=None,
446 endDate=None,
447 walk=True,
447 walk=True,
448 set=None):
448 set=None):
449
449
450
450
451 startDate = datetime.datetime.utcnow().date()
451 startDate = datetime.datetime.utcnow().date()
452 endDate = datetime.datetime.utcnow().date()
452 endDate = datetime.datetime.utcnow().date()
453
453
454 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
454 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
455
455
456 self.__checkPath()
456 self.__checkPath()
457
457
458 fullpath=path
458 fullpath=path
459 print "%s folder was found: " %(fullpath )
459 print "%s folder was found: " %(fullpath )
460
460
461 if set == None:
461 if set == None:
462 self.set=None
462 self.set=None
463 filename =getlastFileFromPath(fullpath,ext)
463 filename =getlastFileFromPath(fullpath,ext)
464 startDate= datetime.datetime.utcnow().date
464 startDate= datetime.datetime.utcnow().date
465 endDate= datetime.datetime.utcnow().date()
465 endDate= datetime.datetime.utcnow().date()
466 #
466 #
467 else:
467 else:
468 filename= getFileFromSet(fullpath,ext,set)
468 filename= getFileFromSet(fullpath,ext,set)
469 startDate=None
469 startDate=None
470 endDate=None
470 endDate=None
471 #
471 #
472 if not (filename):
472 if not (filename):
473 return None,None,None,None,None
473 return None,None,None,None,None
474 #print "%s file was found" %(filename)
474 #print "%s file was found" %(filename)
475
475
476 #
476 #
477 # dir_hf_filename= self.path+"/"+filename
477 # dir_hf_filename= self.path+"/"+filename
478 # fp= h5py.File(dir_hf_filename,'r')
478 # fp= h5py.File(dir_hf_filename,'r')
479 # hipoc=fp['t'].value
479 # hipoc=fp['t'].value
480 # fp.close()
480 # fp.close()
481 # date_time=datetime.datetime.utcfromtimestamp(hipoc)
481 # date_time=datetime.datetime.utcfromtimestamp(hipoc)
482 #
482 #
483 # year =int(date_time[0:4])
483 # year =int(date_time[0:4])
484 # month=int(date_time[5:7])
484 # month=int(date_time[5:7])
485 # dom =int(date_time[8:10])
485 # dom =int(date_time[8:10])
486 # set= int(filename[4:10])
486 # set= int(filename[4:10])
487 # self.set=set-1
487 # self.set=set-1
488 #self.dirnameList=[filename]
488 #self.dirnameList=[filename]
489 filenameList= fullpath+"/"+filename
489 filenameList= fullpath+"/"+filename
490 self.dirnameList=[filename]
490 self.dirnameList=[filename]
491 self.filenameList=[filenameList]
491 self.filenameList=[filenameList]
492 self.flag_nextfile=True
492 self.flag_nextfile=True
493
493
494 #self.__findDataForDates(online=True)
494 #self.__findDataForDates(online=True)
495 #self.dirnameList=[self.dirnameList[-1]]
495 #self.dirnameList=[self.dirnameList[-1]]
496 #print self.dirnameList
496 #print self.dirnameList
497 #self.__selectDataForTimes(online=True)
497 #self.__selectDataForTimes(online=True)
498 #return fullpath,filename,year,month,dom,set
498 #return fullpath,filename,year,month,dom,set
499 return
499 return
500
500
501 def __setNextFile(self,online=False):
501 def __setNextFile(self,online=False):
502 """
502 """
503 """
503 """
504 if not(online):
504 if not(online):
505 newFile = self.__setNextFileOffline()
505 newFile = self.__setNextFileOffline()
506 else:
506 else:
507 newFile = self.__setNextFileOnline()
507 newFile = self.__setNextFileOnline()
508
508
509 if not(newFile):
509 if not(newFile):
510 return 0
510 return 0
511 return 1
511 return 1
512
512
513 def __setNextFileOffline(self):
513 def __setNextFileOffline(self):
514 """
514 """
515 """
515 """
516 idFile= self.fileIndex
516 idFile= self.fileIndex
517 while(True):
517 while(True):
518 idFile += 1
518 idFile += 1
519 if not (idFile < len(self.filenameList)):
519 if not (idFile < len(self.filenameList)):
520 self.flagNoMoreFiles = 1
520 self.flagNoMoreFiles = 1
521 print "No more Files"
521 print "No more Files"
522 return 0
522 return 0
523 filename = self.filenameList[idFile]
523 filename = self.filenameList[idFile]
524 hfFilePointer =h5py.File(filename,'r')
524 hfFilePointer =h5py.File(filename,'r')
525
525
526 epoc=hfFilePointer['t'].value
526 epoc=hfFilePointer['t'].value
527 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
527 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
528 break
528 break
529
529
530 self.flagIsNewFile = 1
530 self.flagIsNewFile = 1
531 self.fileIndex = idFile
531 self.fileIndex = idFile
532 self.filename = filename
532 self.filename = filename
533
533
534 self.hfFilePointer = hfFilePointer
534 self.hfFilePointer = hfFilePointer
535 hfFilePointer.close()
535 hfFilePointer.close()
536 self.__t0=epoc
536 self.__t0=epoc
537 print "Setting the file: %s"%self.filename
537 print "Setting the file: %s"%self.filename
538
538
539 return 1
539 return 1
540
540
541 def __setNextFileOnline(self):
541 def __setNextFileOnline(self):
542 """
542 """
543 """
543 """
544 print "SOY NONE",self.set
544 print "SOY NONE",self.set
545 if self.set==None:
545 if self.set==None:
546 pass
546 pass
547 else:
547 else:
548 self.set +=10
548 self.set +=10
549
549
550 filename = self.filenameList[0]#fullfilename
550 filename = self.filenameList[0]#fullfilename
551 if self.filename_online != None:
551 if self.filename_online != None:
552 self.__selectDataForTimes(online=True)
552 self.__selectDataForTimes(online=True)
553 filename = self.filenameList[0]
553 filename = self.filenameList[0]
554 while self.filename_online == filename:
554 while self.filename_online == filename:
555 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
555 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
556 time.sleep(self.__waitForNewFile)
556 time.sleep(self.__waitForNewFile)
557 #self.__findDataForDates(online=True)
557 #self.__findDataForDates(online=True)
558 self.set=self.filename_next_set
558 self.set=self.filename_next_set
559 self.__selectDataForTimes(online=True)
559 self.__selectDataForTimes(online=True)
560 filename = self.filenameList[0]
560 filename = self.filenameList[0]
561 sizeoffile=os.path.getsize(filename)
561 sizeoffile=os.path.getsize(filename)
562
562
563 #print filename
563 #print filename
564 sizeoffile=os.path.getsize(filename)
564 sizeoffile=os.path.getsize(filename)
565 if sizeoffile<1670240:
565 if sizeoffile<1670240:
566 print "%s is not the rigth size"%filename
566 print "%s is not the rigth size"%filename
567 delay=50
567 delay=50
568 print 'waiting %d seconds for delay...'%(delay)
568 print 'waiting %d seconds for delay...'%(delay)
569 time.sleep(delay)
569 time.sleep(delay)
570 sizeoffile=os.path.getsize(filename)
570 sizeoffile=os.path.getsize(filename)
571 if sizeoffile<1670240:
571 if sizeoffile<1670240:
572 delay=50
572 delay=50
573 print 'waiting %d more seconds for delay...'%(delay)
573 print 'waiting %d more seconds for delay...'%(delay)
574 time.sleep(delay)
574 time.sleep(delay)
575
575
576 sizeoffile=os.path.getsize(filename)
576 sizeoffile=os.path.getsize(filename)
577 if sizeoffile<1670240:
577 if sizeoffile<1670240:
578 delay=50
578 delay=50
579 print 'waiting %d more seconds for delay...'%(delay)
579 print 'waiting %d more seconds for delay...'%(delay)
580 time.sleep(delay)
580 time.sleep(delay)
581
581
582 try:
582 try:
583 hfFilePointer=h5py.File(filename,'r')
583 hfFilePointer=h5py.File(filename,'r')
584
584
585 except:
585 except:
586 print "Error reading file %s"%filename
586 print "Error reading file %s"%filename
587
587
588 self.filename_online=filename
588 self.filename_online=filename
589 epoc=hfFilePointer['t'].value
589 epoc=hfFilePointer['t'].value
590
590
591 self.hfFilePointer=hfFilePointer
591 self.hfFilePointer=hfFilePointer
592 hfFilePointer.close()
592 hfFilePointer.close()
593 self.__t0=epoc
593 self.__t0=epoc
594
594
595
595
596 self.flagIsNewFile = 1
596 self.flagIsNewFile = 1
597 self.filename = filename
597 self.filename = filename
598
598
599 print "Setting the file: %s"%self.filename
599 print "Setting the file: %s"%self.filename
600 return 1
600 return 1
601
601
602 def __getExpParameters(self):
602 def __getExpParameters(self):
603 if not(self.status):
603 if not(self.status):
604 return None
604 return None
605
605
606 def setup(self,
606 def setup(self,
607 path = None,
607 path = None,
608 startDate = None,
608 startDate = None,
609 endDate = None,
609 endDate = None,
610 startTime = datetime.time(0,0,0),
610 startTime = datetime.time(0,0,0),
611 endTime = datetime.time(23,59,59),
611 endTime = datetime.time(23,59,59),
612 set = None,
612 set = None,
613 expLabel = "",
613 expLabel = "",
614 ext = None,
614 ext = None,
615 all=0,
615 all=0,
616 timezone=0,
616 timezone=0,
617 online = False,
617 online = False,
618 delay = 60,
618 delay = 60,
619 walk = True):
619 walk = True):
620 '''
620 '''
621 In this method we should set all initial parameters.
621 In this method we should set all initial parameters.
622
622
623 '''
623 '''
624 if path==None:
624 if path==None:
625 raise ValueError,"The path is not valid"
625 raise ValueError,"The path is not valid"
626
626
627 if ext==None:
627 if ext==None:
628 ext = self.ext
628 ext = self.ext
629
629
630 self.timezone= timezone
630 self.timezone= timezone
631 self.online= online
631 self.online= online
632 self.all=all
632 self.all=all
633 #if set==None:
633 #if set==None:
634
634
635 #print set
635 #print set
636 if not(online):
636 if not(online):
637 print "Searching files in offline mode..."
637 print "Searching files in offline mode..."
638
638
639 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 else:
640 else:
641 print "Searching files in online mode..."
641 print "Searching files in online mode..."
642 self.__searchFilesOnline(path, walk,ext,set=set)
642 self.searchFilesOnLine(path, walk,ext,set=set)
643 if set==None:
643 if set==None:
644 pass
644 pass
645 else:
645 else:
646 self.set=set-10
646 self.set=set-10
647
647
648 # for nTries in range(self.nTries):
648 # for nTries in range(self.nTries):
649 #
649 #
650 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 #
651 #
652 # if fullpath:
652 # if fullpath:
653 # break
653 # break
654 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
654 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
655 # time.sleep(self.delay)
655 # time.sleep(self.delay)
656 # if not(fullpath):
656 # if not(fullpath):
657 # print "There ins't valid files in %s" % path
657 # print "There ins't valid files in %s" % path
658 # return None
658 # return None
659
659
660
660
661 if not(self.filenameList):
661 if not(self.filenameList):
662 print "There is no files into the folder: %s"%(path)
662 print "There is no files into the folder: %s"%(path)
663 sys.exit(-1)
663 sys.exit(-1)
664
664
665 self.__getExpParameters()
665 self.__getExpParameters()
666
666
667
667
668 self.fileIndex = -1
668 self.fileIndex = -1
669
669
670 self.__setNextFile(online)
670 self.__setNextFile(online)
671
671
672 self.__readMetadata()
672 self.__readMetadata()
673
673
674 self.__setLocalVariables()
674 self.__setLocalVariables()
675
675
676 self.__setHeaderDO()
676 self.__setHeaderDO()
677 #self.profileIndex_offset= 0
677 #self.profileIndex_offset= 0
678
678
679 #self.profileIndex = self.profileIndex_offset
679 #self.profileIndex = self.profileIndex_offset
680
680
681 self.isConfig = True
681 self.isConfig = True
682
682
683 def __readMetadata(self):
683 def __readMetadata(self):
684 self.__readHeader()
684 self.__readHeader()
685
685
686
686
687 def __setLocalVariables(self):
687 def __setLocalVariables(self):
688
688
689 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
689 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
690 #
690 #
691
691
692
692
693
693
694 self.profileIndex = 9999
694 self.profileIndex = 9999
695
695
696
696
697 def __setHeaderDO(self):
697 def __setHeaderDO(self):
698
698
699
699
700 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
700 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
701
701
702 self.dataOut.systemHeaderObj = SystemHeader()
702 self.dataOut.systemHeaderObj = SystemHeader()
703
703
704
704
705 #---------------------------------------------------------
705 #---------------------------------------------------------
706 self.dataOut.systemHeaderObj.nProfiles=100
706 self.dataOut.systemHeaderObj.nProfiles=100
707 self.dataOut.systemHeaderObj.nSamples=1000
707 self.dataOut.systemHeaderObj.nSamples=1000
708
708
709
709
710 SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
710 SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
711 self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
711 self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
712 self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
712 self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
713 self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
713 self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
714 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
714 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
715 self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
715 self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
716 self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
716 self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
717 self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
717 self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
718 self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
718 self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
719
719
720 self.dataOut.radarControllerHeaderObj.nWindows=1
720 self.dataOut.radarControllerHeaderObj.nWindows=1
721 self.dataOut.radarControllerHeaderObj.codetype=0
721 self.dataOut.radarControllerHeaderObj.codetype=0
722 self.dataOut.radarControllerHeaderObj.numTaus=0
722 self.dataOut.radarControllerHeaderObj.numTaus=0
723 #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
723 #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
724
724
725
725
726 #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
726 #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
727 #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
727 #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
728 #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
728 #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
729
729
730 self.dataOut.radarControllerHeaderObj.code_size=0
730 self.dataOut.radarControllerHeaderObj.code_size=0
731 self.dataOut.nBaud=0
731 self.dataOut.nBaud=0
732 self.dataOut.nCode=0
732 self.dataOut.nCode=0
733 self.dataOut.nPairs=0
733 self.dataOut.nPairs=0
734
734
735
735
736 #---------------------------------------------------------
736 #---------------------------------------------------------
737
737
738 self.dataOut.type = "Voltage"
738 self.dataOut.type = "Voltage"
739
739
740 self.dataOut.data = None
740 self.dataOut.data = None
741
741
742 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
742 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
743
743
744 self.dataOut.nProfiles = 1
744 self.dataOut.nProfiles = 1
745
745
746 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
746 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
747
747
748 self.dataOut.channelList = range(self.nChannels)
748 self.dataOut.channelList = range(self.nChannels)
749
749
750 #self.dataOut.channelIndexList = None
750 #self.dataOut.channelIndexList = None
751
751
752 self.dataOut.flagNoData = True
752 self.dataOut.flagNoData = True
753
753
754 #Set to TRUE if the data is discontinuous
754 #Set to TRUE if the data is discontinuous
755 self.dataOut.flagDiscontinuousBlock = False
755 self.dataOut.flagDiscontinuousBlock = False
756
756
757 self.dataOut.utctime = None
757 self.dataOut.utctime = None
758
758
759 self.dataOut.timeZone = self.timezone
759 self.dataOut.timeZone = self.timezone
760
760
761 self.dataOut.dstFlag = 0
761 self.dataOut.dstFlag = 0
762
762
763 self.dataOut.errorCount = 0
763 self.dataOut.errorCount = 0
764
764
765 self.dataOut.nCohInt = 1
765 self.dataOut.nCohInt = 1
766
766
767 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
767 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
768
768
769 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
769 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
770
770
771 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
771 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
772
772
773 self.dataOut.flagShiftFFT = False
773 self.dataOut.flagShiftFFT = False
774
774
775 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
775 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
776
776
777 #Time interval between profiles
777 #Time interval between profiles
778 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
778 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
779
779
780
780
781 self.dataOut.frequency = self.__frequency
781 self.dataOut.frequency = self.__frequency
782
782
783 self.dataOut.realtime = self.__online
783 self.dataOut.realtime = self.__online
784
784
785 def __hasNotDataInBuffer(self):
785 def __hasNotDataInBuffer(self):
786
786
787 if self.profileIndex >= self.nProfiles:
787 if self.profileIndex >= self.nProfiles:
788 return 1
788 return 1
789
789
790 return 0
790 return 0
791
791
792 def readNextBlock(self):
792 def readNextBlock(self):
793 if not(self.__setNewBlock()):
793 if not(self.__setNewBlock()):
794 return 0
794 return 0
795
795
796 if not(self.readBlock()):
796 if not(self.readBlock()):
797 return 0
797 return 0
798
798
799 return 1
799 return 1
800
800
801 def __setNewBlock(self):
801 def __setNewBlock(self):
802
802
803 if self.hfFilePointer==None:
803 if self.hfFilePointer==None:
804 return 0
804 return 0
805
805
806 if self.flagIsNewFile:
806 if self.flagIsNewFile:
807 return 1
807 return 1
808
808
809 if self.profileIndex < self.nProfiles:
809 if self.profileIndex < self.nProfiles:
810 return 1
810 return 1
811
811
812 self.__setNextFile(self.online)
812 self.__setNextFile(self.online)
813
813
814 return 1
814 return 1
815
815
816
816
817
817
818 def readBlock(self):
818 def readBlock(self):
819 fp=h5py.File(self.filename,'r')
819 fp=h5py.File(self.filename,'r')
820 #Puntero que apunta al archivo hdf5
820 #Puntero que apunta al archivo hdf5
821 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
821 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
822 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
822 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
823 fp.close()
823 fp.close()
824 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
824 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
825 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
825 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
826 self.datablock = numpy.array([ch0,ch1])
826 self.datablock = numpy.array([ch0,ch1])
827 self.flagIsNewFile=0
827 self.flagIsNewFile=0
828
828
829 self.profileIndex=0
829 self.profileIndex=0
830
830
831 return 1
831 return 1
832
832
833 def getData(self):
833 def getData(self):
834 if self.flagNoMoreFiles:
834 if self.flagNoMoreFiles:
835 self.dataOut.flagNoData = True
835 self.dataOut.flagNoData = True
836 print 'Process finished'
836 print 'Process finished'
837 return 0
837 return 0
838
838
839 if self.__hasNotDataInBuffer():
839 if self.__hasNotDataInBuffer():
840 if not(self.readNextBlock()):
840 if not(self.readNextBlock()):
841 self.dataOut.flagNodata=True
841 self.dataOut.flagNodata=True
842 return 0
842 return 0
843
843
844 ##############################
844 ##############################
845 ##############################
845 ##############################
846 self.dataOut.data = self.datablock[:,:,self.profileIndex]
846 self.dataOut.data = self.datablock[:,:,self.profileIndex]
847 self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
847 self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
848 self.dataOut.profileIndex= self.profileIndex
848 self.dataOut.profileIndex= self.profileIndex
849 self.dataOut.flagNoData=False
849 self.dataOut.flagNoData=False
850 self.profileIndex +=1
850 self.profileIndex +=1
851
851
852 return self.dataOut.data
852 return self.dataOut.data
853
853
854
854
855 def run(self, **kwargs):
855 def run(self, **kwargs):
856 '''
856 '''
857 This method will be called many times so here you should put all your code
857 This method will be called many times so here you should put all your code
858 '''
858 '''
859
859
860 if not self.isConfig:
860 if not self.isConfig:
861 self.setup(**kwargs)
861 self.setup(**kwargs)
862 self.isConfig = True
862 self.isConfig = True
863 self.getData()
863 self.getData()
Show file before | Show file after | Hide comments
@@ -1,675 +1,675
1 '''
1 '''
2 Created on Set 9, 2015
2 Created on Set 9, 2015
3
3
4 @author: roj-idl71 Karim Kuyeng
4 @author: roj-idl71 Karim Kuyeng
5 '''
5 '''
6
6
7 import os
7 import os
8 import sys
8 import sys
9 import glob
9 import glob
10 import fnmatch
10 import fnmatch
11 import datetime
11 import datetime
12 import time
12 import time
13 import re
13 import re
14 import h5py
14 import h5py
15 import numpy
15 import numpy
16
16
17 try:
17 try:
18 from gevent import sleep
18 from gevent import sleep
19 except:
19 except:
20 from time import sleep
20 from time import sleep
21
21
22 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
22 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
23 from schainpy.model.data.jrodata import Voltage
23 from schainpy.model.data.jrodata import Voltage
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 from numpy import imag
25 from numpy import imag
26
26
27 class AMISRReader(ProcessingUnit):
27 class AMISRReader(ProcessingUnit):
28 '''
28 '''
29 classdocs
29 classdocs
30 '''
30 '''
31
31
32 def __init__(self):
32 def __init__(self):
33 '''
33 '''
34 Constructor
34 Constructor
35 '''
35 '''
36
36
37 ProcessingUnit.__init__(self)
37 ProcessingUnit.__init__(self)
38
38
39 self.set = None
39 self.set = None
40 self.subset = None
40 self.subset = None
41 self.extension_file = '.h5'
41 self.extension_file = '.h5'
42 self.dtc_str = 'dtc'
42 self.dtc_str = 'dtc'
43 self.dtc_id = 0
43 self.dtc_id = 0
44 self.status = True
44 self.status = True
45 self.isConfig = False
45 self.isConfig = False
46 self.dirnameList = []
46 self.dirnameList = []
47 self.filenameList = []
47 self.filenameList = []
48 self.fileIndex = None
48 self.fileIndex = None
49 self.flagNoMoreFiles = False
49 self.flagNoMoreFiles = False
50 self.flagIsNewFile = 0
50 self.flagIsNewFile = 0
51 self.filename = ''
51 self.filename = ''
52 self.amisrFilePointer = None
52 self.amisrFilePointer = None
53
53
54
54
55 self.dataset = None
55 self.dataset = None
56
56
57
57
58
58
59
59
60 self.profileIndex = 0
60 self.profileIndex = 0
61
61
62
62
63 self.beamCodeByFrame = None
63 self.beamCodeByFrame = None
64 self.radacTimeByFrame = None
64 self.radacTimeByFrame = None
65
65
66 self.dataset = None
66 self.dataset = None
67
67
68
68
69
69
70
70
71 self.__firstFile = True
71 self.__firstFile = True
72
72
73 self.buffer = None
73 self.buffer = None
74
74
75
75
76 self.timezone = 'ut'
76 self.timezone = 'ut'
77
77
78 self.__waitForNewFile = 20
78 self.__waitForNewFile = 20
79 self.__filename_online = None
79 self.__filename_online = None
80 #Is really necessary create the output object in the initializer
80 #Is really necessary create the output object in the initializer
81 self.dataOut = Voltage()
81 self.dataOut = Voltage()
82
82
83 def setup(self,path=None,
83 def setup(self,path=None,
84 startDate=None,
84 startDate=None,
85 endDate=None,
85 endDate=None,
86 startTime=None,
86 startTime=None,
87 endTime=None,
87 endTime=None,
88 walk=True,
88 walk=True,
89 timezone='ut',
89 timezone='ut',
90 all=0,
90 all=0,
91 code = None,
91 code = None,
92 nCode = 0,
92 nCode = 0,
93 nBaud = 0,
93 nBaud = 0,
94 online=False):
94 online=False):
95
95
96 self.timezone = timezone
96 self.timezone = timezone
97 self.all = all
97 self.all = all
98 self.online = online
98 self.online = online
99
99
100 self.code = code
100 self.code = code
101 self.nCode = int(nCode)
101 self.nCode = int(nCode)
102 self.nBaud = int(nBaud)
102 self.nBaud = int(nBaud)
103
103
104
104
105
105
106 #self.findFiles()
106 #self.findFiles()
107 if not(online):
107 if not(online):
108 #Busqueda de archivos offline
108 #Busqueda de archivos offline
109 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 else:
110 else:
111 self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk)
111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
112
112
113 if not(self.filenameList):
113 if not(self.filenameList):
114 print "There is no files into the folder: %s"%(path)
114 print "There is no files into the folder: %s"%(path)
115
115
116 sys.exit(-1)
116 sys.exit(-1)
117
117
118 self.fileIndex = -1
118 self.fileIndex = -1
119
119
120 self.readNextFile(online)
120 self.readNextFile(online)
121
121
122 '''
122 '''
123 Add code
123 Add code
124 '''
124 '''
125 self.isConfig = True
125 self.isConfig = True
126
126
127 pass
127 pass
128
128
129
129
130 def readAMISRHeader(self,fp):
130 def readAMISRHeader(self,fp):
131 header = 'Raw11/Data/RadacHeader'
131 header = 'Raw11/Data/RadacHeader'
132 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
132 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
133 self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
133 self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
134 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
134 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
135 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
135 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
136 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
136 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
137 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
137 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
138 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
138 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
139 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
139 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
140 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
140 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
141 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
141 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
142 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
142 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
143 self.frequency = fp.get('Rx/Frequency')
143 self.frequency = fp.get('Rx/Frequency')
144 txAus = fp.get('Raw11/Data/Pulsewidth')
144 txAus = fp.get('Raw11/Data/Pulsewidth')
145
145
146
146
147 self.nblocks = self.pulseCount.shape[0] #nblocks
147 self.nblocks = self.pulseCount.shape[0] #nblocks
148
148
149 self.nprofiles = self.pulseCount.shape[1] #nprofile
149 self.nprofiles = self.pulseCount.shape[1] #nprofile
150 self.nsa = self.nsamplesPulse[0,0] #ngates
150 self.nsa = self.nsamplesPulse[0,0] #ngates
151 self.nchannels = self.beamCode.shape[1]
151 self.nchannels = self.beamCode.shape[1]
152 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
152 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
153 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
153 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
154 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
154 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
155
155
156 #filling radar controller header parameters
156 #filling radar controller header parameters
157 self.__ippKm = self.ippSeconds *.15*1e6 # in km
157 self.__ippKm = self.ippSeconds *.15*1e6 # in km
158 self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
158 self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
159 self.__txB = 0
159 self.__txB = 0
160 nWindows=1
160 nWindows=1
161 self.__nSamples = self.nsa
161 self.__nSamples = self.nsa
162 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
162 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
163 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
163 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
164
164
165 #for now until understand why the code saved is different (code included even though code not in tuf file)
165 #for now until understand why the code saved is different (code included even though code not in tuf file)
166 #self.__codeType = 0
166 #self.__codeType = 0
167 # self.__nCode = None
167 # self.__nCode = None
168 # self.__nBaud = None
168 # self.__nBaud = None
169 self.__code = self.code
169 self.__code = self.code
170 self.__codeType = 0
170 self.__codeType = 0
171 if self.code != None:
171 if self.code != None:
172 self.__codeType = 1
172 self.__codeType = 1
173 self.__nCode = self.nCode
173 self.__nCode = self.nCode
174 self.__nBaud = self.nBaud
174 self.__nBaud = self.nBaud
175 #self.__code = 0
175 #self.__code = 0
176
176
177 #filling system header parameters
177 #filling system header parameters
178 self.__nSamples = self.nsa
178 self.__nSamples = self.nsa
179 self.newProfiles = self.nprofiles/self.nchannels
179 self.newProfiles = self.nprofiles/self.nchannels
180 self.__channelList = range(self.nchannels)
180 self.__channelList = range(self.nchannels)
181
181
182 self.__frequency = self.frequency[0][0]
182 self.__frequency = self.frequency[0][0]
183
183
184
184
185
185
186 def createBuffers(self):
186 def createBuffers(self):
187
187
188 pass
188 pass
189
189
190 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
190 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
191 self.path = path
191 self.path = path
192 self.startDate = startDate
192 self.startDate = startDate
193 self.endDate = endDate
193 self.endDate = endDate
194 self.startTime = startTime
194 self.startTime = startTime
195 self.endTime = endTime
195 self.endTime = endTime
196 self.walk = walk
196 self.walk = walk
197
197
198 def __checkPath(self):
198 def __checkPath(self):
199 if os.path.exists(self.path):
199 if os.path.exists(self.path):
200 self.status = 1
200 self.status = 1
201 else:
201 else:
202 self.status = 0
202 self.status = 0
203 print 'Path:%s does not exists'%self.path
203 print 'Path:%s does not exists'%self.path
204
204
205 return
205 return
206
206
207
207
208 def __selDates(self, amisr_dirname_format):
208 def __selDates(self, amisr_dirname_format):
209 try:
209 try:
210 year = int(amisr_dirname_format[0:4])
210 year = int(amisr_dirname_format[0:4])
211 month = int(amisr_dirname_format[4:6])
211 month = int(amisr_dirname_format[4:6])
212 dom = int(amisr_dirname_format[6:8])
212 dom = int(amisr_dirname_format[6:8])
213 thisDate = datetime.date(year,month,dom)
213 thisDate = datetime.date(year,month,dom)
214
214
215 if (thisDate>=self.startDate and thisDate <= self.endDate):
215 if (thisDate>=self.startDate and thisDate <= self.endDate):
216 return amisr_dirname_format
216 return amisr_dirname_format
217 except:
217 except:
218 return None
218 return None
219
219
220
220
221 def __findDataForDates(self,online=False):
221 def __findDataForDates(self,online=False):
222
222
223 if not(self.status):
223 if not(self.status):
224 return None
224 return None
225
225
226 pat = '\d+.\d+'
226 pat = '\d+.\d+'
227 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
227 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
228 dirnameList = filter(lambda x:x!=None,dirnameList)
228 dirnameList = filter(lambda x:x!=None,dirnameList)
229 dirnameList = [x.string for x in dirnameList]
229 dirnameList = [x.string for x in dirnameList]
230 if not(online):
230 if not(online):
231 dirnameList = [self.__selDates(x) for x in dirnameList]
231 dirnameList = [self.__selDates(x) for x in dirnameList]
232 dirnameList = filter(lambda x:x!=None,dirnameList)
232 dirnameList = filter(lambda x:x!=None,dirnameList)
233 if len(dirnameList)>0:
233 if len(dirnameList)>0:
234 self.status = 1
234 self.status = 1
235 self.dirnameList = dirnameList
235 self.dirnameList = dirnameList
236 self.dirnameList.sort()
236 self.dirnameList.sort()
237 else:
237 else:
238 self.status = 0
238 self.status = 0
239 return None
239 return None
240
240
241 def __getTimeFromData(self):
241 def __getTimeFromData(self):
242 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
242 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
243 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
243 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
244
244
245 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
245 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
246 print '........................................'
246 print '........................................'
247 filter_filenameList = []
247 filter_filenameList = []
248 self.filenameList.sort()
248 self.filenameList.sort()
249 #for i in range(len(self.filenameList)-1):
249 #for i in range(len(self.filenameList)-1):
250 for i in range(len(self.filenameList)):
250 for i in range(len(self.filenameList)):
251 filename = self.filenameList[i]
251 filename = self.filenameList[i]
252 fp = h5py.File(filename,'r')
252 fp = h5py.File(filename,'r')
253 time_str = fp.get('Time/RadacTimeString')
253 time_str = fp.get('Time/RadacTimeString')
254
254
255 startDateTimeStr_File = time_str[0][0].split('.')[0]
255 startDateTimeStr_File = time_str[0][0].split('.')[0]
256 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
256 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
257 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
257 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
258
258
259 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
259 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
260 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
260 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
261 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
261 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
262
262
263 fp.close()
263 fp.close()
264
264
265 if self.timezone == 'lt':
265 if self.timezone == 'lt':
266 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
266 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
267 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
267 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
268
268
269 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
269 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
270 #self.filenameList.remove(filename)
270 #self.filenameList.remove(filename)
271 filter_filenameList.append(filename)
271 filter_filenameList.append(filename)
272
272
273 if (endDateTime_File>=endDateTime_Reader):
273 if (endDateTime_File>=endDateTime_Reader):
274 break
274 break
275
275
276
276
277 filter_filenameList.sort()
277 filter_filenameList.sort()
278 self.filenameList = filter_filenameList
278 self.filenameList = filter_filenameList
279 return 1
279 return 1
280
280
281 def __filterByGlob1(self, dirName):
281 def __filterByGlob1(self, dirName):
282 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
282 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
283 filter_files.sort()
283 filter_files.sort()
284 filterDict = {}
284 filterDict = {}
285 filterDict.setdefault(dirName)
285 filterDict.setdefault(dirName)
286 filterDict[dirName] = filter_files
286 filterDict[dirName] = filter_files
287 return filterDict
287 return filterDict
288
288
289 def __getFilenameList(self, fileListInKeys, dirList):
289 def __getFilenameList(self, fileListInKeys, dirList):
290 for value in fileListInKeys:
290 for value in fileListInKeys:
291 dirName = value.keys()[0]
291 dirName = value.keys()[0]
292 for file in value[dirName]:
292 for file in value[dirName]:
293 filename = os.path.join(dirName, file)
293 filename = os.path.join(dirName, file)
294 self.filenameList.append(filename)
294 self.filenameList.append(filename)
295
295
296
296
297 def __selectDataForTimes(self, online=False):
297 def __selectDataForTimes(self, online=False):
298 #aun no esta implementado el filtro for tiempo
298 #aun no esta implementado el filtro for tiempo
299 if not(self.status):
299 if not(self.status):
300 return None
300 return None
301
301
302 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
302 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
303
303
304 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
304 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
305
305
306 self.__getFilenameList(fileListInKeys, dirList)
306 self.__getFilenameList(fileListInKeys, dirList)
307 if not(online):
307 if not(online):
308 #filtro por tiempo
308 #filtro por tiempo
309 if not(self.all):
309 if not(self.all):
310 self.__getTimeFromData()
310 self.__getTimeFromData()
311
311
312 if len(self.filenameList)>0:
312 if len(self.filenameList)>0:
313 self.status = 1
313 self.status = 1
314 self.filenameList.sort()
314 self.filenameList.sort()
315 else:
315 else:
316 self.status = 0
316 self.status = 0
317 return None
317 return None
318
318
319 else:
319 else:
320 #get the last file - 1
320 #get the last file - 1
321 self.filenameList = [self.filenameList[-2]]
321 self.filenameList = [self.filenameList[-2]]
322
322
323 new_dirnameList = []
323 new_dirnameList = []
324 for dirname in self.dirnameList:
324 for dirname in self.dirnameList:
325 junk = numpy.array([dirname in x for x in self.filenameList])
325 junk = numpy.array([dirname in x for x in self.filenameList])
326 junk_sum = junk.sum()
326 junk_sum = junk.sum()
327 if junk_sum > 0:
327 if junk_sum > 0:
328 new_dirnameList.append(dirname)
328 new_dirnameList.append(dirname)
329 self.dirnameList = new_dirnameList
329 self.dirnameList = new_dirnameList
330 return 1
330 return 1
331
331
332 def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 endTime=datetime.time(23,59,59),walk=True):
333 endTime=datetime.time(23,59,59),walk=True):
334
334
335 if endDate ==None:
335 if endDate ==None:
336 startDate = datetime.datetime.utcnow().date()
336 startDate = datetime.datetime.utcnow().date()
337 endDate = datetime.datetime.utcnow().date()
337 endDate = datetime.datetime.utcnow().date()
338
338
339 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
339 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
340
340
341 self.__checkPath()
341 self.__checkPath()
342
342
343 self.__findDataForDates(online=True)
343 self.__findDataForDates(online=True)
344
344
345 self.dirnameList = [self.dirnameList[-1]]
345 self.dirnameList = [self.dirnameList[-1]]
346
346
347 self.__selectDataForTimes(online=True)
347 self.__selectDataForTimes(online=True)
348
348
349 return
349 return
350
350
351
351
352 def __searchFilesOffline(self,
352 def searchFilesOffLine(self,
353 path,
353 path,
354 startDate,
354 startDate,
355 endDate,
355 endDate,
356 startTime=datetime.time(0,0,0),
356 startTime=datetime.time(0,0,0),
357 endTime=datetime.time(23,59,59),
357 endTime=datetime.time(23,59,59),
358 walk=True):
358 walk=True):
359
359
360 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
360 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
361
361
362 self.__checkPath()
362 self.__checkPath()
363
363
364 self.__findDataForDates()
364 self.__findDataForDates()
365
365
366 self.__selectDataForTimes()
366 self.__selectDataForTimes()
367
367
368 for i in range(len(self.filenameList)):
368 for i in range(len(self.filenameList)):
369 print "%s" %(self.filenameList[i])
369 print "%s" %(self.filenameList[i])
370
370
371 return
371 return
372
372
373 def __setNextFileOffline(self):
373 def __setNextFileOffline(self):
374 idFile = self.fileIndex
374 idFile = self.fileIndex
375
375
376 while (True):
376 while (True):
377 idFile += 1
377 idFile += 1
378 if not(idFile < len(self.filenameList)):
378 if not(idFile < len(self.filenameList)):
379 self.flagNoMoreFiles = 1
379 self.flagNoMoreFiles = 1
380 print "No more Files"
380 print "No more Files"
381 return 0
381 return 0
382
382
383 filename = self.filenameList[idFile]
383 filename = self.filenameList[idFile]
384
384
385 amisrFilePointer = h5py.File(filename,'r')
385 amisrFilePointer = h5py.File(filename,'r')
386
386
387 break
387 break
388
388
389 self.flagIsNewFile = 1
389 self.flagIsNewFile = 1
390 self.fileIndex = idFile
390 self.fileIndex = idFile
391 self.filename = filename
391 self.filename = filename
392
392
393 self.amisrFilePointer = amisrFilePointer
393 self.amisrFilePointer = amisrFilePointer
394
394
395 print "Setting the file: %s"%self.filename
395 print "Setting the file: %s"%self.filename
396
396
397 return 1
397 return 1
398
398
399
399
400 def __setNextFileOnline(self):
400 def __setNextFileOnline(self):
401 filename = self.filenameList[0]
401 filename = self.filenameList[0]
402 if self.__filename_online != None:
402 if self.__filename_online != None:
403 self.__selectDataForTimes(online=True)
403 self.__selectDataForTimes(online=True)
404 filename = self.filenameList[0]
404 filename = self.filenameList[0]
405 wait = 0
405 wait = 0
406 while self.__filename_online == filename:
406 while self.__filename_online == filename:
407 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
407 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
408 if wait == 5:
408 if wait == 5:
409 return 0
409 return 0
410 sleep(self.__waitForNewFile)
410 sleep(self.__waitForNewFile)
411 self.__selectDataForTimes(online=True)
411 self.__selectDataForTimes(online=True)
412 filename = self.filenameList[0]
412 filename = self.filenameList[0]
413 wait += 1
413 wait += 1
414
414
415 self.__filename_online = filename
415 self.__filename_online = filename
416
416
417 self.amisrFilePointer = h5py.File(filename,'r')
417 self.amisrFilePointer = h5py.File(filename,'r')
418 self.flagIsNewFile = 1
418 self.flagIsNewFile = 1
419 self.filename = filename
419 self.filename = filename
420 print "Setting the file: %s"%self.filename
420 print "Setting the file: %s"%self.filename
421 return 1
421 return 1
422
422
423
423
424 def readData(self):
424 def readData(self):
425 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
425 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
426 re = buffer[:,:,:,0]
426 re = buffer[:,:,:,0]
427 im = buffer[:,:,:,1]
427 im = buffer[:,:,:,1]
428 dataset = re + im*1j
428 dataset = re + im*1j
429 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
429 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
430 timeset = self.radacTime[:,0]
430 timeset = self.radacTime[:,0]
431 return dataset,timeset
431 return dataset,timeset
432
432
433 def reshapeData(self):
433 def reshapeData(self):
434 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
434 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
435 channels = self.beamCodeByPulse[0,:]
435 channels = self.beamCodeByPulse[0,:]
436 nchan = self.nchannels
436 nchan = self.nchannels
437 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
437 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
438 nblocks = self.nblocks
438 nblocks = self.nblocks
439 nsamples = self.nsa
439 nsamples = self.nsa
440
440
441 #Dimensions : nChannels, nProfiles, nSamples
441 #Dimensions : nChannels, nProfiles, nSamples
442 new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
442 new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
443 ############################################
443 ############################################
444
444
445 for thisChannel in range(nchan):
445 for thisChannel in range(nchan):
446 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
446 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
447
447
448
448
449 new_block = numpy.transpose(new_block, (1,0,2,3))
449 new_block = numpy.transpose(new_block, (1,0,2,3))
450 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
450 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
451
451
452 return new_block
452 return new_block
453
453
454 def updateIndexes(self):
454 def updateIndexes(self):
455
455
456 pass
456 pass
457
457
458 def fillJROHeader(self):
458 def fillJROHeader(self):
459
459
460 #fill radar controller header
460 #fill radar controller header
461 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
461 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
462 txA=self.__txA,
462 txA=self.__txA,
463 txB=0,
463 txB=0,
464 nWindows=1,
464 nWindows=1,
465 nHeights=self.__nSamples,
465 nHeights=self.__nSamples,
466 firstHeight=self.__firstHeight,
466 firstHeight=self.__firstHeight,
467 deltaHeight=self.__deltaHeight,
467 deltaHeight=self.__deltaHeight,
468 codeType=self.__codeType,
468 codeType=self.__codeType,
469 nCode=self.__nCode, nBaud=self.__nBaud,
469 nCode=self.__nCode, nBaud=self.__nBaud,
470 code = self.__code,
470 code = self.__code,
471 fClock=1)
471 fClock=1)
472
472
473
473
474
474
475 #fill system header
475 #fill system header
476 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
476 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
477 nProfiles=self.newProfiles,
477 nProfiles=self.newProfiles,
478 nChannels=len(self.__channelList),
478 nChannels=len(self.__channelList),
479 adcResolution=14,
479 adcResolution=14,
480 pciDioBusWith=32)
480 pciDioBusWith=32)
481
481
482 self.dataOut.type = "Voltage"
482 self.dataOut.type = "Voltage"
483
483
484 self.dataOut.data = None
484 self.dataOut.data = None
485
485
486 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
486 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
487
487
488 # self.dataOut.nChannels = 0
488 # self.dataOut.nChannels = 0
489
489
490 # self.dataOut.nHeights = 0
490 # self.dataOut.nHeights = 0
491
491
492 self.dataOut.nProfiles = self.newProfiles*self.nblocks
492 self.dataOut.nProfiles = self.newProfiles*self.nblocks
493
493
494 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
494 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
495 ranges = numpy.reshape(self.rangeFromFile.value,(-1))
495 ranges = numpy.reshape(self.rangeFromFile.value,(-1))
496 self.dataOut.heightList = ranges/1000.0 #km
496 self.dataOut.heightList = ranges/1000.0 #km
497
497
498
498
499 self.dataOut.channelList = self.__channelList
499 self.dataOut.channelList = self.__channelList
500
500
501 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
501 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
502
502
503 # self.dataOut.channelIndexList = None
503 # self.dataOut.channelIndexList = None
504
504
505 self.dataOut.flagNoData = True
505 self.dataOut.flagNoData = True
506
506
507 #Set to TRUE if the data is discontinuous
507 #Set to TRUE if the data is discontinuous
508 self.dataOut.flagDiscontinuousBlock = False
508 self.dataOut.flagDiscontinuousBlock = False
509
509
510 self.dataOut.utctime = None
510 self.dataOut.utctime = None
511
511
512 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
512 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
513 if self.timezone == 'lt':
513 if self.timezone == 'lt':
514 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
514 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
515 else:
515 else:
516 self.dataOut.timeZone = 0 #by default time is UTC
516 self.dataOut.timeZone = 0 #by default time is UTC
517
517
518 self.dataOut.dstFlag = 0
518 self.dataOut.dstFlag = 0
519
519
520 self.dataOut.errorCount = 0
520 self.dataOut.errorCount = 0
521
521
522 self.dataOut.nCohInt = 1
522 self.dataOut.nCohInt = 1
523
523
524 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
524 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
525
525
526 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
526 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
527
527
528 self.dataOut.flagShiftFFT = False
528 self.dataOut.flagShiftFFT = False
529
529
530 self.dataOut.ippSeconds = self.ippSeconds
530 self.dataOut.ippSeconds = self.ippSeconds
531
531
532 #Time interval between profiles
532 #Time interval between profiles
533 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
533 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
534
534
535 self.dataOut.frequency = self.__frequency
535 self.dataOut.frequency = self.__frequency
536
536
537 self.dataOut.realtime = self.online
537 self.dataOut.realtime = self.online
538 pass
538 pass
539
539
540 def readNextFile(self,online=False):
540 def readNextFile(self,online=False):
541
541
542 if not(online):
542 if not(online):
543 newFile = self.__setNextFileOffline()
543 newFile = self.__setNextFileOffline()
544 else:
544 else:
545 newFile = self.__setNextFileOnline()
545 newFile = self.__setNextFileOnline()
546
546
547 if not(newFile):
547 if not(newFile):
548 return 0
548 return 0
549
549
550 #if self.__firstFile:
550 #if self.__firstFile:
551 self.readAMISRHeader(self.amisrFilePointer)
551 self.readAMISRHeader(self.amisrFilePointer)
552 self.createBuffers()
552 self.createBuffers()
553 self.fillJROHeader()
553 self.fillJROHeader()
554 #self.__firstFile = False
554 #self.__firstFile = False
555
555
556
556
557
557
558 self.dataset,self.timeset = self.readData()
558 self.dataset,self.timeset = self.readData()
559
559
560 if self.endDate!=None:
560 if self.endDate!=None:
561 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
561 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
562 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
562 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
563 startDateTimeStr_File = time_str[0][0].split('.')[0]
563 startDateTimeStr_File = time_str[0][0].split('.')[0]
564 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
564 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
565 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
565 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
566 if self.timezone == 'lt':
566 if self.timezone == 'lt':
567 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
567 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
568 if (startDateTime_File>endDateTime_Reader):
568 if (startDateTime_File>endDateTime_Reader):
569 return 0
569 return 0
570
570
571 self.jrodataset = self.reshapeData()
571 self.jrodataset = self.reshapeData()
572 #----self.updateIndexes()
572 #----self.updateIndexes()
573 self.profileIndex = 0
573 self.profileIndex = 0
574
574
575 return 1
575 return 1
576
576
577
577
578 def __hasNotDataInBuffer(self):
578 def __hasNotDataInBuffer(self):
579 if self.profileIndex >= (self.newProfiles*self.nblocks):
579 if self.profileIndex >= (self.newProfiles*self.nblocks):
580 return 1
580 return 1
581 return 0
581 return 0
582
582
583
583
584 def getData(self):
584 def getData(self):
585
585
586 if self.flagNoMoreFiles:
586 if self.flagNoMoreFiles:
587 self.dataOut.flagNoData = True
587 self.dataOut.flagNoData = True
588 print 'Process finished'
588 print 'Process finished'
589 return 0
589 return 0
590
590
591 if self.__hasNotDataInBuffer():
591 if self.__hasNotDataInBuffer():
592 if not (self.readNextFile(self.online)):
592 if not (self.readNextFile(self.online)):
593 return 0
593 return 0
594
594
595
595
596 if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
596 if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
597 self.dataOut.flagNoData = True
597 self.dataOut.flagNoData = True
598 return 0
598 return 0
599
599
600 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
600 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
601
601
602 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
602 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
603
603
604 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
604 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
605 #verificar basic header de jro data y ver si es compatible con este valor
605 #verificar basic header de jro data y ver si es compatible con este valor
606 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
606 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
607 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
607 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
608 indexblock = self.profileIndex/self.newProfiles
608 indexblock = self.profileIndex/self.newProfiles
609 #print indexblock, indexprof
609 #print indexblock, indexprof
610 self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
610 self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
611 self.dataOut.profileIndex = self.profileIndex
611 self.dataOut.profileIndex = self.profileIndex
612 self.dataOut.flagNoData = False
612 self.dataOut.flagNoData = False
613 # if indexprof == 0:
613 # if indexprof == 0:
614 # print self.dataOut.utctime
614 # print self.dataOut.utctime
615
615
616 self.profileIndex += 1
616 self.profileIndex += 1
617
617
618 return self.dataOut.data
618 return self.dataOut.data
619
619
620
620
621 def run(self, **kwargs):
621 def run(self, **kwargs):
622 '''
622 '''
623 This method will be called many times so here you should put all your code
623 This method will be called many times so here you should put all your code
624 '''
624 '''
625
625
626 if not self.isConfig:
626 if not self.isConfig:
627 self.setup(**kwargs)
627 self.setup(**kwargs)
628 self.isConfig = True
628 self.isConfig = True
629
629
630 self.getData()
630 self.getData()
631
631
632 class Writer(Operation):
632 class Writer(Operation):
633 '''
633 '''
634 classdocs
634 classdocs
635 '''
635 '''
636
636
637 def __init__(self):
637 def __init__(self):
638 '''
638 '''
639 Constructor
639 Constructor
640 '''
640 '''
641 self.dataOut = None
641 self.dataOut = None
642
642
643 self.isConfig = False
643 self.isConfig = False
644
644
645 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
645 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
646 '''
646 '''
647 In this method we should set all initial parameters.
647 In this method we should set all initial parameters.
648
648
649 Input:
649 Input:
650 dataIn : Input data will also be outputa data
650 dataIn : Input data will also be outputa data
651
651
652 '''
652 '''
653 self.dataOut = dataIn
653 self.dataOut = dataIn
654
654
655
655
656
656
657
657
658
658
659 self.isConfig = True
659 self.isConfig = True
660
660
661 return
661 return
662
662
663 def run(self, dataIn, **kwargs):
663 def run(self, dataIn, **kwargs):
664 '''
664 '''
665 This method will be called many times so here you should put all your code
665 This method will be called many times so here you should put all your code
666
666
667 Inputs:
667 Inputs:
668
668
669 dataIn : object with the data
669 dataIn : object with the data
670
670
671 '''
671 '''
672
672
673 if not self.isConfig:
673 if not self.isConfig:
674 self.setup(dataIn, **kwargs)
674 self.setup(dataIn, **kwargs)
675 No newline at end of file
675
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1095
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class ParamReader(ProcessingUnit):
15 class ParamReader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self, **kwargs):
77 def __init__(self, **kwargs):
78 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self, **kwargs)
79 self.dataOut = Parameters()
79 self.dataOut = Parameters()
80 return
80 return
81
81
82 def setup(self, **kwargs):
82 def setup(self, **kwargs):
83
83
84 path = kwargs['path']
84 path = kwargs['path']
85 startDate = kwargs['startDate']
85 startDate = kwargs['startDate']
86 endDate = kwargs['endDate']
86 endDate = kwargs['endDate']
87 startTime = kwargs['startTime']
87 startTime = kwargs['startTime']
88 endTime = kwargs['endTime']
88 endTime = kwargs['endTime']
89 walk = kwargs['walk']
89 walk = kwargs['walk']
90 if kwargs.has_key('ext'):
90 if kwargs.has_key('ext'):
91 ext = kwargs['ext']
91 ext = kwargs['ext']
92 else:
92 else:
93 ext = '.hdf5'
93 ext = '.hdf5'
94 if kwargs.has_key('timezone'):
94 if kwargs.has_key('timezone'):
95 self.timezone = kwargs['timezone']
95 self.timezone = kwargs['timezone']
96 else:
96 else:
97 self.timezone = 'lt'
97 self.timezone = 'lt'
98
98
99 print "[Reading] Searching files in offline mode ..."
99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 startTime=startTime, endTime=endTime,
101 startTime=startTime, endTime=endTime,
102 ext=ext, walk=walk)
102 ext=ext, walk=walk)
103
103
104 if not(filenameList):
104 if not(filenameList):
105 print "There is no files into the folder: %s"%(path)
105 print "There is no files into the folder: %s"%(path)
106 sys.exit(-1)
106 sys.exit(-1)
107
107
108 self.fileIndex = -1
108 self.fileIndex = -1
109 self.startTime = startTime
109 self.startTime = startTime
110 self.endTime = endTime
110 self.endTime = endTime
111
111
112 self.__readMetadata()
112 self.__readMetadata()
113
113
114 self.__setNextFileOffline()
114 self.__setNextFileOffline()
115
115
116 return
116 return
117
117
118 def __searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 path,
119 path,
120 startDate=None,
120 startDate=None,
121 endDate=None,
121 endDate=None,
122 startTime=datetime.time(0,0,0),
122 startTime=datetime.time(0,0,0),
123 endTime=datetime.time(23,59,59),
123 endTime=datetime.time(23,59,59),
124 ext='.hdf5',
124 ext='.hdf5',
125 walk=True):
125 walk=True):
126
126
127 expLabel = ''
127 expLabel = ''
128 self.filenameList = []
128 self.filenameList = []
129 self.datetimeList = []
129 self.datetimeList = []
130
130
131 pathList = []
131 pathList = []
132
132
133 JRODataObj = JRODataReader()
133 JRODataObj = JRODataReader()
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135
135
136 if dateList == []:
136 if dateList == []:
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 datetime.datetime.combine(startDate,startTime).ctime(),
138 datetime.datetime.combine(startDate,startTime).ctime(),
139 datetime.datetime.combine(endDate,endTime).ctime())
139 datetime.datetime.combine(endDate,endTime).ctime())
140
140
141 return None, None
141 return None, None
142
142
143 if len(dateList) > 1:
143 if len(dateList) > 1:
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
145 else:
145 else:
146 print "[Reading] data was found for the date %s" %(dateList[0])
146 print "[Reading] data was found for the date %s" %(dateList[0])
147
147
148 filenameList = []
148 filenameList = []
149 datetimeList = []
149 datetimeList = []
150
150
151 #----------------------------------------------------------------------------------
151 #----------------------------------------------------------------------------------
152
152
153 for thisPath in pathList:
153 for thisPath in pathList:
154 # thisPath = pathList[pathDict[file]]
154 # thisPath = pathList[pathDict[file]]
155
155
156 fileList = glob.glob1(thisPath, "*%s" %ext)
156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 fileList.sort()
157 fileList.sort()
158
158
159 for file in fileList:
159 for file in fileList:
160
160
161 filename = os.path.join(thisPath,file)
161 filename = os.path.join(thisPath,file)
162
162
163 if not isFileInDateRange(filename, startDate, endDate):
163 if not isFileInDateRange(filename, startDate, endDate):
164 continue
164 continue
165
165
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167
167
168 if not(thisDatetime):
168 if not(thisDatetime):
169 continue
169 continue
170
170
171 filenameList.append(filename)
171 filenameList.append(filename)
172 datetimeList.append(thisDatetime)
172 datetimeList.append(thisDatetime)
173
173
174 if not(filenameList):
174 if not(filenameList):
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
176 return None, None
176 return None, None
177
177
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
179 print
179 print
180
180
181 for i in range(len(filenameList)):
181 for i in range(len(filenameList)):
182 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
182 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183
183
184 self.filenameList = filenameList
184 self.filenameList = filenameList
185 self.datetimeList = datetimeList
185 self.datetimeList = datetimeList
186
186
187 return pathList, filenameList
187 return pathList, filenameList
188
188
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190
190
191 """
191 """
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193
193
194 Inputs:
194 Inputs:
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196
196
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198
198
199 endDate : fecha final del rango seleccionado en formato datetime.date
199 endDate : fecha final del rango seleccionado en formato datetime.date
200
200
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202
202
203 endTime : tiempo final del rango seleccionado en formato datetime.time
203 endTime : tiempo final del rango seleccionado en formato datetime.time
204
204
205 Return:
205 Return:
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 fecha especificado, de lo contrario retorna False.
207 fecha especificado, de lo contrario retorna False.
208
208
209 Excepciones:
209 Excepciones:
210 Si el archivo no existe o no puede ser abierto
210 Si el archivo no existe o no puede ser abierto
211 Si la cabecera no puede ser leida.
211 Si la cabecera no puede ser leida.
212
212
213 """
213 """
214
214
215 try:
215 try:
216 fp = h5py.File(filename,'r')
216 fp = h5py.File(filename,'r')
217 grp1 = fp['Data']
217 grp1 = fp['Data']
218
218
219 except IOError:
219 except IOError:
220 traceback.print_exc()
220 traceback.print_exc()
221 raise IOError, "The file %s can't be opened" %(filename)
221 raise IOError, "The file %s can't be opened" %(filename)
222 #chino rata
222 #chino rata
223 #In case has utctime attribute
223 #In case has utctime attribute
224 grp2 = grp1['utctime']
224 grp2 = grp1['utctime']
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 thisUtcTime = grp2.value[0]
226 thisUtcTime = grp2.value[0]
227
227
228 fp.close()
228 fp.close()
229
229
230 if self.timezone == 'lt':
230 if self.timezone == 'lt':
231 thisUtcTime -= 5*3600
231 thisUtcTime -= 5*3600
232
232
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 thisDate = thisDatetime.date()
235 thisDate = thisDatetime.date()
236 thisTime = thisDatetime.time()
236 thisTime = thisDatetime.time()
237
237
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240
240
241 #General case
241 #General case
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 #-----------o----------------------------o-----------
243 #-----------o----------------------------o-----------
244 # startTime endTime
244 # startTime endTime
245
245
246 if endTime >= startTime:
246 if endTime >= startTime:
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 return thisDatetime
249 return thisDatetime
250 return None
250 return None
251
251
252 #If endTime < startTime then endTime belongs to the next day
252 #If endTime < startTime then endTime belongs to the next day
253 #<<<<<<<<<<<o o>>>>>>>>>>>
253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 #-----------o----------------------------o-----------
254 #-----------o----------------------------o-----------
255 # endTime startTime
255 # endTime startTime
256
256
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 return None
258 return None
259
259
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 return None
261 return None
262
262
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 return None
264 return None
265
265
266 return thisDatetime
266 return thisDatetime
267
267
268 def __setNextFileOffline(self):
268 def __setNextFileOffline(self):
269
269
270 self.fileIndex += 1
270 self.fileIndex += 1
271 idFile = self.fileIndex
271 idFile = self.fileIndex
272
272
273 if not(idFile < len(self.filenameList)):
273 if not(idFile < len(self.filenameList)):
274 print "No more Files"
274 print "No more Files"
275 return 0
275 return 0
276
276
277 filename = self.filenameList[idFile]
277 filename = self.filenameList[idFile]
278
278
279 filePointer = h5py.File(filename,'r')
279 filePointer = h5py.File(filename,'r')
280
280
281 self.filename = filename
281 self.filename = filename
282
282
283 self.fp = filePointer
283 self.fp = filePointer
284
284
285 print "Setting the file: %s"%self.filename
285 print "Setting the file: %s"%self.filename
286
286
287 # self.__readMetadata()
287 # self.__readMetadata()
288 self.__setBlockList()
288 self.__setBlockList()
289 self.__readData()
289 self.__readData()
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 self.blockIndex = 0
292 self.blockIndex = 0
293 return 1
293 return 1
294
294
295 def __setBlockList(self):
295 def __setBlockList(self):
296 '''
296 '''
297 Selects the data within the times defined
297 Selects the data within the times defined
298
298
299 self.fp
299 self.fp
300 self.startTime
300 self.startTime
301 self.endTime
301 self.endTime
302
302
303 self.blockList
303 self.blockList
304 self.blocksPerFile
304 self.blocksPerFile
305
305
306 '''
306 '''
307 fp = self.fp
307 fp = self.fp
308 startTime = self.startTime
308 startTime = self.startTime
309 endTime = self.endTime
309 endTime = self.endTime
310
310
311 grp = fp['Data']
311 grp = fp['Data']
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313
313
314 #ERROOOOR
314 #ERROOOOR
315 if self.timezone == 'lt':
315 if self.timezone == 'lt':
316 thisUtcTime -= 5*3600
316 thisUtcTime -= 5*3600
317
317
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319
319
320 thisDate = thisDatetime.date()
320 thisDate = thisDatetime.date()
321 thisTime = thisDatetime.time()
321 thisTime = thisDatetime.time()
322
322
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325
325
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327
327
328 self.blockList = ind
328 self.blockList = ind
329 self.blocksPerFile = len(ind)
329 self.blocksPerFile = len(ind)
330
330
331 return
331 return
332
332
333 def __readMetadata(self):
333 def __readMetadata(self):
334 '''
334 '''
335 Reads Metadata
335 Reads Metadata
336
336
337 self.pathMeta
337 self.pathMeta
338
338
339 self.listShapes
339 self.listShapes
340 self.listMetaname
340 self.listMetaname
341 self.listMeta
341 self.listMeta
342
342
343 '''
343 '''
344
344
345 # grp = self.fp['Data']
345 # grp = self.fp['Data']
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 #
347 #
348 # if pathMeta == self.pathMeta:
348 # if pathMeta == self.pathMeta:
349 # return
349 # return
350 # else:
350 # else:
351 # self.pathMeta = pathMeta
351 # self.pathMeta = pathMeta
352 #
352 #
353 # filePointer = h5py.File(self.pathMeta,'r')
353 # filePointer = h5py.File(self.pathMeta,'r')
354 # groupPointer = filePointer['Metadata']
354 # groupPointer = filePointer['Metadata']
355
355
356 filename = self.filenameList[0]
356 filename = self.filenameList[0]
357
357
358 fp = h5py.File(filename,'r')
358 fp = h5py.File(filename,'r')
359
359
360 gp = fp['Metadata']
360 gp = fp['Metadata']
361
361
362 listMetaname = []
362 listMetaname = []
363 listMetadata = []
363 listMetadata = []
364 for item in gp.items():
364 for item in gp.items():
365 name = item[0]
365 name = item[0]
366
366
367 if name=='array dimensions':
367 if name=='array dimensions':
368 table = gp[name][:]
368 table = gp[name][:]
369 listShapes = {}
369 listShapes = {}
370 for shapes in table:
370 for shapes in table:
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 else:
372 else:
373 data = gp[name].value
373 data = gp[name].value
374 listMetaname.append(name)
374 listMetaname.append(name)
375 listMetadata.append(data)
375 listMetadata.append(data)
376
376
377 # if name=='type':
377 # if name=='type':
378 # self.__initDataOut(data)
378 # self.__initDataOut(data)
379
379
380 self.listShapes = listShapes
380 self.listShapes = listShapes
381 self.listMetaname = listMetaname
381 self.listMetaname = listMetaname
382 self.listMeta = listMetadata
382 self.listMeta = listMetadata
383
383
384 fp.close()
384 fp.close()
385 return
385 return
386
386
387 def __readData(self):
387 def __readData(self):
388 grp = self.fp['Data']
388 grp = self.fp['Data']
389 listdataname = []
389 listdataname = []
390 listdata = []
390 listdata = []
391
391
392 for item in grp.items():
392 for item in grp.items():
393 name = item[0]
393 name = item[0]
394 listdataname.append(name)
394 listdataname.append(name)
395
395
396 array = self.__setDataArray(grp[name],self.listShapes[name])
396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 listdata.append(array)
397 listdata.append(array)
398
398
399 self.listDataname = listdataname
399 self.listDataname = listdataname
400 self.listData = listdata
400 self.listData = listdata
401 return
401 return
402
402
403 def __setDataArray(self, dataset, shapes):
403 def __setDataArray(self, dataset, shapes):
404
404
405 nDims = shapes[0]
405 nDims = shapes[0]
406
406
407 nDim2 = shapes[1] #Dimension 0
407 nDim2 = shapes[1] #Dimension 0
408
408
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410
410
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412
412
413 mode = shapes[4] #Mode of storing
413 mode = shapes[4] #Mode of storing
414
414
415 blockList = self.blockList
415 blockList = self.blockList
416
416
417 blocksPerFile = self.blocksPerFile
417 blocksPerFile = self.blocksPerFile
418
418
419 #Depending on what mode the data was stored
419 #Depending on what mode the data was stored
420 if mode == 0: #Divided in channels
420 if mode == 0: #Divided in channels
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 if mode == 1: #Divided in parameter
422 if mode == 1: #Divided in parameter
423 strds = 'table'
423 strds = 'table'
424 nDatas = nDim1
424 nDatas = nDim1
425 newShapes = (blocksPerFile,nDim2,nDim0)
425 newShapes = (blocksPerFile,nDim2,nDim0)
426 elif mode==2: #Concatenated in a table
426 elif mode==2: #Concatenated in a table
427 strds = 'table0'
427 strds = 'table0'
428 arrayData = dataset[strds].value
428 arrayData = dataset[strds].value
429 #Selecting part of the dataset
429 #Selecting part of the dataset
430 utctime = arrayData[:,0]
430 utctime = arrayData[:,0]
431 u, indices = numpy.unique(utctime, return_index=True)
431 u, indices = numpy.unique(utctime, return_index=True)
432
432
433 if blockList.size != indices.size:
433 if blockList.size != indices.size:
434 indMin = indices[blockList[0]]
434 indMin = indices[blockList[0]]
435 if blockList[1] + 1 >= indices.size:
435 if blockList[1] + 1 >= indices.size:
436 arrayData = arrayData[indMin:,:]
436 arrayData = arrayData[indMin:,:]
437 else:
437 else:
438 indMax = indices[blockList[1] + 1]
438 indMax = indices[blockList[1] + 1]
439 arrayData = arrayData[indMin:indMax,:]
439 arrayData = arrayData[indMin:indMax,:]
440 return arrayData
440 return arrayData
441
441
442 # One dimension
442 # One dimension
443 if nDims == 0:
443 if nDims == 0:
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445
445
446 # Two dimensions
446 # Two dimensions
447 elif nDims == 2:
447 elif nDims == 2:
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 newShapes = (blocksPerFile,nDim0)
449 newShapes = (blocksPerFile,nDim0)
450 nDatas = nDim1
450 nDatas = nDim1
451
451
452 for i in range(nDatas):
452 for i in range(nDatas):
453 data = dataset[strds + str(i)].value
453 data = dataset[strds + str(i)].value
454 arrayData[:,i,:] = data[blockList,:]
454 arrayData[:,i,:] = data[blockList,:]
455
455
456 # Three dimensions
456 # Three dimensions
457 else:
457 else:
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 for i in range(nDatas):
459 for i in range(nDatas):
460
460
461 data = dataset[strds + str(i)].value
461 data = dataset[strds + str(i)].value
462
462
463 for b in range(blockList.size):
463 for b in range(blockList.size):
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465
465
466 return arrayData
466 return arrayData
467
467
468 def __setDataOut(self):
468 def __setDataOut(self):
469 listMeta = self.listMeta
469 listMeta = self.listMeta
470 listMetaname = self.listMetaname
470 listMetaname = self.listMetaname
471 listDataname = self.listDataname
471 listDataname = self.listDataname
472 listData = self.listData
472 listData = self.listData
473 listShapes = self.listShapes
473 listShapes = self.listShapes
474
474
475 blockIndex = self.blockIndex
475 blockIndex = self.blockIndex
476 # blockList = self.blockList
476 # blockList = self.blockList
477
477
478 for i in range(len(listMeta)):
478 for i in range(len(listMeta)):
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480
480
481 for j in range(len(listData)):
481 for j in range(len(listData)):
482 nShapes = listShapes[listDataname[j]][0]
482 nShapes = listShapes[listDataname[j]][0]
483 mode = listShapes[listDataname[j]][4]
483 mode = listShapes[listDataname[j]][4]
484 if nShapes == 1:
484 if nShapes == 1:
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 elif nShapes > 1:
486 elif nShapes > 1:
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 elif mode==0:
488 elif mode==0:
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 #Mode Meteors
490 #Mode Meteors
491 elif mode ==2:
491 elif mode ==2:
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 setattr(self.dataOut, listDataname[j], selectedData)
493 setattr(self.dataOut, listDataname[j], selectedData)
494 return
494 return
495
495
496 def __selectDataMode2(self, data, blockIndex):
496 def __selectDataMode2(self, data, blockIndex):
497 utctime = data[:,0]
497 utctime = data[:,0]
498 aux, indices = numpy.unique(utctime, return_inverse=True)
498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 selInd = numpy.where(indices == blockIndex)[0]
499 selInd = numpy.where(indices == blockIndex)[0]
500 selData = data[selInd,:]
500 selData = data[selInd,:]
501
501
502 return selData
502 return selData
503
503
504 def getData(self):
504 def getData(self):
505
505
506 # if self.flagNoMoreFiles:
506 # if self.flagNoMoreFiles:
507 # self.dataOut.flagNoData = True
507 # self.dataOut.flagNoData = True
508 # print 'Process finished'
508 # print 'Process finished'
509 # return 0
509 # return 0
510 #
510 #
511 if self.blockIndex==self.blocksPerFile:
511 if self.blockIndex==self.blocksPerFile:
512 if not( self.__setNextFileOffline() ):
512 if not( self.__setNextFileOffline() ):
513 self.dataOut.flagNoData = True
513 self.dataOut.flagNoData = True
514 return 0
514 return 0
515
515
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 # self.dataOut.flagNoData = True
517 # self.dataOut.flagNoData = True
518 # return 0
518 # return 0
519 # self.__readData()
519 # self.__readData()
520 self.__setDataOut()
520 self.__setDataOut()
521 self.dataOut.flagNoData = False
521 self.dataOut.flagNoData = False
522
522
523 self.blockIndex += 1
523 self.blockIndex += 1
524
524
525 return
525 return
526
526
527 def run(self, **kwargs):
527 def run(self, **kwargs):
528
528
529 if not(self.isConfig):
529 if not(self.isConfig):
530 self.setup(**kwargs)
530 self.setup(**kwargs)
531 # self.setObjProperties()
531 # self.setObjProperties()
532 self.isConfig = True
532 self.isConfig = True
533
533
534 self.getData()
534 self.getData()
535
535
536 return
536 return
537
537
538 class ParamWriter(Operation):
538 class ParamWriter(Operation):
539 '''
539 '''
540 HDF5 Writer, stores parameters data in HDF5 format files
540 HDF5 Writer, stores parameters data in HDF5 format files
541
541
542 path: path where the files will be stored
542 path: path where the files will be stored
543
543
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545
545
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547
547
548 metadataList: list of attributes that will be stored as metadata
548 metadataList: list of attributes that will be stored as metadata
549
549
550 dataList: list of attributes that will be stores as data
550 dataList: list of attributes that will be stores as data
551
551
552 '''
552 '''
553
553
554
554
555 ext = ".hdf5"
555 ext = ".hdf5"
556
556
557 optchar = "D"
557 optchar = "D"
558
558
559 metaoptchar = "M"
559 metaoptchar = "M"
560
560
561 metaFile = None
561 metaFile = None
562
562
563 filename = None
563 filename = None
564
564
565 path = None
565 path = None
566
566
567 setFile = None
567 setFile = None
568
568
569 fp = None
569 fp = None
570
570
571 grp = None
571 grp = None
572
572
573 ds = None
573 ds = None
574
574
575 firsttime = True
575 firsttime = True
576
576
577 #Configurations
577 #Configurations
578
578
579 blocksPerFile = None
579 blocksPerFile = None
580
580
581 blockIndex = None
581 blockIndex = None
582
582
583 dataOut = None
583 dataOut = None
584
584
585 #Data Arrays
585 #Data Arrays
586
586
587 dataList = None
587 dataList = None
588
588
589 metadataList = None
589 metadataList = None
590
590
591 # arrayDim = None
591 # arrayDim = None
592
592
593 dsList = None #List of dictionaries with dataset properties
593 dsList = None #List of dictionaries with dataset properties
594
594
595 tableDim = None
595 tableDim = None
596
596
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598
598
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600
600
601 currentDay = None
601 currentDay = None
602
602
603 lastTime = None
603 lastTime = None
604
604
605 def __init__(self, **kwargs):
605 def __init__(self, **kwargs):
606 Operation.__init__(self, **kwargs)
606 Operation.__init__(self, **kwargs)
607 self.isConfig = False
607 self.isConfig = False
608 return
608 return
609
609
610 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
610 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 self.path = path
611 self.path = path
612 self.blocksPerFile = blocksPerFile
612 self.blocksPerFile = blocksPerFile
613 self.metadataList = metadataList
613 self.metadataList = metadataList
614 self.dataList = dataList
614 self.dataList = dataList
615 self.dataOut = dataOut
615 self.dataOut = dataOut
616 self.mode = mode
616 self.mode = mode
617
617
618 if self.mode is not None:
618 if self.mode is not None:
619 self.mode = numpy.zeros(len(self.dataList)) + mode
619 self.mode = numpy.zeros(len(self.dataList)) + mode
620 else:
620 else:
621 self.mode = numpy.ones(len(self.dataList))
621 self.mode = numpy.ones(len(self.dataList))
622
622
623 arrayDim = numpy.zeros((len(self.dataList),5))
623 arrayDim = numpy.zeros((len(self.dataList),5))
624
624
625 #Table dimensions
625 #Table dimensions
626 dtype0 = self.dtype
626 dtype0 = self.dtype
627 tableList = []
627 tableList = []
628
628
629 #Dictionary and list of tables
629 #Dictionary and list of tables
630 dsList = []
630 dsList = []
631
631
632 for i in range(len(self.dataList)):
632 for i in range(len(self.dataList)):
633 dsDict = {}
633 dsDict = {}
634 dataAux = getattr(self.dataOut, self.dataList[i])
634 dataAux = getattr(self.dataOut, self.dataList[i])
635 dsDict['variable'] = self.dataList[i]
635 dsDict['variable'] = self.dataList[i]
636 #--------------------- Conditionals ------------------------
636 #--------------------- Conditionals ------------------------
637 #There is no data
637 #There is no data
638 if dataAux is None:
638 if dataAux is None:
639 return 0
639 return 0
640
640
641 #Not array, just a number
641 #Not array, just a number
642 #Mode 0
642 #Mode 0
643 if type(dataAux)==float or type(dataAux)==int:
643 if type(dataAux)==float or type(dataAux)==int:
644 dsDict['mode'] = 0
644 dsDict['mode'] = 0
645 dsDict['nDim'] = 0
645 dsDict['nDim'] = 0
646 arrayDim[i,0] = 0
646 arrayDim[i,0] = 0
647 dsList.append(dsDict)
647 dsList.append(dsDict)
648
648
649 #Mode 2: meteors
649 #Mode 2: meteors
650 elif mode[i] == 2:
650 elif mode[i] == 2:
651 # dsDict['nDim'] = 0
651 # dsDict['nDim'] = 0
652 dsDict['dsName'] = 'table0'
652 dsDict['dsName'] = 'table0'
653 dsDict['mode'] = 2 # Mode meteors
653 dsDict['mode'] = 2 # Mode meteors
654 dsDict['shape'] = dataAux.shape[-1]
654 dsDict['shape'] = dataAux.shape[-1]
655 dsDict['nDim'] = 0
655 dsDict['nDim'] = 0
656 dsDict['dsNumber'] = 1
656 dsDict['dsNumber'] = 1
657
657
658 arrayDim[i,3] = dataAux.shape[-1]
658 arrayDim[i,3] = dataAux.shape[-1]
659 arrayDim[i,4] = mode[i] #Mode the data was stored
659 arrayDim[i,4] = mode[i] #Mode the data was stored
660
660
661 dsList.append(dsDict)
661 dsList.append(dsDict)
662
662
663 #Mode 1
663 #Mode 1
664 else:
664 else:
665 arrayDim0 = dataAux.shape #Data dimensions
665 arrayDim0 = dataAux.shape #Data dimensions
666 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
666 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 arrayDim[i,4] = mode[i] #Mode the data was stored
667 arrayDim[i,4] = mode[i] #Mode the data was stored
668
668
669 strtable = 'table'
669 strtable = 'table'
670 dsDict['mode'] = 1 # Mode parameters
670 dsDict['mode'] = 1 # Mode parameters
671
671
672 # Three-dimension arrays
672 # Three-dimension arrays
673 if len(arrayDim0) == 3:
673 if len(arrayDim0) == 3:
674 arrayDim[i,1:-1] = numpy.array(arrayDim0)
674 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 nTables = int(arrayDim[i,2])
675 nTables = int(arrayDim[i,2])
676 dsDict['dsNumber'] = nTables
676 dsDict['dsNumber'] = nTables
677 dsDict['shape'] = arrayDim[i,2:4]
677 dsDict['shape'] = arrayDim[i,2:4]
678 dsDict['nDim'] = 3
678 dsDict['nDim'] = 3
679
679
680 for j in range(nTables):
680 for j in range(nTables):
681 dsDict = dsDict.copy()
681 dsDict = dsDict.copy()
682 dsDict['dsName'] = strtable + str(j)
682 dsDict['dsName'] = strtable + str(j)
683 dsList.append(dsDict)
683 dsList.append(dsDict)
684
684
685 # Two-dimension arrays
685 # Two-dimension arrays
686 elif len(arrayDim0) == 2:
686 elif len(arrayDim0) == 2:
687 arrayDim[i,2:-1] = numpy.array(arrayDim0)
687 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 nTables = int(arrayDim[i,2])
688 nTables = int(arrayDim[i,2])
689 dsDict['dsNumber'] = nTables
689 dsDict['dsNumber'] = nTables
690 dsDict['shape'] = arrayDim[i,3]
690 dsDict['shape'] = arrayDim[i,3]
691 dsDict['nDim'] = 2
691 dsDict['nDim'] = 2
692
692
693 for j in range(nTables):
693 for j in range(nTables):
694 dsDict = dsDict.copy()
694 dsDict = dsDict.copy()
695 dsDict['dsName'] = strtable + str(j)
695 dsDict['dsName'] = strtable + str(j)
696 dsList.append(dsDict)
696 dsList.append(dsDict)
697
697
698 # One-dimension arrays
698 # One-dimension arrays
699 elif len(arrayDim0) == 1:
699 elif len(arrayDim0) == 1:
700 arrayDim[i,3] = arrayDim0[0]
700 arrayDim[i,3] = arrayDim0[0]
701 dsDict['shape'] = arrayDim0[0]
701 dsDict['shape'] = arrayDim0[0]
702 dsDict['dsNumber'] = 1
702 dsDict['dsNumber'] = 1
703 dsDict['dsName'] = strtable + str(0)
703 dsDict['dsName'] = strtable + str(0)
704 dsDict['nDim'] = 1
704 dsDict['nDim'] = 1
705 dsList.append(dsDict)
705 dsList.append(dsDict)
706
706
707 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
707 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 tableList.append(table)
708 tableList.append(table)
709
709
710 # self.arrayDim = arrayDim
710 # self.arrayDim = arrayDim
711 self.dsList = dsList
711 self.dsList = dsList
712 self.tableDim = numpy.array(tableList, dtype = dtype0)
712 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 self.blockIndex = 0
713 self.blockIndex = 0
714
714
715 timeTuple = time.localtime(dataOut.utctime)
715 timeTuple = time.localtime(dataOut.utctime)
716 self.currentDay = timeTuple.tm_yday
716 self.currentDay = timeTuple.tm_yday
717 return 1
717 return 1
718
718
719 def putMetadata(self):
719 def putMetadata(self):
720
720
721 fp = self.createMetadataFile()
721 fp = self.createMetadataFile()
722 self.writeMetadata(fp)
722 self.writeMetadata(fp)
723 fp.close()
723 fp.close()
724 return
724 return
725
725
726 def createMetadataFile(self):
726 def createMetadataFile(self):
727 ext = self.ext
727 ext = self.ext
728 path = self.path
728 path = self.path
729 setFile = self.setFile
729 setFile = self.setFile
730
730
731 timeTuple = time.localtime(self.dataOut.utctime)
731 timeTuple = time.localtime(self.dataOut.utctime)
732
732
733 subfolder = ''
733 subfolder = ''
734 fullpath = os.path.join( path, subfolder )
734 fullpath = os.path.join( path, subfolder )
735
735
736 if not( os.path.exists(fullpath) ):
736 if not( os.path.exists(fullpath) ):
737 os.mkdir(fullpath)
737 os.mkdir(fullpath)
738 setFile = -1 #inicializo mi contador de seteo
738 setFile = -1 #inicializo mi contador de seteo
739
739
740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 fullpath = os.path.join( path, subfolder )
741 fullpath = os.path.join( path, subfolder )
742
742
743 if not( os.path.exists(fullpath) ):
743 if not( os.path.exists(fullpath) ):
744 os.mkdir(fullpath)
744 os.mkdir(fullpath)
745 setFile = -1 #inicializo mi contador de seteo
745 setFile = -1 #inicializo mi contador de seteo
746
746
747 else:
747 else:
748 filesList = os.listdir( fullpath )
748 filesList = os.listdir( fullpath )
749 filesList = sorted( filesList, key=str.lower )
749 filesList = sorted( filesList, key=str.lower )
750 if len( filesList ) > 0:
750 if len( filesList ) > 0:
751 filesList = [k for k in filesList if 'M' in k]
751 filesList = [k for k in filesList if 'M' in k]
752 filen = filesList[-1]
752 filen = filesList[-1]
753 # el filename debera tener el siguiente formato
753 # el filename debera tener el siguiente formato
754 # 0 1234 567 89A BCDE (hex)
754 # 0 1234 567 89A BCDE (hex)
755 # x YYYY DDD SSS .ext
755 # x YYYY DDD SSS .ext
756 if isNumber( filen[8:11] ):
756 if isNumber( filen[8:11] ):
757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 else:
758 else:
759 setFile = -1
759 setFile = -1
760 else:
760 else:
761 setFile = -1 #inicializo mi contador de seteo
761 setFile = -1 #inicializo mi contador de seteo
762
762
763 if self.setType is None:
763 if self.setType is None:
764 setFile += 1
764 setFile += 1
765 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
765 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 timeTuple.tm_year,
766 timeTuple.tm_year,
767 timeTuple.tm_yday,
767 timeTuple.tm_yday,
768 setFile,
768 setFile,
769 ext )
769 ext )
770 else:
770 else:
771 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
771 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
772 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 timeTuple.tm_year,
773 timeTuple.tm_year,
774 timeTuple.tm_yday,
774 timeTuple.tm_yday,
775 setFile,
775 setFile,
776 ext )
776 ext )
777
777
778 filename = os.path.join( path, subfolder, file )
778 filename = os.path.join( path, subfolder, file )
779 self.metaFile = file
779 self.metaFile = file
780 #Setting HDF5 File
780 #Setting HDF5 File
781 fp = h5py.File(filename,'w')
781 fp = h5py.File(filename,'w')
782
782
783 return fp
783 return fp
784
784
785 def writeMetadata(self, fp):
785 def writeMetadata(self, fp):
786
786
787 grp = fp.create_group("Metadata")
787 grp = fp.create_group("Metadata")
788 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
788 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789
789
790 for i in range(len(self.metadataList)):
790 for i in range(len(self.metadataList)):
791 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
791 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 return
792 return
793
793
794 def timeFlag(self):
794 def timeFlag(self):
795 currentTime = self.dataOut.utctime
795 currentTime = self.dataOut.utctime
796
796
797 if self.lastTime is None:
797 if self.lastTime is None:
798 self.lastTime = currentTime
798 self.lastTime = currentTime
799
799
800 #Day
800 #Day
801 timeTuple = time.localtime(currentTime)
801 timeTuple = time.localtime(currentTime)
802 dataDay = timeTuple.tm_yday
802 dataDay = timeTuple.tm_yday
803
803
804 #Time
804 #Time
805 timeDiff = currentTime - self.lastTime
805 timeDiff = currentTime - self.lastTime
806
806
807 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
807 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 if dataDay != self.currentDay:
808 if dataDay != self.currentDay:
809 self.currentDay = dataDay
809 self.currentDay = dataDay
810 return True
810 return True
811 elif timeDiff > 3*60*60:
811 elif timeDiff > 3*60*60:
812 self.lastTime = currentTime
812 self.lastTime = currentTime
813 return True
813 return True
814 else:
814 else:
815 self.lastTime = currentTime
815 self.lastTime = currentTime
816 return False
816 return False
817
817
818 def setNextFile(self):
818 def setNextFile(self):
819
819
820 ext = self.ext
820 ext = self.ext
821 path = self.path
821 path = self.path
822 setFile = self.setFile
822 setFile = self.setFile
823 mode = self.mode
823 mode = self.mode
824
824
825 timeTuple = time.localtime(self.dataOut.utctime)
825 timeTuple = time.localtime(self.dataOut.utctime)
826 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
826 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827
827
828 fullpath = os.path.join( path, subfolder )
828 fullpath = os.path.join( path, subfolder )
829
829
830 if os.path.exists(fullpath):
830 if os.path.exists(fullpath):
831 filesList = os.listdir( fullpath )
831 filesList = os.listdir( fullpath )
832 filesList = [k for k in filesList if 'D' in k]
832 filesList = [k for k in filesList if 'D' in k]
833 if len( filesList ) > 0:
833 if len( filesList ) > 0:
834 filesList = sorted( filesList, key=str.lower )
834 filesList = sorted( filesList, key=str.lower )
835 filen = filesList[-1]
835 filen = filesList[-1]
836 # el filename debera tener el siguiente formato
836 # el filename debera tener el siguiente formato
837 # 0 1234 567 89A BCDE (hex)
837 # 0 1234 567 89A BCDE (hex)
838 # x YYYY DDD SSS .ext
838 # x YYYY DDD SSS .ext
839 if isNumber( filen[8:11] ):
839 if isNumber( filen[8:11] ):
840 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
840 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 else:
841 else:
842 setFile = -1
842 setFile = -1
843 else:
843 else:
844 setFile = -1 #inicializo mi contador de seteo
844 setFile = -1 #inicializo mi contador de seteo
845 else:
845 else:
846 os.makedirs(fullpath)
846 os.makedirs(fullpath)
847 setFile = -1 #inicializo mi contador de seteo
847 setFile = -1 #inicializo mi contador de seteo
848
848
849 if self.setType is None:
849 if self.setType is None:
850 setFile += 1
850 setFile += 1
851 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
851 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 timeTuple.tm_year,
852 timeTuple.tm_year,
853 timeTuple.tm_yday,
853 timeTuple.tm_yday,
854 setFile,
854 setFile,
855 ext )
855 ext )
856 else:
856 else:
857 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
857 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
858 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 timeTuple.tm_year,
859 timeTuple.tm_year,
860 timeTuple.tm_yday,
860 timeTuple.tm_yday,
861 setFile,
861 setFile,
862 ext )
862 ext )
863
863
864 filename = os.path.join( path, subfolder, file )
864 filename = os.path.join( path, subfolder, file )
865
865
866 #Setting HDF5 File
866 #Setting HDF5 File
867 fp = h5py.File(filename,'w')
867 fp = h5py.File(filename,'w')
868 #write metadata
868 #write metadata
869 self.writeMetadata(fp)
869 self.writeMetadata(fp)
870 #Write data
870 #Write data
871 grp = fp.create_group("Data")
871 grp = fp.create_group("Data")
872 # grp.attrs['metadata'] = self.metaFile
872 # grp.attrs['metadata'] = self.metaFile
873
873
874 # grp.attrs['blocksPerFile'] = 0
874 # grp.attrs['blocksPerFile'] = 0
875 ds = []
875 ds = []
876 data = []
876 data = []
877 dsList = self.dsList
877 dsList = self.dsList
878 i = 0
878 i = 0
879 while i < len(dsList):
879 while i < len(dsList):
880 dsInfo = dsList[i]
880 dsInfo = dsList[i]
881 #One-dimension data
881 #One-dimension data
882 if dsInfo['mode'] == 0:
882 if dsInfo['mode'] == 0:
883 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
883 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
884 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 ds.append(ds0)
885 ds.append(ds0)
886 data.append([])
886 data.append([])
887 i += 1
887 i += 1
888 continue
888 continue
889 # nDimsForDs.append(nDims[i])
889 # nDimsForDs.append(nDims[i])
890
890
891 elif dsInfo['mode'] == 2:
891 elif dsInfo['mode'] == 2:
892 grp0 = grp.create_group(dsInfo['variable'])
892 grp0 = grp.create_group(dsInfo['variable'])
893 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
893 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 ds.append(ds0)
894 ds.append(ds0)
895 data.append([])
895 data.append([])
896 i += 1
896 i += 1
897 continue
897 continue
898
898
899 elif dsInfo['mode'] == 1:
899 elif dsInfo['mode'] == 1:
900 grp0 = grp.create_group(dsInfo['variable'])
900 grp0 = grp.create_group(dsInfo['variable'])
901
901
902 for j in range(dsInfo['dsNumber']):
902 for j in range(dsInfo['dsNumber']):
903 dsInfo = dsList[i]
903 dsInfo = dsList[i]
904 tableName = dsInfo['dsName']
904 tableName = dsInfo['dsName']
905 shape = int(dsInfo['shape'])
905 shape = int(dsInfo['shape'])
906
906
907 if dsInfo['nDim'] == 3:
907 if dsInfo['nDim'] == 3:
908 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
908 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 else:
909 else:
910 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
910 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911
911
912 ds.append(ds0)
912 ds.append(ds0)
913 data.append([])
913 data.append([])
914 i += 1
914 i += 1
915 # nDimsForDs.append(nDims[i])
915 # nDimsForDs.append(nDims[i])
916
916
917 fp.flush()
917 fp.flush()
918 fp.close()
918 fp.close()
919
919
920 # self.nDatas = nDatas
920 # self.nDatas = nDatas
921 # self.nDims = nDims
921 # self.nDims = nDims
922 # self.nDimsForDs = nDimsForDs
922 # self.nDimsForDs = nDimsForDs
923 #Saving variables
923 #Saving variables
924 print 'Writing the file: %s'%filename
924 print 'Writing the file: %s'%filename
925 self.filename = filename
925 self.filename = filename
926 # self.fp = fp
926 # self.fp = fp
927 # self.grp = grp
927 # self.grp = grp
928 # self.grp.attrs.modify('nRecords', 1)
928 # self.grp.attrs.modify('nRecords', 1)
929 self.ds = ds
929 self.ds = ds
930 self.data = data
930 self.data = data
931 # self.setFile = setFile
931 # self.setFile = setFile
932 self.firsttime = True
932 self.firsttime = True
933 self.blockIndex = 0
933 self.blockIndex = 0
934 return
934 return
935
935
936 def putData(self):
936 def putData(self):
937
937
938 if self.blockIndex == self.blocksPerFile or self.timeFlag():
938 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 self.setNextFile()
939 self.setNextFile()
940
940
941 # if not self.firsttime:
941 # if not self.firsttime:
942 self.readBlock()
942 self.readBlock()
943 self.setBlock() #Prepare data to be written
943 self.setBlock() #Prepare data to be written
944 self.writeBlock() #Write data
944 self.writeBlock() #Write data
945
945
946 return
946 return
947
947
948 def readBlock(self):
948 def readBlock(self):
949
949
950 '''
950 '''
951 data Array configured
951 data Array configured
952
952
953
953
954 self.data
954 self.data
955 '''
955 '''
956 dsList = self.dsList
956 dsList = self.dsList
957 ds = self.ds
957 ds = self.ds
958 #Setting HDF5 File
958 #Setting HDF5 File
959 fp = h5py.File(self.filename,'r+')
959 fp = h5py.File(self.filename,'r+')
960 grp = fp["Data"]
960 grp = fp["Data"]
961 ind = 0
961 ind = 0
962
962
963 # grp.attrs['blocksPerFile'] = 0
963 # grp.attrs['blocksPerFile'] = 0
964 while ind < len(dsList):
964 while ind < len(dsList):
965 dsInfo = dsList[ind]
965 dsInfo = dsList[ind]
966
966
967 if dsInfo['mode'] == 0:
967 if dsInfo['mode'] == 0:
968 ds0 = grp[dsInfo['variable']]
968 ds0 = grp[dsInfo['variable']]
969 ds[ind] = ds0
969 ds[ind] = ds0
970 ind += 1
970 ind += 1
971 else:
971 else:
972
972
973 grp0 = grp[dsInfo['variable']]
973 grp0 = grp[dsInfo['variable']]
974
974
975 for j in range(dsInfo['dsNumber']):
975 for j in range(dsInfo['dsNumber']):
976 dsInfo = dsList[ind]
976 dsInfo = dsList[ind]
977 ds0 = grp0[dsInfo['dsName']]
977 ds0 = grp0[dsInfo['dsName']]
978 ds[ind] = ds0
978 ds[ind] = ds0
979 ind += 1
979 ind += 1
980
980
981 self.fp = fp
981 self.fp = fp
982 self.grp = grp
982 self.grp = grp
983 self.ds = ds
983 self.ds = ds
984
984
985 return
985 return
986
986
987 def setBlock(self):
987 def setBlock(self):
988 '''
988 '''
989 data Array configured
989 data Array configured
990
990
991
991
992 self.data
992 self.data
993 '''
993 '''
994 #Creating Arrays
994 #Creating Arrays
995 dsList = self.dsList
995 dsList = self.dsList
996 data = self.data
996 data = self.data
997 ind = 0
997 ind = 0
998
998
999 while ind < len(dsList):
999 while ind < len(dsList):
1000 dsInfo = dsList[ind]
1000 dsInfo = dsList[ind]
1001 dataAux = getattr(self.dataOut, dsInfo['variable'])
1001 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002
1002
1003 mode = dsInfo['mode']
1003 mode = dsInfo['mode']
1004 nDim = dsInfo['nDim']
1004 nDim = dsInfo['nDim']
1005
1005
1006 if mode == 0 or mode == 2 or nDim == 1:
1006 if mode == 0 or mode == 2 or nDim == 1:
1007 data[ind] = dataAux
1007 data[ind] = dataAux
1008 ind += 1
1008 ind += 1
1009 # elif nDim == 1:
1009 # elif nDim == 1:
1010 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1010 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 # ind += 1
1011 # ind += 1
1012 elif nDim == 2:
1012 elif nDim == 2:
1013 for j in range(dsInfo['dsNumber']):
1013 for j in range(dsInfo['dsNumber']):
1014 data[ind] = dataAux[j,:]
1014 data[ind] = dataAux[j,:]
1015 ind += 1
1015 ind += 1
1016 elif nDim == 3:
1016 elif nDim == 3:
1017 for j in range(dsInfo['dsNumber']):
1017 for j in range(dsInfo['dsNumber']):
1018 data[ind] = dataAux[:,j,:]
1018 data[ind] = dataAux[:,j,:]
1019 ind += 1
1019 ind += 1
1020
1020
1021 self.data = data
1021 self.data = data
1022 return
1022 return
1023
1023
1024 def writeBlock(self):
1024 def writeBlock(self):
1025 '''
1025 '''
1026 Saves the block in the HDF5 file
1026 Saves the block in the HDF5 file
1027 '''
1027 '''
1028 dsList = self.dsList
1028 dsList = self.dsList
1029
1029
1030 for i in range(len(self.ds)):
1030 for i in range(len(self.ds)):
1031 dsInfo = dsList[i]
1031 dsInfo = dsList[i]
1032 nDim = dsInfo['nDim']
1032 nDim = dsInfo['nDim']
1033 mode = dsInfo['mode']
1033 mode = dsInfo['mode']
1034
1034
1035 # First time
1035 # First time
1036 if self.firsttime:
1036 if self.firsttime:
1037 # self.ds[i].resize(self.data[i].shape)
1037 # self.ds[i].resize(self.data[i].shape)
1038 # self.ds[i][self.blockIndex,:] = self.data[i]
1038 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 if type(self.data[i]) == numpy.ndarray:
1039 if type(self.data[i]) == numpy.ndarray:
1040
1040
1041 if nDim == 3:
1041 if nDim == 3:
1042 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1042 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 self.ds[i].resize(self.data[i].shape)
1043 self.ds[i].resize(self.data[i].shape)
1044 if mode == 2:
1044 if mode == 2:
1045 self.ds[i].resize(self.data[i].shape)
1045 self.ds[i].resize(self.data[i].shape)
1046 self.ds[i][:] = self.data[i]
1046 self.ds[i][:] = self.data[i]
1047 else:
1047 else:
1048
1048
1049 # From second time
1049 # From second time
1050 # Meteors!
1050 # Meteors!
1051 if mode == 2:
1051 if mode == 2:
1052 dataShape = self.data[i].shape
1052 dataShape = self.data[i].shape
1053 dsShape = self.ds[i].shape
1053 dsShape = self.ds[i].shape
1054 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1054 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 self.ds[i][dsShape[0]:,:] = self.data[i]
1055 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 # No dimension
1056 # No dimension
1057 elif mode == 0:
1057 elif mode == 0:
1058 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1058 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 self.ds[i][0,-1] = self.data[i]
1059 self.ds[i][0,-1] = self.data[i]
1060 # One dimension
1060 # One dimension
1061 elif nDim == 1:
1061 elif nDim == 1:
1062 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1062 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 self.ds[i][-1,:] = self.data[i]
1063 self.ds[i][-1,:] = self.data[i]
1064 # Two dimension
1064 # Two dimension
1065 elif nDim == 2:
1065 elif nDim == 2:
1066 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1066 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 self.ds[i][self.blockIndex,:] = self.data[i]
1067 self.ds[i][self.blockIndex,:] = self.data[i]
1068 # Three dimensions
1068 # Three dimensions
1069 elif nDim == 3:
1069 elif nDim == 3:
1070 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1070 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 self.ds[i][:,:,-1] = self.data[i]
1071 self.ds[i][:,:,-1] = self.data[i]
1072
1072
1073 self.firsttime = False
1073 self.firsttime = False
1074 self.blockIndex += 1
1074 self.blockIndex += 1
1075
1075
1076 #Close to save changes
1076 #Close to save changes
1077 self.fp.flush()
1077 self.fp.flush()
1078 self.fp.close()
1078 self.fp.close()
1079 return
1079 return
1080
1080
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082
1082
1083 if not(self.isConfig):
1083 if not(self.isConfig):
1084 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1084 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1085 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086
1086
1087 if not(flagdata):
1087 if not(flagdata):
1088 return
1088 return
1089
1089
1090 self.isConfig = True
1090 self.isConfig = True
1091 # self.putMetadata()
1091 # self.putMetadata()
1092 self.setNextFile()
1092 self.setNextFile()
1093
1093
1094 self.putData()
1094 self.putData()
1095 return
1095 return
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now