##// END OF EJS Templates
schain
RSS
Alarm working ok
jespinoza -
r1130:d903be96a0b2
parent child
Show More
Side by Side Unified
Context file:
r1130:d903be96a0b2 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,498 +1,503
1 """The admin module contains all administrative classes relating to the schain python api.
1 """
2 The admin module contains all administrative classes relating to the schain python api.
2 3
3 4 The main role of this module is to send some reports. It contains a
4 5 notification class and a standard error handing class.
5 6
6 7 $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
7 8 """
8 9 import os
9 10 import sys
10 11 import time
11 12 import traceback
12 13 import smtplib
13 14 import ConfigParser
14 15 import StringIO
15 16 from threading import Thread
16 17 from multiprocessing import Process
17 18 from email.mime.text import MIMEText
18 19 from email.mime.application import MIMEApplication
19 20 from email.mime.multipart import MIMEMultipart
20 21
22 import schainpy
21 23 from schainpy.utils import log
22 24 from schainpy.model.graphics.jroplot_data import popup
23 25
24 26 def get_path():
25 27 '''
26 28 Return schainpy path
27 29 '''
28 30
29 31 try:
30 32 root = __file__
31 33 if os.path.islink(root):
32 34 root = os.path.realpath(root)
33 35
34 36 return os.path.dirname(os.path.abspath(root))
35 37 except:
36 38 log.error('I am sorry, but something is wrong... __file__ not found')
37 39
38 40 class Alarm(Process):
39 41 '''
40 42 modes:
41 43 0 - All
42 1 - Sound alarm
43 2 - Send email
44 3 - Popup message
44 1 - Send email
45 2 - Popup message
46 3 - Sound alarm
45 47 4 - Send to alarm system TODO
46 48 '''
47 49
48 def __init__(self, modes=[1], **kwargs):
50 def __init__(self, modes=[], **kwargs):
49 51 Process.__init__(self)
50 52 self.modes = modes
51 53 self.kwargs = kwargs
52 54
53 55 @staticmethod
54 56 def play_sound():
55 57 sound = os.path.join(get_path(), 'alarm1.oga')
56 58 if os.path.exists(sound):
57 59 for __ in range(2):
58 60 os.system('paplay {}'.format(sound))
59 61 time.sleep(0.5)
60 62 else:
61 63 log.warning('Unable to play alarm, sound file not found', 'ADMIN')
62 64
63 65 @staticmethod
64 66 def send_email(**kwargs):
65 67 notifier = SchainNotify()
68 print kwargs
66 69 notifier.notify(**kwargs)
67 70
68 71 @staticmethod
69 def show_popup(message='Error'):
72 def show_popup(message):
73 if isinstance(message, list):
74 message = message[-1]
70 75 popup(message)
71 76
72 77 @staticmethod
73 78 def send_alarm():
74 79 pass
75 80
76 81 @staticmethod
77 82 def get_kwargs(kwargs, keys):
78 83 ret = {}
79 84 for key in keys:
80 85 ret[key] = kwargs[key]
81 86 return ret
82 87
83 88 def run(self):
84 89 tasks = {
85 90 1 : self.send_email,
86 2 : self.play_sound,
87 3 : self.show_popup,
91 2 : self.show_popup,
92 3 : self.play_sound,
88 93 4 : self.send_alarm,
89 94 }
90 95
91 96 tasks_args = {
92 97 1: ['email', 'message', 'subject', 'subtitle', 'filename'],
93 2: [],
94 3: ['message'],
98 2: ['message'],
99 3: [],
95 100 4: [],
96 101 }
97 102 procs = []
98 103 for mode in self.modes:
99 104 if 0 in self.modes:
100 105 for x in tasks:
101 106 t = Thread(target=tasks[x], kwargs=self.get_kwargs(self.kwargs, tasks_args[x]))
102 107 t.start()
103 108 procs.append(t)
104 109 break
105 110 else:
106 111 t = Thread(target=tasks[mode], kwargs=self.get_kwargs(self.kwargs, tasks_args[mode]))
107 112 t.start()
108 113 procs.append(t)
109 114 for t in procs:
110 115 t.join()
111 116
112 117
113 118 class SchainConfigure():
114 119
115 120 __DEFAULT_ADMINISTRATOR_EMAIL = "juan.espinoza@jro.igp.gob.pe"
116 121 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
117 122 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
118 123 __DEFAULT_SENDER_PASS = ""
119 124
120 125 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
121 126 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
122 127 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
123 128 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
124 129
125 130 def __init__(self, initFile = None):
126 131
127 132 # Set configuration file
128 133 if (initFile == None):
129 134 self.__confFilePath = "/etc/schain.conf"
130 135 else:
131 136 self.__confFilePath = initFile
132 137
133 138 # open configuration file
134 139 try:
135 140 self.__confFile = open(self.__confFilePath, "r")
136 141 except IOError:
137 142 # can't read from file - use all hard-coded values
138 143 self.__initFromHardCode()
139 144 return
140 145
141 146 # create Parser using standard module ConfigParser
142 147 self.__parser = ConfigParser.ConfigParser()
143 148
144 149 # read conf file into a StringIO with "[madrigal]\n" section heading prepended
145 150 strConfFile = StringIO.StringIO("[schain]\n" + self.__confFile.read())
146 151
147 152 # parse StringIO configuration file
148 153 self.__parser.readfp(strConfFile)
149 154
150 155 # read information from configuration file
151 156 self.__readConfFile()
152 157
153 158 # close conf file
154 159 self.__confFile.close()
155 160
156 161
157 162 def __initFromHardCode(self):
158 163
159 164 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
160 165 self.__sender_pass = self.__DEFAULT_SENDER_PASS
161 166 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
162 167 self.__email_server = self.__DEFAULT_EMAIL_SERVER
163 168
164 169 def __readConfFile(self):
165 170 """__readConfFile is a private helper function that reads information from the parsed config file.
166 171
167 172 Inputs: None
168 173
169 174 Returns: Void.
170 175
171 176 Affects: Initializes class member variables that are found in the config file.
172 177
173 178 Exceptions: MadrigalError thrown if any key not found.
174 179 """
175 180
176 181 # get the sender email
177 182 try:
178 183 self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
179 184 except:
180 185 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
181 186
182 187 # get the sender password
183 188 try:
184 189 self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
185 190 except:
186 191 self.__sender_pass = self.__DEFAULT_SENDER_PASS
187 192
188 193 # get the administrator email
189 194 try:
190 195 self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
191 196 except:
192 197 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
193 198
194 199 # get the server email
195 200 try:
196 201 self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
197 202 except:
198 203 self.__email_server = self.__DEFAULT_EMAIL_SERVER
199 204
200 205 def getEmailServer(self):
201 206
202 207 return self.__email_server
203 208
204 209 def getSenderEmail(self):
205 210
206 211 return self.__sender_email
207 212
208 213 def getSenderPass(self):
209 214
210 215 return self.__sender_pass
211 216
212 217 def getAdminEmail(self):
213 218
214 219 return self.__admin_email
215 220
216 221 class SchainNotify:
217 222 """SchainNotify is an object used to send messages to an administrator about a Schain software.
218 223
219 224 This object provides functions needed to send messages to an administrator about a Schain , for now
220 225 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
221 226
222 227 Usage example:
223 228
224 229 import schainpy.admin
225 230
226 231 try:
227 232
228 233 adminObj = schainpy.admin.SchainNotify()
229 234 adminObj.sendAlert('This is important!', 'Important Message')
230 235
231 236 except schainpy.admin.SchainError, e:
232 237
233 238 print e.getExceptionStr()
234 239
235 240
236 241 Non-standard Python modules used:
237 242 None
238 243
239 244 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
240 245 throwing exceptions, since this is the class that will generally be called when there is a problem.
241 246
242 247 Change history:
243 248
244 249 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Dec. 1, 2015
245 250 """
246 251
247 252 #constants
248 253
249 254 def __init__(self):
250 255 """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
251 256
252 257 Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
253 258 this is the class that will generally be called when there is a problem.
254 259
255 260 Inputs: Existing SchainDB object, by default = None.
256 261
257 262 Returns: void
258 263
259 264 Affects: Initializes self.__binDir.
260 265
261 266 Exceptions: None.
262 267 """
263 268
264 269 # note that the main configuration file is unavailable
265 270 # the best that can be done is send an email to root using localhost mailserver
266 271 confObj = SchainConfigure()
267 272
268 273 self.__emailFromAddress = confObj.getSenderEmail()
269 274 self.__emailPass = confObj.getSenderPass()
270 275 self.__emailToAddress = confObj.getAdminEmail()
271 276 self.__emailServer = confObj.getEmailServer()
272 277
273 278 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
274 279
275 280 if not email_to:
276 281 return 0
277 282
278 283 if not self.__emailServer:
279 284 return 0
280 285
281 286 log.success('Sending email to {}...'.format(email_to), 'System')
282 287
283 288 msg = MIMEMultipart()
284 289 msg['Subject'] = subject
285 290 msg['From'] = "(Python SChain API): " + email_from
286 291 msg['Reply-to'] = email_from
287 292 msg['To'] = email_to
288 293
289 294 # That is what u see if dont have an email reader:
290 295 msg.preamble = 'SChainPy'
291 296
292 297 if html_format:
293 298 message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
294 299 message = "<html>\n" + message + '</html>'
295 300
296 301 # This is the textual part:
297 302 part = MIMEText(message, "html")
298 303 else:
299 304 message = subject + "\n" + subtitle + "\n" + message
300 305 part = MIMEText(message)
301 306
302 307 msg.attach(part)
303 308
304 309 if filename and os.path.isfile(filename):
305 310 # This is the binary part(The Attachment):
306 311 part = MIMEApplication(open(filename,"rb").read())
307 312 part.add_header('Content-Disposition',
308 313 'attachment',
309 314 filename=os.path.basename(filename))
310 315 msg.attach(part)
311 316
312 317 # Create an instance in SMTP server
313 318 try:
314 319 smtp = smtplib.SMTP(self.__emailServer)
315 320 except:
316 321 log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
317 322 return 0
318 323
319 324 # Start the server:
320 325 # smtp.ehlo()
321 326 if self.__emailPass:
322 327 smtp.login(self.__emailFromAddress, self.__emailPass)
323 328
324 329 # Send the email
325 330 try:
326 331 smtp.sendmail(msg['From'], msg['To'], msg.as_string())
327 332 except:
328 333 log.error('Could not send the email to {}'.format(msg['To']), 'System')
329 334 smtp.quit()
330 335 return 0
331 336
332 337 smtp.quit()
333 338
334 339 log.success('Email sent ', 'System')
335 340
336 341 return 1
337 342
338 343 def sendAlert(self, message, subject = "", subtitle="", filename=""):
339 344 """sendAlert sends an email with the given message and optional title.
340 345
341 346 Inputs: message (string), and optional title (string)
342 347
343 348 Returns: void
344 349
345 350 Affects: none
346 351
347 352 Exceptions: None.
348 353 """
349 354
350 355 if not self.__emailToAddress:
351 356 return 0
352 357
353 358 print "***** Sending alert to %s *****" %self.__emailToAddress
354 359 # set up message
355 360
356 361 sent=self.sendEmail(email_from=self.__emailFromAddress,
357 362 email_to=self.__emailToAddress,
358 363 subject=subject,
359 364 message=message,
360 365 subtitle=subtitle,
361 366 filename=filename)
362 367
363 368 if not sent:
364 369 return 0
365 370
366 371 return 1
367 372
368 373 def notify(self, email, message, subject = "", subtitle="", filename=""):
369 374 """notify sends an email with the given message and title to email.
370 375
371 376 Inputs: email (string), message (string), and subject (string)
372 377
373 378 Returns: void
374 379
375 380 Affects: none
376 381
377 382 Exceptions: None.
378 383 """
379 384
380 385 if email is None:
381 386 email = self.__emailToAddress
382 387
383 388 self.sendEmail(
384 389 email_from=self.__emailFromAddress,
385 390 email_to=email,
386 391 subject=subject,
387 392 message=message,
388 393 subtitle=subtitle,
389 394 filename=filename
390 395 )
391 396
392 397
393 398 class SchainError(Exception):
394 399 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
395 400
396 401 Usage example:
397 402
398 403 import sys, traceback
399 404 import schainpy.admin
400 405
401 406 try:
402 407
403 408 test = open('ImportantFile.txt', 'r')
404 409
405 410 except:
406 411
407 412 raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
408 413 traceback.format_exception(sys.exc_info()[0],
409 414 sys.exc_info()[1],
410 415 sys.exc_info()[2]))
411 416 """
412 417
413 418
414 419 def __init__(self, strInterpretation, exceptionList=None):
415 420 """ __init__ gathers the interpretation string along with all information from sys.exc_info().
416 421
417 422 Inputs:
418 423 strIntepretation - A string representing the programmer's interpretation of
419 424 why the exception occurred
420 425
421 426 exceptionList - a list of strings completely describing the exception.
422 427 Generated by traceback.format_exception(sys.exc_info()[0],
423 428 sys.exc_info()[1],
424 429 sys.exc_info()[2])
425 430
426 431 Returns: Void.
427 432
428 433 Affects: Initializes class member variables _strInterp, _strExcList.
429 434
430 435 Exceptions: None.
431 436 """
432 437
433 438 if not exceptionList:
434 439 exceptionList = traceback.format_exception(sys.exc_info()[0],
435 440 sys.exc_info()[1],
436 441 sys.exc_info()[2])
437 442
438 443 self._strInterp = strInterpretation
439 444 self._strExcList = exceptionList
440 445
441 446
442 447 def getExceptionStr(self):
443 448 """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
444 449
445 450 Inputs: None
446 451
447 452 Returns: A formatted string ready for printing completely describing the exception.
448 453
449 454 Affects: None
450 455
451 456 Exceptions: None.
452 457 """
453 458 excStr = ''
454 459 excStr = excStr + self._strInterp + '\n\n'
455 460
456 461 if self._strExcList != None:
457 462 for item in self._strExcList:
458 463 excStr = excStr + str(item) + '\n'
459 464
460 465 return excStr
461 466
462 467 def __str__(self):
463 468
464 469 return(self.getExceptionStr())
465 470
466 471
467 472 def getExceptionHtml(self):
468 473 """ getExceptionHtml returns an Html formatted string completely describing the exception.
469 474
470 475 Inputs: None
471 476
472 477 Returns: A formatted string ready for printing completely describing the exception.
473 478
474 479 Affects: None
475 480
476 481 Exceptions: None.
477 482 """
478 483
479 484 excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
480 485 excStr = excStr + self._strInterp + '\n<BR>\n'
481 486
482 487 if self._strExcList != None:
483 488 for item in self._strExcList:
484 489 excStr = excStr + str(item) + '\n<BR>'
485 490
486 491 return excStr
487 492
488 493 class SchainWarning(Exception):
489 494 pass
490 495
491 496
492 497 if __name__ == '__main__':
493 498
494 499 test = SchainNotify()
495 500
496 501 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
497 502
498 503 print 'Hopefully message sent - check.'
Show file before | Show file after | Hide comments
@@ -1,1330 +1,1337
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, cpu_count
13 13
14 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 15 from xml.dom import minidom
16 16
17 17 import schainpy
18 18 from schainpy.admin import Alarm, SchainWarning
19 19 from schainpy.model import *
20 20 from schainpy.utils import log
21 21
22 22 DTYPES = {
23 23 'Voltage': '.r',
24 24 'Spectra': '.pdata'
25 25 }
26 26
27 27
28 28 def MPProject(project, n=cpu_count()):
29 29 '''
30 30 Project wrapper to run schain in n processes
31 31 '''
32 32
33 33 rconf = project.getReadUnitObj()
34 34 op = rconf.getOperationObj('run')
35 35 dt1 = op.getParameterValue('startDate')
36 36 dt2 = op.getParameterValue('endDate')
37 37 tm1 = op.getParameterValue('startTime')
38 38 tm2 = op.getParameterValue('endTime')
39 39 days = (dt2 - dt1).days
40 40
41 41 for day in range(days + 1):
42 42 skip = 0
43 43 cursor = 0
44 44 processes = []
45 45 dt = dt1 + datetime.timedelta(day)
46 46 dt_str = dt.strftime('%Y/%m/%d')
47 47 reader = JRODataReader()
48 48 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 49 startDate=dt,
50 50 endDate=dt,
51 51 startTime=tm1,
52 52 endTime=tm2,
53 53 ext=DTYPES[rconf.datatype])
54 54 nFiles = len(files)
55 55 if nFiles == 0:
56 56 continue
57 57 skip = int(math.ceil(nFiles / n))
58 58 while nFiles > cursor * skip:
59 59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 60 skip=skip)
61 61 p = project.clone()
62 62 p.start()
63 63 processes.append(p)
64 64 cursor += 1
65 65
66 66 def beforeExit(exctype, value, trace):
67 67 for process in processes:
68 68 process.terminate()
69 69 process.join()
70 70 print traceback.print_tb(trace)
71 71
72 72 sys.excepthook = beforeExit
73 73
74 74 for process in processes:
75 75 process.join()
76 76 process.terminate()
77 77
78 78 time.sleep(3)
79 79
80 80
81 81 class ParameterConf():
82 82
83 83 id = None
84 84 name = None
85 85 value = None
86 86 format = None
87 87
88 88 __formated_value = None
89 89
90 90 ELEMENTNAME = 'Parameter'
91 91
92 92 def __init__(self):
93 93
94 94 self.format = 'str'
95 95
96 96 def getElementName(self):
97 97
98 98 return self.ELEMENTNAME
99 99
100 100 def getValue(self):
101 101
102 102 value = self.value
103 103 format = self.format
104 104
105 105 if self.__formated_value != None:
106 106
107 107 return self.__formated_value
108 108
109 109 if format == 'obj':
110 110 return value
111 111
112 112 if format == 'str':
113 113 self.__formated_value = str(value)
114 114 return self.__formated_value
115 115
116 116 if value == '':
117 117 raise ValueError, '%s: This parameter value is empty' % self.name
118 118
119 119 if format == 'list':
120 120 strList = value.split(',')
121 121
122 122 self.__formated_value = strList
123 123
124 124 return self.__formated_value
125 125
126 126 if format == 'intlist':
127 127 '''
128 128 Example:
129 129 value = (0,1,2)
130 130 '''
131 131
132 132 new_value = ast.literal_eval(value)
133 133
134 134 if type(new_value) not in (tuple, list):
135 135 new_value = [int(new_value)]
136 136
137 137 self.__formated_value = new_value
138 138
139 139 return self.__formated_value
140 140
141 141 if format == 'floatlist':
142 142 '''
143 143 Example:
144 144 value = (0.5, 1.4, 2.7)
145 145 '''
146 146
147 147 new_value = ast.literal_eval(value)
148 148
149 149 if type(new_value) not in (tuple, list):
150 150 new_value = [float(new_value)]
151 151
152 152 self.__formated_value = new_value
153 153
154 154 return self.__formated_value
155 155
156 156 if format == 'date':
157 157 strList = value.split('/')
158 158 intList = [int(x) for x in strList]
159 159 date = datetime.date(intList[0], intList[1], intList[2])
160 160
161 161 self.__formated_value = date
162 162
163 163 return self.__formated_value
164 164
165 165 if format == 'time':
166 166 strList = value.split(':')
167 167 intList = [int(x) for x in strList]
168 168 time = datetime.time(intList[0], intList[1], intList[2])
169 169
170 170 self.__formated_value = time
171 171
172 172 return self.__formated_value
173 173
174 174 if format == 'pairslist':
175 175 '''
176 176 Example:
177 177 value = (0,1),(1,2)
178 178 '''
179 179
180 180 new_value = ast.literal_eval(value)
181 181
182 182 if type(new_value) not in (tuple, list):
183 183 raise ValueError, '%s has to be a tuple or list of pairs' % value
184 184
185 185 if type(new_value[0]) not in (tuple, list):
186 186 if len(new_value) != 2:
187 187 raise ValueError, '%s has to be a tuple or list of pairs' % value
188 188 new_value = [new_value]
189 189
190 190 for thisPair in new_value:
191 191 if len(thisPair) != 2:
192 192 raise ValueError, '%s has to be a tuple or list of pairs' % value
193 193
194 194 self.__formated_value = new_value
195 195
196 196 return self.__formated_value
197 197
198 198 if format == 'multilist':
199 199 '''
200 200 Example:
201 201 value = (0,1,2),(3,4,5)
202 202 '''
203 203 multiList = ast.literal_eval(value)
204 204
205 205 if type(multiList[0]) == int:
206 206 multiList = ast.literal_eval('(' + value + ')')
207 207
208 208 self.__formated_value = multiList
209 209
210 210 return self.__formated_value
211 211
212 212 if format == 'bool':
213 213 value = int(value)
214 214
215 215 if format == 'int':
216 216 value = float(value)
217 217
218 218 format_func = eval(format)
219 219
220 220 self.__formated_value = format_func(value)
221 221
222 222 return self.__formated_value
223 223
224 224 def updateId(self, new_id):
225 225
226 226 self.id = str(new_id)
227 227
228 228 def setup(self, id, name, value, format='str'):
229 229 self.id = str(id)
230 230 self.name = name
231 231 if format == 'obj':
232 232 self.value = value
233 233 else:
234 234 self.value = str(value)
235 235 self.format = str.lower(format)
236 236
237 237 self.getValue()
238 238
239 239 return 1
240 240
241 241 def update(self, name, value, format='str'):
242 242
243 243 self.name = name
244 244 self.value = str(value)
245 245 self.format = format
246 246
247 247 def makeXml(self, opElement):
248 248 if self.name not in ('queue',):
249 249 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 250 parmElement.set('id', str(self.id))
251 251 parmElement.set('name', self.name)
252 252 parmElement.set('value', self.value)
253 253 parmElement.set('format', self.format)
254 254
255 255 def readXml(self, parmElement):
256 256
257 257 self.id = parmElement.get('id')
258 258 self.name = parmElement.get('name')
259 259 self.value = parmElement.get('value')
260 260 self.format = str.lower(parmElement.get('format'))
261 261
262 262 # Compatible with old signal chain version
263 263 if self.format == 'int' and self.name == 'idfigure':
264 264 self.name = 'id'
265 265
266 266 def printattr(self):
267 267
268 268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
269 269
270 270
271 271 class OperationConf():
272 272
273 273 id = None
274 274 name = None
275 275 priority = None
276 276 type = None
277 277
278 278 parmConfObjList = []
279 279
280 280 ELEMENTNAME = 'Operation'
281 281
282 282 def __init__(self):
283 283
284 284 self.id = '0'
285 285 self.name = None
286 286 self.priority = None
287 287 self.type = 'self'
288 288
289 289 def __getNewId(self):
290 290
291 291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292 292
293 293 def updateId(self, new_id):
294 294
295 295 self.id = str(new_id)
296 296
297 297 n = 1
298 298 for parmObj in self.parmConfObjList:
299 299
300 300 idParm = str(int(new_id) * 10 + n)
301 301 parmObj.updateId(idParm)
302 302
303 303 n += 1
304 304
305 305 def getElementName(self):
306 306
307 307 return self.ELEMENTNAME
308 308
309 309 def getParameterObjList(self):
310 310
311 311 return self.parmConfObjList
312 312
313 313 def getParameterObj(self, parameterName):
314 314
315 315 for parmConfObj in self.parmConfObjList:
316 316
317 317 if parmConfObj.name != parameterName:
318 318 continue
319 319
320 320 return parmConfObj
321 321
322 322 return None
323 323
324 324 def getParameterObjfromValue(self, parameterValue):
325 325
326 326 for parmConfObj in self.parmConfObjList:
327 327
328 328 if parmConfObj.getValue() != parameterValue:
329 329 continue
330 330
331 331 return parmConfObj.getValue()
332 332
333 333 return None
334 334
335 335 def getParameterValue(self, parameterName):
336 336
337 337 parameterObj = self.getParameterObj(parameterName)
338 338
339 339 # if not parameterObj:
340 340 # return None
341 341
342 342 value = parameterObj.getValue()
343 343
344 344 return value
345 345
346 346 def getKwargs(self):
347 347
348 348 kwargs = {}
349 349
350 350 for parmConfObj in self.parmConfObjList:
351 351 if self.name == 'run' and parmConfObj.name == 'datatype':
352 352 continue
353 353
354 354 kwargs[parmConfObj.name] = parmConfObj.getValue()
355 355
356 356 return kwargs
357 357
358 358 def setup(self, id, name, priority, type):
359 359
360 360 self.id = str(id)
361 361 self.name = name
362 362 self.type = type
363 363 self.priority = priority
364 364
365 365 self.parmConfObjList = []
366 366
367 367 def removeParameters(self):
368 368
369 369 for obj in self.parmConfObjList:
370 370 del obj
371 371
372 372 self.parmConfObjList = []
373 373
374 374 def addParameter(self, name, value, format='str'):
375 375
376 376 if value is None:
377 377 return None
378 378 id = self.__getNewId()
379 379
380 380 parmConfObj = ParameterConf()
381 381 if not parmConfObj.setup(id, name, value, format):
382 382 return None
383 383
384 384 self.parmConfObjList.append(parmConfObj)
385 385
386 386 return parmConfObj
387 387
388 388 def changeParameter(self, name, value, format='str'):
389 389
390 390 parmConfObj = self.getParameterObj(name)
391 391 parmConfObj.update(name, value, format)
392 392
393 393 return parmConfObj
394 394
395 395 def makeXml(self, procUnitElement):
396 396
397 397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 398 opElement.set('id', str(self.id))
399 399 opElement.set('name', self.name)
400 400 opElement.set('type', self.type)
401 401 opElement.set('priority', str(self.priority))
402 402
403 403 for parmConfObj in self.parmConfObjList:
404 404 parmConfObj.makeXml(opElement)
405 405
406 406 def readXml(self, opElement):
407 407
408 408 self.id = opElement.get('id')
409 409 self.name = opElement.get('name')
410 410 self.type = opElement.get('type')
411 411 self.priority = opElement.get('priority')
412 412
413 413 # Compatible with old signal chain version
414 414 # Use of 'run' method instead 'init'
415 415 if self.type == 'self' and self.name == 'init':
416 416 self.name = 'run'
417 417
418 418 self.parmConfObjList = []
419 419
420 420 parmElementList = opElement.iter(ParameterConf().getElementName())
421 421
422 422 for parmElement in parmElementList:
423 423 parmConfObj = ParameterConf()
424 424 parmConfObj.readXml(parmElement)
425 425
426 426 # Compatible with old signal chain version
427 427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 428 if self.type != 'self' and self.name == 'Plot':
429 429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 430 self.name = parmConfObj.value
431 431 continue
432 432
433 433 self.parmConfObjList.append(parmConfObj)
434 434
435 435 def printattr(self):
436 436
437 437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 438 self.id,
439 439 self.name,
440 440 self.type,
441 441 self.priority)
442 442
443 443 for parmConfObj in self.parmConfObjList:
444 444 parmConfObj.printattr()
445 445
446 446 def createObject(self, plotter_queue=None):
447 447
448 448 if self.type == 'self':
449 449 raise ValueError, 'This operation type cannot be created'
450 450
451 451 if self.type == 'plotter':
452 452 if not plotter_queue:
453 453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
454 454
455 455 opObj = Plotter(self.name, plotter_queue)
456 456
457 457 if self.type == 'external' or self.type == 'other':
458 458
459 459 className = eval(self.name)
460 460 kwargs = self.getKwargs()
461 461
462 462 opObj = className(**kwargs)
463 463
464 464 return opObj
465 465
466 466
467 467 class ProcUnitConf():
468 468
469 469 id = None
470 470 name = None
471 471 datatype = None
472 472 inputId = None
473 473 parentId = None
474 474
475 475 opConfObjList = []
476 476
477 477 procUnitObj = None
478 478 opObjList = []
479 479
480 480 ELEMENTNAME = 'ProcUnit'
481 481
482 482 def __init__(self):
483 483
484 484 self.id = None
485 485 self.datatype = None
486 486 self.name = None
487 487 self.inputId = None
488 488
489 489 self.opConfObjList = []
490 490
491 491 self.procUnitObj = None
492 492 self.opObjDict = {}
493 493
494 494 def __getPriority(self):
495 495
496 496 return len(self.opConfObjList) + 1
497 497
498 498 def __getNewId(self):
499 499
500 500 return int(self.id) * 10 + len(self.opConfObjList) + 1
501 501
502 502 def getElementName(self):
503 503
504 504 return self.ELEMENTNAME
505 505
506 506 def getId(self):
507 507
508 508 return self.id
509 509
510 510 def updateId(self, new_id, parentId=parentId):
511 511
512 512 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514 514
515 515 # If this proc unit has not inputs
516 516 if self.inputId == '0':
517 517 new_inputId = 0
518 518
519 519 n = 1
520 520 for opConfObj in self.opConfObjList:
521 521
522 522 idOp = str(int(new_id) * 10 + n)
523 523 opConfObj.updateId(idOp)
524 524
525 525 n += 1
526 526
527 527 self.parentId = str(parentId)
528 528 self.id = str(new_id)
529 529 self.inputId = str(new_inputId)
530 530
531 531 def getInputId(self):
532 532
533 533 return self.inputId
534 534
535 535 def getOperationObjList(self):
536 536
537 537 return self.opConfObjList
538 538
539 539 def getOperationObj(self, name=None):
540 540
541 541 for opConfObj in self.opConfObjList:
542 542
543 543 if opConfObj.name != name:
544 544 continue
545 545
546 546 return opConfObj
547 547
548 548 return None
549 549
550 550 def getOpObjfromParamValue(self, value=None):
551 551
552 552 for opConfObj in self.opConfObjList:
553 553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 554 continue
555 555 return opConfObj
556 556 return None
557 557
558 558 def getProcUnitObj(self):
559 559
560 560 return self.procUnitObj
561 561
562 562 def setup(self, id, name, datatype, inputId, parentId=None):
563 563
564 564 # Compatible with old signal chain version
565 565 if datatype == None and name == None:
566 566 raise ValueError, 'datatype or name should be defined'
567 567
568 568 if name == None:
569 569 if 'Proc' in datatype:
570 570 name = datatype
571 571 else:
572 572 name = '%sProc' % (datatype)
573 573
574 574 if datatype == None:
575 575 datatype = name.replace('Proc', '')
576 576
577 577 self.id = str(id)
578 578 self.name = name
579 579 self.datatype = datatype
580 580 self.inputId = inputId
581 581 self.parentId = parentId
582 582
583 583 self.opConfObjList = []
584 584
585 585 self.addOperation(name='run', optype='self')
586 586
587 587 def removeOperations(self):
588 588
589 589 for obj in self.opConfObjList:
590 590 del obj
591 591
592 592 self.opConfObjList = []
593 593 self.addOperation(name='run')
594 594
595 595 def addParameter(self, **kwargs):
596 596 '''
597 597 Add parameters to 'run' operation
598 598 '''
599 599 opObj = self.opConfObjList[0]
600 600
601 601 opObj.addParameter(**kwargs)
602 602
603 603 return opObj
604 604
605 605 def addOperation(self, name, optype='self'):
606 606
607 607 id = self.__getNewId()
608 608 priority = self.__getPriority()
609 609
610 610 opConfObj = OperationConf()
611 611 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 612
613 613 self.opConfObjList.append(opConfObj)
614 614
615 615 return opConfObj
616 616
617 617 def makeXml(self, projectElement):
618 618
619 619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 620 procUnitElement.set('id', str(self.id))
621 621 procUnitElement.set('name', self.name)
622 622 procUnitElement.set('datatype', self.datatype)
623 623 procUnitElement.set('inputId', str(self.inputId))
624 624
625 625 for opConfObj in self.opConfObjList:
626 626 opConfObj.makeXml(procUnitElement)
627 627
628 628 def readXml(self, upElement):
629 629
630 630 self.id = upElement.get('id')
631 631 self.name = upElement.get('name')
632 632 self.datatype = upElement.get('datatype')
633 633 self.inputId = upElement.get('inputId')
634 634
635 635 if self.ELEMENTNAME == 'ReadUnit':
636 636 self.datatype = self.datatype.replace('Reader', '')
637 637
638 638 if self.ELEMENTNAME == 'ProcUnit':
639 639 self.datatype = self.datatype.replace('Proc', '')
640 640
641 641 if self.inputId == 'None':
642 642 self.inputId = '0'
643 643
644 644 self.opConfObjList = []
645 645
646 646 opElementList = upElement.iter(OperationConf().getElementName())
647 647
648 648 for opElement in opElementList:
649 649 opConfObj = OperationConf()
650 650 opConfObj.readXml(opElement)
651 651 self.opConfObjList.append(opConfObj)
652 652
653 653 def printattr(self):
654 654
655 655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 656 self.id,
657 657 self.name,
658 658 self.datatype,
659 659 self.inputId)
660 660
661 661 for opConfObj in self.opConfObjList:
662 662 opConfObj.printattr()
663 663
664 664 def getKwargs(self):
665 665
666 666 opObj = self.opConfObjList[0]
667 667 kwargs = opObj.getKwargs()
668 668
669 669 return kwargs
670 670
671 671 def createObjects(self, plotter_queue=None):
672 672
673 673 className = eval(self.name)
674 674 kwargs = self.getKwargs()
675 675 procUnitObj = className(**kwargs)
676 676
677 677 for opConfObj in self.opConfObjList:
678 678
679 679 if opConfObj.type == 'self' and self.name == 'run':
680 680 continue
681 681 elif opConfObj.type == 'self':
682 682 procUnitObj.addOperationKwargs(
683 683 opConfObj.id, **opConfObj.getKwargs())
684 684 continue
685 685
686 686 opObj = opConfObj.createObject(plotter_queue)
687 687
688 688 self.opObjDict[opConfObj.id] = opObj
689 689
690 690 procUnitObj.addOperation(opObj, opConfObj.id)
691 691
692 692 self.procUnitObj = procUnitObj
693 693
694 694 return procUnitObj
695 695
696 696 def run(self):
697 697
698 698 is_ok = False
699 699
700 700 for opConfObj in self.opConfObjList:
701 701
702 702 kwargs = {}
703 703 for parmConfObj in opConfObj.getParameterObjList():
704 704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 705 continue
706 706
707 707 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 708
709 709 sts = self.procUnitObj.call(opType=opConfObj.type,
710 710 opName=opConfObj.name,
711 711 opId=opConfObj.id)
712 712
713 713 is_ok = is_ok or sts
714 714
715 715 return is_ok
716 716
717 717 def close(self):
718 718
719 719 for opConfObj in self.opConfObjList:
720 720 if opConfObj.type == 'self':
721 721 continue
722 722
723 723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 724 opObj.close()
725 725
726 726 self.procUnitObj.close()
727 727
728 728 return
729 729
730 730
731 731 class ReadUnitConf(ProcUnitConf):
732 732
733 733 path = None
734 734 startDate = None
735 735 endDate = None
736 736 startTime = None
737 737 endTime = None
738 738
739 739 ELEMENTNAME = 'ReadUnit'
740 740
741 741 def __init__(self):
742 742
743 743 self.id = None
744 744 self.datatype = None
745 745 self.name = None
746 746 self.inputId = None
747 747
748 748 self.parentId = None
749 749
750 750 self.opConfObjList = []
751 751 self.opObjList = []
752 752
753 753 def getElementName(self):
754 754
755 755 return self.ELEMENTNAME
756 756
757 757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 758 startTime='', endTime='', parentId=None, server=None, **kwargs):
759 759
760 760 # Compatible with old signal chain version
761 761 if datatype == None and name == None:
762 762 raise ValueError, 'datatype or name should be defined'
763 763 if name == None:
764 764 if 'Reader' in datatype:
765 765 name = datatype
766 766 datatype = name.replace('Reader','')
767 767 else:
768 768 name = '{}Reader'.format(datatype)
769 769 if datatype == None:
770 770 if 'Reader' in name:
771 771 datatype = name.replace('Reader','')
772 772 else:
773 773 datatype = name
774 774 name = '{}Reader'.format(name)
775 775
776 776 self.id = id
777 777 self.name = name
778 778 self.datatype = datatype
779 779 if path != '':
780 780 self.path = os.path.abspath(path)
781 781 self.startDate = startDate
782 782 self.endDate = endDate
783 783 self.startTime = startTime
784 784 self.endTime = endTime
785 785 self.inputId = '0'
786 786 self.parentId = parentId
787 787 self.server = server
788 788 self.addRunOperation(**kwargs)
789 789
790 790 def update(self, **kwargs):
791 791
792 792 if 'datatype' in kwargs:
793 793 datatype = kwargs.pop('datatype')
794 794 if 'Reader' in datatype:
795 795 self.name = datatype
796 796 else:
797 797 self.name = '%sReader' % (datatype)
798 798 self.datatype = self.name.replace('Reader', '')
799 799
800 800 attrs = ('path', 'startDate', 'endDate',
801 801 'startTime', 'endTime', 'parentId')
802 802
803 803 for attr in attrs:
804 804 if attr in kwargs:
805 805 setattr(self, attr, kwargs.pop(attr))
806 806
807 807 self.inputId = '0'
808 808 self.updateRunOperation(**kwargs)
809 809
810 810 def removeOperations(self):
811 811
812 812 for obj in self.opConfObjList:
813 813 del obj
814 814
815 815 self.opConfObjList = []
816 816
817 817 def addRunOperation(self, **kwargs):
818 818
819 819 opObj = self.addOperation(name='run', optype='self')
820 820
821 821 if self.server is None:
822 822 opObj.addParameter(
823 823 name='datatype', value=self.datatype, format='str')
824 824 opObj.addParameter(name='path', value=self.path, format='str')
825 825 opObj.addParameter(
826 826 name='startDate', value=self.startDate, format='date')
827 827 opObj.addParameter(
828 828 name='endDate', value=self.endDate, format='date')
829 829 opObj.addParameter(
830 830 name='startTime', value=self.startTime, format='time')
831 831 opObj.addParameter(
832 832 name='endTime', value=self.endTime, format='time')
833 833
834 834 for key, value in kwargs.items():
835 835 opObj.addParameter(name=key, value=value,
836 836 format=type(value).__name__)
837 837 else:
838 838 opObj.addParameter(name='server', value=self.server, format='str')
839 839
840 840 return opObj
841 841
842 842 def updateRunOperation(self, **kwargs):
843 843
844 844 opObj = self.getOperationObj(name='run')
845 845 opObj.removeParameters()
846 846
847 847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 848 opObj.addParameter(name='path', value=self.path, format='str')
849 849 opObj.addParameter(
850 850 name='startDate', value=self.startDate, format='date')
851 851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 852 opObj.addParameter(
853 853 name='startTime', value=self.startTime, format='time')
854 854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855 855
856 856 for key, value in kwargs.items():
857 857 opObj.addParameter(name=key, value=value,
858 858 format=type(value).__name__)
859 859
860 860 return opObj
861 861
862 862 def readXml(self, upElement):
863 863
864 864 self.id = upElement.get('id')
865 865 self.name = upElement.get('name')
866 866 self.datatype = upElement.get('datatype')
867 867 self.inputId = upElement.get('inputId')
868 868
869 869 if self.ELEMENTNAME == 'ReadUnit':
870 870 self.datatype = self.datatype.replace('Reader', '')
871 871
872 872 if self.inputId == 'None':
873 873 self.inputId = '0'
874 874
875 875 self.opConfObjList = []
876 876
877 877 opElementList = upElement.iter(OperationConf().getElementName())
878 878
879 879 for opElement in opElementList:
880 880 opConfObj = OperationConf()
881 881 opConfObj.readXml(opElement)
882 882 self.opConfObjList.append(opConfObj)
883 883
884 884 if opConfObj.name == 'run':
885 885 self.path = opConfObj.getParameterValue('path')
886 886 self.startDate = opConfObj.getParameterValue('startDate')
887 887 self.endDate = opConfObj.getParameterValue('endDate')
888 888 self.startTime = opConfObj.getParameterValue('startTime')
889 889 self.endTime = opConfObj.getParameterValue('endTime')
890 890
891 891
892 892 class Project(Process):
893 893
894 894 id = None
895 895 # name = None
896 896 description = None
897 897 filename = None
898 898
899 899 procUnitConfObjDict = None
900 900
901 901 ELEMENTNAME = 'Project'
902 902
903 903 plotterQueue = None
904 904
905 905 def __init__(self, plotter_queue=None):
906 906
907 907 Process.__init__(self)
908 908 self.id = None
909 909 self.description = None
910 910 self.email = None
911 self.alarm = [0]
911 self.alarm = None
912 912 self.plotterQueue = plotter_queue
913 913 self.procUnitConfObjDict = {}
914 914
915 915 def __getNewId(self):
916 916
917 917 idList = self.procUnitConfObjDict.keys()
918 918
919 919 id = int(self.id) * 10
920 920
921 921 while True:
922 922 id += 1
923 923
924 924 if str(id) in idList:
925 925 continue
926 926
927 927 break
928 928
929 929 return str(id)
930 930
931 931 def getElementName(self):
932 932
933 933 return self.ELEMENTNAME
934 934
935 935 def getId(self):
936 936
937 937 return self.id
938 938
939 939 def updateId(self, new_id):
940 940
941 941 self.id = str(new_id)
942 942
943 943 keyList = self.procUnitConfObjDict.keys()
944 944 keyList.sort()
945 945
946 946 n = 1
947 947 newProcUnitConfObjDict = {}
948 948
949 949 for procKey in keyList:
950 950
951 951 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 952 idProcUnit = str(int(self.id) * 10 + n)
953 953 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 954 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
955 955 n += 1
956 956
957 957 self.procUnitConfObjDict = newProcUnitConfObjDict
958 958
959 def setup(self, id, name='', description='', email=None, alarm=[0]):
959 def setup(self, id, name='', description='', email=None, alarm=[3]):
960 960
961 961 print
962 962 print '*' * 60
963 963 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
964 964 print '*' * 60
965 965 print
966 966 self.id = str(id)
967 967 self.description = description
968 968 self.email = email
969 969 self.alarm = alarm
970 970
971 971 def update(self, **kwargs):
972 972
973 973 for key, value in kwargs:
974 974 setattr(self, key, value)
975 975
976 976 def clone(self):
977 977
978 978 p = Project()
979 979 p.procUnitConfObjDict = self.procUnitConfObjDict
980 980 return p
981 981
982 982 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
983 983
984 984 if id is None:
985 985 idReadUnit = self.__getNewId()
986 986 else:
987 987 idReadUnit = str(id)
988 988
989 989 readUnitConfObj = ReadUnitConf()
990 990 readUnitConfObj.setup(idReadUnit, name, datatype,
991 991 parentId=self.id, **kwargs)
992 992
993 993 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
994 994
995 995 return readUnitConfObj
996 996
997 997 def addProcUnit(self, inputId='0', datatype=None, name=None):
998 998
999 999 idProcUnit = self.__getNewId()
1000 1000
1001 1001 procUnitConfObj = ProcUnitConf()
1002 1002 procUnitConfObj.setup(idProcUnit, name, datatype,
1003 1003 inputId, parentId=self.id)
1004 1004
1005 1005 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 1006
1007 1007 return procUnitConfObj
1008 1008
1009 1009 def removeProcUnit(self, id):
1010 1010
1011 1011 if id in self.procUnitConfObjDict.keys():
1012 1012 self.procUnitConfObjDict.pop(id)
1013 1013
1014 1014 def getReadUnitId(self):
1015 1015
1016 1016 readUnitConfObj = self.getReadUnitObj()
1017 1017
1018 1018 return readUnitConfObj.id
1019 1019
1020 1020 def getReadUnitObj(self):
1021 1021
1022 1022 for obj in self.procUnitConfObjDict.values():
1023 1023 if obj.getElementName() == 'ReadUnit':
1024 1024 return obj
1025 1025
1026 1026 return None
1027 1027
1028 1028 def getProcUnitObj(self, id=None, name=None):
1029 1029
1030 1030 if id != None:
1031 1031 return self.procUnitConfObjDict[id]
1032 1032
1033 1033 if name != None:
1034 1034 return self.getProcUnitObjByName(name)
1035 1035
1036 1036 return None
1037 1037
1038 1038 def getProcUnitObjByName(self, name):
1039 1039
1040 1040 for obj in self.procUnitConfObjDict.values():
1041 1041 if obj.name == name:
1042 1042 return obj
1043 1043
1044 1044 return None
1045 1045
1046 1046 def procUnitItems(self):
1047 1047
1048 1048 return self.procUnitConfObjDict.items()
1049 1049
1050 1050 def makeXml(self):
1051 1051
1052 1052 projectElement = Element('Project')
1053 1053 projectElement.set('id', str(self.id))
1054 1054 projectElement.set('name', self.name)
1055 1055 projectElement.set('description', self.description)
1056 1056
1057 1057 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 1058 procUnitConfObj.makeXml(projectElement)
1059 1059
1060 1060 self.projectElement = projectElement
1061 1061
1062 1062 def writeXml(self, filename=None):
1063 1063
1064 1064 if filename == None:
1065 1065 if self.filename:
1066 1066 filename = self.filename
1067 1067 else:
1068 1068 filename = 'schain.xml'
1069 1069
1070 1070 if not filename:
1071 1071 print 'filename has not been defined. Use setFilename(filename) for do it.'
1072 1072 return 0
1073 1073
1074 1074 abs_file = os.path.abspath(filename)
1075 1075
1076 1076 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 1077 print 'No write permission on %s' % os.path.dirname(abs_file)
1078 1078 return 0
1079 1079
1080 1080 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 1081 print 'File %s already exists and it could not be overwriten' % abs_file
1082 1082 return 0
1083 1083
1084 1084 self.makeXml()
1085 1085
1086 1086 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 1087
1088 1088 self.filename = abs_file
1089 1089
1090 1090 return 1
1091 1091
1092 1092 def readXml(self, filename=None):
1093 1093
1094 1094 if not filename:
1095 1095 print 'filename is not defined'
1096 1096 return 0
1097 1097
1098 1098 abs_file = os.path.abspath(filename)
1099 1099
1100 1100 if not os.path.isfile(abs_file):
1101 1101 print '%s file does not exist' % abs_file
1102 1102 return 0
1103 1103
1104 1104 self.projectElement = None
1105 1105 self.procUnitConfObjDict = {}
1106 1106
1107 1107 try:
1108 1108 self.projectElement = ElementTree().parse(abs_file)
1109 1109 except:
1110 1110 print 'Error reading %s, verify file format' % filename
1111 1111 return 0
1112 1112
1113 1113 self.project = self.projectElement.tag
1114 1114
1115 1115 self.id = self.projectElement.get('id')
1116 1116 self.name = self.projectElement.get('name')
1117 1117 self.description = self.projectElement.get('description')
1118 1118
1119 1119 readUnitElementList = self.projectElement.iter(
1120 1120 ReadUnitConf().getElementName())
1121 1121
1122 1122 for readUnitElement in readUnitElementList:
1123 1123 readUnitConfObj = ReadUnitConf()
1124 1124 readUnitConfObj.readXml(readUnitElement)
1125 1125
1126 1126 if readUnitConfObj.parentId == None:
1127 1127 readUnitConfObj.parentId = self.id
1128 1128
1129 1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130 1130
1131 1131 procUnitElementList = self.projectElement.iter(
1132 1132 ProcUnitConf().getElementName())
1133 1133
1134 1134 for procUnitElement in procUnitElementList:
1135 1135 procUnitConfObj = ProcUnitConf()
1136 1136 procUnitConfObj.readXml(procUnitElement)
1137 1137
1138 1138 if procUnitConfObj.parentId == None:
1139 1139 procUnitConfObj.parentId = self.id
1140 1140
1141 1141 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1142 1142
1143 1143 self.filename = abs_file
1144 1144
1145 1145 return 1
1146 1146
1147 1147 def printattr(self):
1148 1148
1149 1149 print 'Project[%s]: name = %s, description = %s' % (self.id,
1150 1150 self.name,
1151 1151 self.description)
1152 1152
1153 1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 1154 procUnitConfObj.printattr()
1155 1155
1156 1156 def createObjects(self):
1157 1157
1158 1158 for procUnitConfObj in self.procUnitConfObjDict.values():
1159 1159 procUnitConfObj.createObjects(self.plotterQueue)
1160 1160
1161 1161 def __connect(self, objIN, thisObj):
1162 1162
1163 1163 thisObj.setInput(objIN.getOutputObj())
1164 1164
1165 1165 def connectObjects(self):
1166 1166
1167 1167 for thisPUConfObj in self.procUnitConfObjDict.values():
1168 1168
1169 1169 inputId = thisPUConfObj.getInputId()
1170 1170
1171 1171 if int(inputId) == 0:
1172 1172 continue
1173 1173
1174 1174 # Get input object
1175 1175 puConfINObj = self.procUnitConfObjDict[inputId]
1176 1176 puObjIN = puConfINObj.getProcUnitObj()
1177 1177
1178 1178 # Get current object
1179 1179 thisPUObj = thisPUConfObj.getProcUnitObj()
1180 1180
1181 1181 self.__connect(puObjIN, thisPUObj)
1182 1182
1183 def __handleError(self, procUnitConfObj, modes=None):
1183 def __handleError(self, procUnitConfObj, modes=None, stdout=True):
1184 1184
1185 1185 import socket
1186 1186
1187 1187 if modes is None:
1188 1188 modes = self.alarm
1189 1189
1190 1190 err = traceback.format_exception(sys.exc_info()[0],
1191 1191 sys.exc_info()[1],
1192 1192 sys.exc_info()[2])
1193 1193
1194 1194 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1195 1195
1196 1196 message = ''.join(err)
1197 1197
1198 sys.stderr.write(message)
1198 if stdout:
1199 sys.stderr.write(message)
1199 1200
1200 1201 subject = 'SChain v%s: Error running %s\n' % (
1201 1202 schainpy.__version__, procUnitConfObj.name)
1202 1203
1203 1204 subtitle = '%s: %s\n' % (
1204 1205 procUnitConfObj.getElementName(), procUnitConfObj.name)
1205 1206 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1206 1207 socket.gethostname())
1207 1208 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1208 1209 subtitle += 'Configuration file: %s\n' % self.filename
1209 1210 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1210 1211
1211 1212 readUnitConfObj = self.getReadUnitObj()
1212 1213 if readUnitConfObj:
1213 1214 subtitle += '\nInput parameters:\n'
1214 1215 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1215 1216 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1216 1217 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1217 1218 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1218 1219 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1219 1220 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1220 1221
1221 1222 a = Alarm(
1222 1223 modes=modes,
1223 1224 email=self.email,
1224 1225 message=message,
1225 1226 subject=subject,
1226 1227 subtitle=subtitle,
1227 1228 filename=self.filename
1228 1229 )
1229 1230
1230 a.start()
1231 return a
1231 1232
1232 1233 def isPaused(self):
1233 1234 return 0
1234 1235
1235 1236 def isStopped(self):
1236 1237 return 0
1237 1238
1238 1239 def runController(self):
1239 1240 '''
1240 1241 returns 0 when this process has been stopped, 1 otherwise
1241 1242 '''
1242 1243
1243 1244 if self.isPaused():
1244 1245 print 'Process suspended'
1245 1246
1246 1247 while True:
1247 1248 time.sleep(0.1)
1248 1249
1249 1250 if not self.isPaused():
1250 1251 break
1251 1252
1252 1253 if self.isStopped():
1253 1254 break
1254 1255
1255 1256 print 'Process reinitialized'
1256 1257
1257 1258 if self.isStopped():
1258 1259 print 'Process stopped'
1259 1260 return 0
1260 1261
1261 1262 return 1
1262 1263
1263 1264 def setFilename(self, filename):
1264 1265
1265 1266 self.filename = filename
1266 1267
1267 1268 def setPlotterQueue(self, plotter_queue):
1268 1269
1269 1270 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270 1271
1271 1272 def getPlotterQueue(self):
1272 1273
1273 1274 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1274 1275
1275 1276 def useExternalPlotter(self):
1276 1277
1277 1278 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1278 1279
1279 1280 def run(self):
1280 1281
1281 1282 log.success('Starting {}'.format(self.name))
1282 1283 self.start_time = time.time()
1283 1284 self.createObjects()
1284 1285 self.connectObjects()
1285 1286
1286 1287 keyList = self.procUnitConfObjDict.keys()
1287 1288 keyList.sort()
1288 1289
1290 err = None
1291
1289 1292 while(True):
1290 1293
1291 1294 is_ok = False
1292 1295
1293 1296 for procKey in keyList:
1294 1297
1295 1298 procUnitConfObj = self.procUnitConfObjDict[procKey]
1296 1299
1297 1300 try:
1298 1301 sts = procUnitConfObj.run()
1299 1302 is_ok = is_ok or sts
1300 1303 except SchainWarning:
1301 self.__handleError(procUnitConfObj, modes=[2, 3])
1304 err = self.__handleError(procUnitConfObj, modes=[2, 3], stdout=False)
1302 1305 except KeyboardInterrupt:
1303 1306 is_ok = False
1304 1307 break
1305 1308 except ValueError, e:
1306 1309 time.sleep(0.5)
1307 self.__handleError(procUnitConfObj)
1310 err = self.__handleError(procUnitConfObj)
1308 1311 is_ok = False
1309 1312 break
1310 1313 except:
1311 1314 time.sleep(0.5)
1312 self.__handleError(procUnitConfObj)
1315 err = self.__handleError(procUnitConfObj)
1313 1316 is_ok = False
1314 1317 break
1315 1318
1316 1319 # If every process unit finished so end process
1317 1320 if not(is_ok):
1318 1321 break
1319 1322
1320 1323 if not self.runController():
1321 1324 break
1322 1325
1323 1326 # Closing every process
1324 1327 for procKey in keyList:
1325 1328 procUnitConfObj = self.procUnitConfObjDict[procKey]
1326 1329 procUnitConfObj.close()
1327 1330
1331 if err is not None:
1332 err.start()
1333 # err.join()
1334
1328 1335 log.success('{} finished (time: {}s)'.format(
1329 1336 self.name,
1330 1337 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,971 +1,978
1 1
2 2 import os
3 3 import time
4 4 import glob
5 5 import datetime
6 6 from multiprocessing import Process
7 7
8 8 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 14
15 15 from schainpy.model.proc.jroproc_base import Operation
16 16 from schainpy.utils import log
17 17
18 18 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
19 19 blu_values = matplotlib.pyplot.get_cmap(
20 20 'seismic_r', 20)(numpy.arange(20))[10:15]
21 21 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
22 22 'jro', numpy.vstack((blu_values, jet_values)))
23 23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 24
25 25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
26 26
27 27
28 28 def figpause(interval):
29 29 backend = plt.rcParams['backend']
30 30 if backend in matplotlib.rcsetup.interactive_bk:
31 31 figManager = matplotlib._pylab_helpers.Gcf.get_active()
32 32 if figManager is not None:
33 33 canvas = figManager.canvas
34 34 if canvas.figure.stale:
35 35 canvas.draw()
36 canvas.start_event_loop(interval)
36 try:
37 canvas.start_event_loop(interval)
38 except:
39 pass
37 40 return
38 41
39 42 def popup(message):
40 fig = plt.figure(figsize=(12, 8), facecolor='r')
41 fig.text(0.5, 0.5, message, ha='center', va='center', size='20', weight='heavy', color='w')
43 '''
44 '''
45
46 fig = plt.figure(figsize=(12, 8), facecolor='r')
47 text = '\n'.join([s.strip() for s in message.split(':')])
48 fig.text(0.01, 0.5, text, ha='left', va='center', size='20', weight='heavy', color='w')
42 49 fig.show()
43 50 figpause(1000)
44 51
45 52
46 53 class PlotData(Operation, Process):
47 54 '''
48 55 Base class for Schain plotting operations
49 56 '''
50 57
51 58 CODE = 'Figure'
52 59 colormap = 'jro'
53 60 bgcolor = 'white'
54 61 CONFLATE = False
55 62 __missing = 1E30
56 63
57 64 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
58 65 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
59 66 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
60 67 'showprofile', 'decimation']
61 68
62 69 def __init__(self, **kwargs):
63 70
64 71 Operation.__init__(self, plot=True, **kwargs)
65 72 Process.__init__(self)
66 73
67 74 self.kwargs['code'] = self.CODE
68 75 self.mp = False
69 76 self.data = None
70 77 self.isConfig = False
71 78 self.figures = []
72 79 self.axes = []
73 80 self.cb_axes = []
74 81 self.localtime = kwargs.pop('localtime', True)
75 82 self.show = kwargs.get('show', True)
76 83 self.save = kwargs.get('save', False)
77 84 self.colormap = kwargs.get('colormap', self.colormap)
78 85 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
79 86 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
80 87 self.colormaps = kwargs.get('colormaps', None)
81 88 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
82 89 self.showprofile = kwargs.get('showprofile', False)
83 90 self.title = kwargs.get('wintitle', self.CODE.upper())
84 91 self.cb_label = kwargs.get('cb_label', None)
85 92 self.cb_labels = kwargs.get('cb_labels', None)
86 93 self.xaxis = kwargs.get('xaxis', 'frequency')
87 94 self.zmin = kwargs.get('zmin', None)
88 95 self.zmax = kwargs.get('zmax', None)
89 96 self.zlimits = kwargs.get('zlimits', None)
90 97 self.xmin = kwargs.get('xmin', None)
91 98 self.xmax = kwargs.get('xmax', None)
92 99 self.xrange = kwargs.get('xrange', 24)
93 100 self.ymin = kwargs.get('ymin', None)
94 101 self.ymax = kwargs.get('ymax', None)
95 102 self.xlabel = kwargs.get('xlabel', None)
96 103 self.decimation = kwargs.get('decimation', None)
97 104 self.showSNR = kwargs.get('showSNR', False)
98 105 self.oneFigure = kwargs.get('oneFigure', True)
99 106 self.width = kwargs.get('width', None)
100 107 self.height = kwargs.get('height', None)
101 108 self.colorbar = kwargs.get('colorbar', True)
102 109 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
103 110 self.titles = kwargs.get('titles', [])
104 111 self.polar = False
105 112
106 113 def __fmtTime(self, x, pos):
107 114 '''
108 115 '''
109 116
110 117 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
111 118
112 119 def __setup(self):
113 120 '''
114 121 Common setup for all figures, here figures and axes are created
115 122 '''
116 123
117 124 if self.CODE not in self.data:
118 125 raise ValueError(log.error('Missing data for {}'.format(self.CODE),
119 126 self.name))
120 127
121 128 self.setup()
122 129
123 130 self.time_label = 'LT' if self.localtime else 'UTC'
124 131 if self.data.localtime:
125 132 self.getDateTime = datetime.datetime.fromtimestamp
126 133 else:
127 134 self.getDateTime = datetime.datetime.utcfromtimestamp
128 135
129 136 if self.width is None:
130 137 self.width = 8
131 138
132 139 self.figures = []
133 140 self.axes = []
134 141 self.cb_axes = []
135 142 self.pf_axes = []
136 143 self.cmaps = []
137 144
138 145 size = '15%' if self.ncols == 1 else '30%'
139 146 pad = '4%' if self.ncols == 1 else '8%'
140 147
141 148 if self.oneFigure:
142 149 if self.height is None:
143 150 self.height = 1.4 * self.nrows + 1
144 151 fig = plt.figure(figsize=(self.width, self.height),
145 152 edgecolor='k',
146 153 facecolor='w')
147 154 self.figures.append(fig)
148 155 for n in range(self.nplots):
149 156 ax = fig.add_subplot(self.nrows, self.ncols,
150 157 n + 1, polar=self.polar)
151 158 ax.tick_params(labelsize=8)
152 159 ax.firsttime = True
153 160 ax.index = 0
154 161 ax.press = None
155 162 self.axes.append(ax)
156 163 if self.showprofile:
157 164 cax = self.__add_axes(ax, size=size, pad=pad)
158 165 cax.tick_params(labelsize=8)
159 166 self.pf_axes.append(cax)
160 167 else:
161 168 if self.height is None:
162 169 self.height = 3
163 170 for n in range(self.nplots):
164 171 fig = plt.figure(figsize=(self.width, self.height),
165 172 edgecolor='k',
166 173 facecolor='w')
167 174 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
168 175 ax.tick_params(labelsize=8)
169 176 ax.firsttime = True
170 177 ax.index = 0
171 178 ax.press = None
172 179 self.figures.append(fig)
173 180 self.axes.append(ax)
174 181 if self.showprofile:
175 182 cax = self.__add_axes(ax, size=size, pad=pad)
176 183 cax.tick_params(labelsize=8)
177 184 self.pf_axes.append(cax)
178 185
179 186 for n in range(self.nrows):
180 187 if self.colormaps is not None:
181 188 cmap = plt.get_cmap(self.colormaps[n])
182 189 else:
183 190 cmap = plt.get_cmap(self.colormap)
184 191 cmap.set_bad(self.bgcolor, 1.)
185 192 self.cmaps.append(cmap)
186 193
187 194 for fig in self.figures:
188 195 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
189 196 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
190 197 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
191 198 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
192 199 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
193 200 if self.show:
194 201 fig.show()
195 202
196 203 def OnKeyPress(self, event):
197 204 '''
198 205 Event for pressing keys (up, down) change colormap
199 206 '''
200 207 ax = event.inaxes
201 208 if ax in self.axes:
202 209 if event.key == 'down':
203 210 ax.index += 1
204 211 elif event.key == 'up':
205 212 ax.index -= 1
206 213 if ax.index < 0:
207 214 ax.index = len(CMAPS) - 1
208 215 elif ax.index == len(CMAPS):
209 216 ax.index = 0
210 217 cmap = CMAPS[ax.index]
211 218 ax.cbar.set_cmap(cmap)
212 219 ax.cbar.draw_all()
213 220 ax.plt.set_cmap(cmap)
214 221 ax.cbar.patch.figure.canvas.draw()
215 222 self.colormap = cmap.name
216 223
217 224 def OnBtnScroll(self, event):
218 225 '''
219 226 Event for scrolling, scale figure
220 227 '''
221 228 cb_ax = event.inaxes
222 229 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
223 230 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
224 231 pt = ax.cbar.ax.bbox.get_points()[:, 1]
225 232 nrm = ax.cbar.norm
226 233 vmin, vmax, p0, p1, pS = (
227 234 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
228 235 scale = 2 if event.step == 1 else 0.5
229 236 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
230 237 ax.cbar.norm.vmin = point - scale * (point - vmin)
231 238 ax.cbar.norm.vmax = point - scale * (point - vmax)
232 239 ax.plt.set_norm(ax.cbar.norm)
233 240 ax.cbar.draw_all()
234 241 ax.cbar.patch.figure.canvas.draw()
235 242
236 243 def onBtnPress(self, event):
237 244 '''
238 245 Event for mouse button press
239 246 '''
240 247 cb_ax = event.inaxes
241 248 if cb_ax is None:
242 249 return
243 250
244 251 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
245 252 cb_ax.press = event.x, event.y
246 253 else:
247 254 cb_ax.press = None
248 255
249 256 def onMotion(self, event):
250 257 '''
251 258 Event for move inside colorbar
252 259 '''
253 260 cb_ax = event.inaxes
254 261 if cb_ax is None:
255 262 return
256 263 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
257 264 return
258 265 if cb_ax.press is None:
259 266 return
260 267
261 268 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
262 269 xprev, yprev = cb_ax.press
263 270 dx = event.x - xprev
264 271 dy = event.y - yprev
265 272 cb_ax.press = event.x, event.y
266 273 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
267 274 perc = 0.03
268 275
269 276 if event.button == 1:
270 277 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
271 278 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
272 279 elif event.button == 3:
273 280 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
274 281 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
275 282
276 283 ax.cbar.draw_all()
277 284 ax.plt.set_norm(ax.cbar.norm)
278 285 ax.cbar.patch.figure.canvas.draw()
279 286
280 287 def onBtnRelease(self, event):
281 288 '''
282 289 Event for mouse button release
283 290 '''
284 291 cb_ax = event.inaxes
285 292 if cb_ax is not None:
286 293 cb_ax.press = None
287 294
288 295 def __add_axes(self, ax, size='30%', pad='8%'):
289 296 '''
290 297 Add new axes to the given figure
291 298 '''
292 299 divider = make_axes_locatable(ax)
293 300 nax = divider.new_horizontal(size=size, pad=pad)
294 301 ax.figure.add_axes(nax)
295 302 return nax
296 303
297 304 self.setup()
298 305
299 306 def setup(self):
300 307 '''
301 308 This method should be implemented in the child class, the following
302 309 attributes should be set:
303 310
304 311 self.nrows: number of rows
305 312 self.ncols: number of cols
306 313 self.nplots: number of plots (channels or pairs)
307 314 self.ylabel: label for Y axes
308 315 self.titles: list of axes title
309 316
310 317 '''
311 318 raise(NotImplementedError, 'Implement this method in child class')
312 319
313 320 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
314 321 '''
315 322 Create a masked array for missing data
316 323 '''
317 324 if x_buffer.shape[0] < 2:
318 325 return x_buffer, y_buffer, z_buffer
319 326
320 327 deltas = x_buffer[1:] - x_buffer[0:-1]
321 328 x_median = numpy.median(deltas)
322 329
323 330 index = numpy.where(deltas > 5 * x_median)
324 331
325 332 if len(index[0]) != 0:
326 333 z_buffer[::, index[0], ::] = self.__missing
327 334 z_buffer = numpy.ma.masked_inside(z_buffer,
328 335 0.99 * self.__missing,
329 336 1.01 * self.__missing)
330 337
331 338 return x_buffer, y_buffer, z_buffer
332 339
333 340 def decimate(self):
334 341
335 342 # dx = int(len(self.x)/self.__MAXNUMX) + 1
336 343 dy = int(len(self.y) / self.decimation) + 1
337 344
338 345 # x = self.x[::dx]
339 346 x = self.x
340 347 y = self.y[::dy]
341 348 z = self.z[::, ::, ::dy]
342 349
343 350 return x, y, z
344 351
345 352 def format(self):
346 353 '''
347 354 Set min and max values, labels, ticks and titles
348 355 '''
349 356
350 357 if self.xmin is None:
351 358 xmin = self.min_time
352 359 else:
353 360 if self.xaxis is 'time':
354 361 dt = self.getDateTime(self.min_time)
355 362 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
356 363 datetime.datetime(1970, 1, 1)).total_seconds()
357 364 if self.data.localtime:
358 365 xmin += time.timezone
359 366 else:
360 367 xmin = self.xmin
361 368
362 369 if self.xmax is None:
363 370 xmax = xmin + self.xrange * 60 * 60
364 371 else:
365 372 if self.xaxis is 'time':
366 373 dt = self.getDateTime(self.max_time)
367 374 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
368 375 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
369 376 if self.data.localtime:
370 377 xmax += time.timezone
371 378 else:
372 379 xmax = self.xmax
373 380
374 381 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
375 382 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
376 383
377 384 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])
378 385 i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]
379 386 ystep = Y[i] / 5
380 387
381 388 for n, ax in enumerate(self.axes):
382 389 if ax.firsttime:
383 390 ax.set_facecolor(self.bgcolor)
384 391 ax.yaxis.set_major_locator(MultipleLocator(ystep))
385 392 if self.xaxis is 'time':
386 393 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
387 394 ax.xaxis.set_major_locator(LinearLocator(9))
388 395 if self.xlabel is not None:
389 396 ax.set_xlabel(self.xlabel)
390 397 ax.set_ylabel(self.ylabel)
391 398 ax.firsttime = False
392 399 if self.showprofile:
393 400 self.pf_axes[n].set_ylim(ymin, ymax)
394 401 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
395 402 self.pf_axes[n].set_xlabel('dB')
396 403 self.pf_axes[n].grid(b=True, axis='x')
397 404 [tick.set_visible(False)
398 405 for tick in self.pf_axes[n].get_yticklabels()]
399 406 if self.colorbar:
400 407 ax.cbar = plt.colorbar(
401 408 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
402 409 ax.cbar.ax.tick_params(labelsize=8)
403 410 ax.cbar.ax.press = None
404 411 if self.cb_label:
405 412 ax.cbar.set_label(self.cb_label, size=8)
406 413 elif self.cb_labels:
407 414 ax.cbar.set_label(self.cb_labels[n], size=8)
408 415 else:
409 416 ax.cbar = None
410 417
411 418 if not self.polar:
412 419 ax.set_xlim(xmin, xmax)
413 420 ax.set_ylim(ymin, ymax)
414 421 ax.set_title('{} - {} {}'.format(
415 422 self.titles[n],
416 423 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
417 424 self.time_label),
418 425 size=8)
419 426 else:
420 427 ax.set_title('{}'.format(self.titles[n]), size=8)
421 428 ax.set_ylim(0, 90)
422 429 ax.set_yticks(numpy.arange(0, 90, 20))
423 430 ax.yaxis.labelpad = 40
424 431
425 432 def __plot(self):
426 433 '''
427 434 '''
428 435 log.success('Plotting', self.name)
429 436
430 437 try:
431 438 self.plot()
432 439 self.format()
433 440 except:
434 441 log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
435 442
436 443 for n, fig in enumerate(self.figures):
437 444 if self.nrows == 0 or self.nplots == 0:
438 445 log.warning('No data', self.name)
439 446 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
440 447 fig.canvas.manager.set_window_title(self.CODE)
441 448 continue
442 449
443 450 fig.tight_layout()
444 451 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
445 452 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
446 453 fig.canvas.draw()
447 454
448 455 if self.save and self.data.ended:
449 456 channels = range(self.nrows)
450 457 if self.oneFigure:
451 458 label = ''
452 459 else:
453 460 label = '_{}'.format(channels[n])
454 461 figname = os.path.join(
455 462 self.save,
456 463 '{}{}_{}.png'.format(
457 464 self.CODE,
458 465 label,
459 466 self.getDateTime(self.saveTime).strftime(
460 467 '%Y%m%d_%H%M%S'),
461 468 )
462 469 )
463 470 log.log('Saving figure: {}'.format(figname), self.name)
464 471 fig.savefig(figname)
465 472
466 473 def plot(self):
467 474 '''
468 475 '''
469 476 raise(NotImplementedError, 'Implement this method in child class')
470 477
471 478 def run(self):
472 479
473 480 log.success('Starting', self.name)
474 481
475 482 context = zmq.Context()
476 483 receiver = context.socket(zmq.SUB)
477 484 receiver.setsockopt(zmq.SUBSCRIBE, '')
478 485 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
479 486
480 487 if 'server' in self.kwargs['parent']:
481 488 receiver.connect(
482 489 'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
483 490 else:
484 491 receiver.connect("ipc:///tmp/zmq.plots")
485 492
486 493 while True:
487 494 try:
488 495 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
489 496 if self.data.localtime and self.localtime:
490 497 self.times = self.data.times
491 498 elif self.data.localtime and not self.localtime:
492 499 self.times = self.data.times + time.timezone
493 500 elif not self.data.localtime and self.localtime:
494 501 self.times = self.data.times - time.timezone
495 502 else:
496 503 self.times = self.data.times
497 504
498 505 self.min_time = self.times[0]
499 506 self.max_time = self.times[-1]
500 507
501 508 if self.isConfig is False:
502 509 self.__setup()
503 510 self.isConfig = True
504 511
505 512 self.__plot()
506 513
507 514 except zmq.Again as e:
508 515 log.log('Waiting for data...')
509 516 if self.data:
510 517 figpause(self.data.throttle)
511 518 else:
512 519 time.sleep(2)
513 520
514 521 def close(self):
515 522 if self.data:
516 523 self.__plot()
517 524
518 525
519 526 class PlotSpectraData(PlotData):
520 527 '''
521 528 Plot for Spectra data
522 529 '''
523 530
524 531 CODE = 'spc'
525 532 colormap = 'jro'
526 533
527 534 def setup(self):
528 535 self.nplots = len(self.data.channels)
529 536 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
530 537 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
531 538 self.width = 3.4 * self.ncols
532 539 self.height = 3 * self.nrows
533 540 self.cb_label = 'dB'
534 541 if self.showprofile:
535 542 self.width += 0.8 * self.ncols
536 543
537 544 self.ylabel = 'Range [km]'
538 545
539 546 def plot(self):
540 547 if self.xaxis == "frequency":
541 548 x = self.data.xrange[0]
542 549 self.xlabel = "Frequency (kHz)"
543 550 elif self.xaxis == "time":
544 551 x = self.data.xrange[1]
545 552 self.xlabel = "Time (ms)"
546 553 else:
547 554 x = self.data.xrange[2]
548 555 self.xlabel = "Velocity (m/s)"
549 556
550 557 if self.CODE == 'spc_mean':
551 558 x = self.data.xrange[2]
552 559 self.xlabel = "Velocity (m/s)"
553 560
554 561 self.titles = []
555 562
556 563 y = self.data.heights
557 564 self.y = y
558 565 z = self.data['spc']
559 566
560 567 for n, ax in enumerate(self.axes):
561 568 noise = self.data['noise'][n][-1]
562 569 if self.CODE == 'spc_mean':
563 570 mean = self.data['mean'][n][-1]
564 571 if ax.firsttime:
565 572 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
566 573 self.xmin = self.xmin if self.xmin else -self.xmax
567 574 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
568 575 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
569 576 ax.plt = ax.pcolormesh(x, y, z[n].T,
570 577 vmin=self.zmin,
571 578 vmax=self.zmax,
572 579 cmap=plt.get_cmap(self.colormap)
573 580 )
574 581
575 582 if self.showprofile:
576 583 ax.plt_profile = self.pf_axes[n].plot(
577 584 self.data['rti'][n][-1], y)[0]
578 585 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
579 586 color="k", linestyle="dashed", lw=1)[0]
580 587 if self.CODE == 'spc_mean':
581 588 ax.plt_mean = ax.plot(mean, y, color='k')[0]
582 589 else:
583 590 ax.plt.set_array(z[n].T.ravel())
584 591 if self.showprofile:
585 592 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
586 593 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
587 594 if self.CODE == 'spc_mean':
588 595 ax.plt_mean.set_data(mean, y)
589 596
590 597 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
591 598 self.saveTime = self.max_time
592 599
593 600
594 601 class PlotCrossSpectraData(PlotData):
595 602
596 603 CODE = 'cspc'
597 604 zmin_coh = None
598 605 zmax_coh = None
599 606 zmin_phase = None
600 607 zmax_phase = None
601 608
602 609 def setup(self):
603 610
604 611 self.ncols = 4
605 612 self.nrows = len(self.data.pairs)
606 613 self.nplots = self.nrows * 4
607 614 self.width = 3.4 * self.ncols
608 615 self.height = 3 * self.nrows
609 616 self.ylabel = 'Range [km]'
610 617 self.showprofile = False
611 618
612 619 def plot(self):
613 620
614 621 if self.xaxis == "frequency":
615 622 x = self.data.xrange[0]
616 623 self.xlabel = "Frequency (kHz)"
617 624 elif self.xaxis == "time":
618 625 x = self.data.xrange[1]
619 626 self.xlabel = "Time (ms)"
620 627 else:
621 628 x = self.data.xrange[2]
622 629 self.xlabel = "Velocity (m/s)"
623 630
624 631 self.titles = []
625 632
626 633 y = self.data.heights
627 634 self.y = y
628 635 spc = self.data['spc']
629 636 cspc = self.data['cspc']
630 637
631 638 for n in range(self.nrows):
632 639 noise = self.data['noise'][n][-1]
633 640 pair = self.data.pairs[n]
634 641 ax = self.axes[4 * n]
635 642 ax3 = self.axes[4 * n + 3]
636 643 if ax.firsttime:
637 644 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
638 645 self.xmin = self.xmin if self.xmin else -self.xmax
639 646 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
640 647 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
641 648 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
642 649 vmin=self.zmin,
643 650 vmax=self.zmax,
644 651 cmap=plt.get_cmap(self.colormap)
645 652 )
646 653 else:
647 654 ax.plt.set_array(spc[pair[0]].T.ravel())
648 655 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
649 656
650 657 ax = self.axes[4 * n + 1]
651 658 if ax.firsttime:
652 659 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
653 660 vmin=self.zmin,
654 661 vmax=self.zmax,
655 662 cmap=plt.get_cmap(self.colormap)
656 663 )
657 664 else:
658 665 ax.plt.set_array(spc[pair[1]].T.ravel())
659 666 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
660 667
661 668 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
662 669 coh = numpy.abs(out)
663 670 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
664 671
665 672 ax = self.axes[4 * n + 2]
666 673 if ax.firsttime:
667 674 ax.plt = ax.pcolormesh(x, y, coh.T,
668 675 vmin=0,
669 676 vmax=1,
670 677 cmap=plt.get_cmap(self.colormap_coh)
671 678 )
672 679 else:
673 680 ax.plt.set_array(coh.T.ravel())
674 681 self.titles.append(
675 682 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
676 683
677 684 ax = self.axes[4 * n + 3]
678 685 if ax.firsttime:
679 686 ax.plt = ax.pcolormesh(x, y, phase.T,
680 687 vmin=-180,
681 688 vmax=180,
682 689 cmap=plt.get_cmap(self.colormap_phase)
683 690 )
684 691 else:
685 692 ax.plt.set_array(phase.T.ravel())
686 693 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
687 694
688 695 self.saveTime = self.max_time
689 696
690 697
691 698 class PlotSpectraMeanData(PlotSpectraData):
692 699 '''
693 700 Plot for Spectra and Mean
694 701 '''
695 702 CODE = 'spc_mean'
696 703 colormap = 'jro'
697 704
698 705
699 706 class PlotRTIData(PlotData):
700 707 '''
701 708 Plot for RTI data
702 709 '''
703 710
704 711 CODE = 'rti'
705 712 colormap = 'jro'
706 713
707 714 def setup(self):
708 715 self.xaxis = 'time'
709 716 self.ncols = 1
710 717 self.nrows = len(self.data.channels)
711 718 self.nplots = len(self.data.channels)
712 719 self.ylabel = 'Range [km]'
713 720 self.cb_label = 'dB'
714 721 self.titles = ['{} Channel {}'.format(
715 722 self.CODE.upper(), x) for x in range(self.nrows)]
716 723
717 724 def plot(self):
718 725 self.x = self.times
719 726 self.y = self.data.heights
720 727 self.z = self.data[self.CODE]
721 728 self.z = numpy.ma.masked_invalid(self.z)
722 729
723 730 if self.decimation is None:
724 731 x, y, z = self.fill_gaps(self.x, self.y, self.z)
725 732 else:
726 733 x, y, z = self.fill_gaps(*self.decimate())
727 734
728 735 for n, ax in enumerate(self.axes):
729 736 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
730 737 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
731 738 if ax.firsttime:
732 739 ax.plt = ax.pcolormesh(x, y, z[n].T,
733 740 vmin=self.zmin,
734 741 vmax=self.zmax,
735 742 cmap=plt.get_cmap(self.colormap)
736 743 )
737 744 if self.showprofile:
738 745 ax.plot_profile = self.pf_axes[n].plot(
739 746 self.data['rti'][n][-1], self.y)[0]
740 747 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
741 748 color="k", linestyle="dashed", lw=1)[0]
742 749 else:
743 750 ax.collections.remove(ax.collections[0])
744 751 ax.plt = ax.pcolormesh(x, y, z[n].T,
745 752 vmin=self.zmin,
746 753 vmax=self.zmax,
747 754 cmap=plt.get_cmap(self.colormap)
748 755 )
749 756 if self.showprofile:
750 757 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
751 758 ax.plot_noise.set_data(numpy.repeat(
752 759 self.data['noise'][n][-1], len(self.y)), self.y)
753 760
754 761 self.saveTime = self.min_time
755 762
756 763
757 764 class PlotCOHData(PlotRTIData):
758 765 '''
759 766 Plot for Coherence data
760 767 '''
761 768
762 769 CODE = 'coh'
763 770
764 771 def setup(self):
765 772 self.xaxis = 'time'
766 773 self.ncols = 1
767 774 self.nrows = len(self.data.pairs)
768 775 self.nplots = len(self.data.pairs)
769 776 self.ylabel = 'Range [km]'
770 777 if self.CODE == 'coh':
771 778 self.cb_label = ''
772 779 self.titles = [
773 780 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
774 781 else:
775 782 self.cb_label = 'Degrees'
776 783 self.titles = [
777 784 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
778 785
779 786
780 787 class PlotPHASEData(PlotCOHData):
781 788 '''
782 789 Plot for Phase map data
783 790 '''
784 791
785 792 CODE = 'phase'
786 793 colormap = 'seismic'
787 794
788 795
789 796 class PlotNoiseData(PlotData):
790 797 '''
791 798 Plot for noise
792 799 '''
793 800
794 801 CODE = 'noise'
795 802
796 803 def setup(self):
797 804 self.xaxis = 'time'
798 805 self.ncols = 1
799 806 self.nrows = 1
800 807 self.nplots = 1
801 808 self.ylabel = 'Intensity [dB]'
802 809 self.titles = ['Noise']
803 810 self.colorbar = False
804 811
805 812 def plot(self):
806 813
807 814 x = self.times
808 815 xmin = self.min_time
809 816 xmax = xmin + self.xrange * 60 * 60
810 817 Y = self.data[self.CODE]
811 818
812 819 if self.axes[0].firsttime:
813 820 for ch in self.data.channels:
814 821 y = Y[ch]
815 822 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
816 823 plt.legend()
817 824 else:
818 825 for ch in self.data.channels:
819 826 y = Y[ch]
820 827 self.axes[0].lines[ch].set_data(x, y)
821 828
822 829 self.ymin = numpy.nanmin(Y) - 5
823 830 self.ymax = numpy.nanmax(Y) + 5
824 831 self.saveTime = self.min_time
825 832
826 833
827 834 class PlotSNRData(PlotRTIData):
828 835 '''
829 836 Plot for SNR Data
830 837 '''
831 838
832 839 CODE = 'snr'
833 840 colormap = 'jet'
834 841
835 842
836 843 class PlotDOPData(PlotRTIData):
837 844 '''
838 845 Plot for DOPPLER Data
839 846 '''
840 847
841 848 CODE = 'dop'
842 849 colormap = 'jet'
843 850
844 851
845 852 class PlotSkyMapData(PlotData):
846 853 '''
847 854 Plot for meteors detection data
848 855 '''
849 856
850 857 CODE = 'param'
851 858
852 859 def setup(self):
853 860
854 861 self.ncols = 1
855 862 self.nrows = 1
856 863 self.width = 7.2
857 864 self.height = 7.2
858 865 self.nplots = 1
859 866 self.xlabel = 'Zonal Zenith Angle (deg)'
860 867 self.ylabel = 'Meridional Zenith Angle (deg)'
861 868 self.polar = True
862 869 self.ymin = -180
863 870 self.ymax = 180
864 871 self.colorbar = False
865 872
866 873 def plot(self):
867 874
868 875 arrayParameters = numpy.concatenate(self.data['param'])
869 876 error = arrayParameters[:, -1]
870 877 indValid = numpy.where(error == 0)[0]
871 878 finalMeteor = arrayParameters[indValid, :]
872 879 finalAzimuth = finalMeteor[:, 3]
873 880 finalZenith = finalMeteor[:, 4]
874 881
875 882 x = finalAzimuth * numpy.pi / 180
876 883 y = finalZenith
877 884
878 885 ax = self.axes[0]
879 886
880 887 if ax.firsttime:
881 888 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
882 889 else:
883 890 ax.plot.set_data(x, y)
884 891
885 892 dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
886 893 dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
887 894 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
888 895 dt2,
889 896 len(x))
890 897 self.titles[0] = title
891 898 self.saveTime = self.max_time
892 899
893 900
894 901 class PlotParamData(PlotRTIData):
895 902 '''
896 903 Plot for data_param object
897 904 '''
898 905
899 906 CODE = 'param'
900 907 colormap = 'seismic'
901 908
902 909 def setup(self):
903 910 self.xaxis = 'time'
904 911 self.ncols = 1
905 912 self.nrows = self.data.shape(self.CODE)[0]
906 913 self.nplots = self.nrows
907 914 if self.showSNR:
908 915 self.nrows += 1
909 916 self.nplots += 1
910 917
911 918 self.ylabel = 'Height [km]'
912 919 if not self.titles:
913 920 self.titles = self.data.parameters \
914 921 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
915 922 if self.showSNR:
916 923 self.titles.append('SNR')
917 924
918 925 def plot(self):
919 926 self.data.normalize_heights()
920 927 self.x = self.times
921 928 self.y = self.data.heights
922 929 if self.showSNR:
923 930 self.z = numpy.concatenate(
924 931 (self.data[self.CODE], self.data['snr'])
925 932 )
926 933 else:
927 934 self.z = self.data[self.CODE]
928 935
929 936 self.z = numpy.ma.masked_invalid(self.z)
930 937
931 938 if self.decimation is None:
932 939 x, y, z = self.fill_gaps(self.x, self.y, self.z)
933 940 else:
934 941 x, y, z = self.fill_gaps(*self.decimate())
935 942
936 943 for n, ax in enumerate(self.axes):
937 944
938 945 self.zmax = self.zmax if self.zmax is not None else numpy.max(
939 946 self.z[n])
940 947 self.zmin = self.zmin if self.zmin is not None else numpy.min(
941 948 self.z[n])
942 949
943 950 if ax.firsttime:
944 951 if self.zlimits is not None:
945 952 self.zmin, self.zmax = self.zlimits[n]
946 953
947 954 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
948 955 vmin=self.zmin,
949 956 vmax=self.zmax,
950 957 cmap=self.cmaps[n]
951 958 )
952 959 else:
953 960 if self.zlimits is not None:
954 961 self.zmin, self.zmax = self.zlimits[n]
955 962 ax.collections.remove(ax.collections[0])
956 963 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
957 964 vmin=self.zmin,
958 965 vmax=self.zmax,
959 966 cmap=self.cmaps[n]
960 967 )
961 968
962 969 self.saveTime = self.min_time
963 970
964 971
965 972 class PlotOutputData(PlotParamData):
966 973 '''
967 974 Plot data_output object
968 975 '''
969 976
970 977 CODE = 'output'
971 978 colormap = 'seismic'
Show file before | Show file after | Hide comments
@@ -1,1824 +1,1824
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 import schainpy.admin
23 24 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 25 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 26 from schainpy.utils import log
26 27
27 28 LOCALTIME = True
28 29
29 30
30 31 def isNumber(cad):
31 32 """
32 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
33 34
34 35 Excepciones:
35 36 Si un determinado string no puede ser convertido a numero
36 37 Input:
37 38 str, string al cual se le analiza para determinar si convertible a un numero o no
38 39
39 40 Return:
40 41 True : si el string es uno numerico
41 42 False : no es un string numerico
42 43 """
43 44 try:
44 45 float(cad)
45 46 return True
46 47 except:
47 48 return False
48 49
49 50
50 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
51 52 """
52 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
53 54
54 55 Inputs:
55 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
56 57
57 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
58 59 segundos contados desde 01/01/1970.
59 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
60 61 segundos contados desde 01/01/1970.
61 62
62 63 Return:
63 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
64 65 fecha especificado, de lo contrario retorna False.
65 66
66 67 Excepciones:
67 68 Si el archivo no existe o no puede ser abierto
68 69 Si la cabecera no puede ser leida.
69 70
70 71 """
71 72 basicHeaderObj = BasicHeader(LOCALTIME)
72 73
73 74 try:
74 75 fp = open(filename, 'rb')
75 76 except IOError:
76 77 print "The file %s can't be opened" % (filename)
77 78 return 0
78 79
79 80 sts = basicHeaderObj.read(fp)
80 81 fp.close()
81 82
82 83 if not(sts):
83 84 print "Skipping the file %s because it has not a valid header" % (filename)
84 85 return 0
85 86
86 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
87 88 return 0
88 89
89 90 return 1
90 91
91 92
92 93 def isTimeInRange(thisTime, startTime, endTime):
93 94 if endTime >= startTime:
94 95 if (thisTime < startTime) or (thisTime > endTime):
95 96 return 0
96 97 return 1
97 98 else:
98 99 if (thisTime < startTime) and (thisTime > endTime):
99 100 return 0
100 101 return 1
101 102
102 103
103 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
104 105 """
105 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
106 107
107 108 Inputs:
108 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
109 110
110 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
111 112
112 113 endDate : fecha final del rango seleccionado en formato datetime.date
113 114
114 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
115 116
116 117 endTime : tiempo final del rango seleccionado en formato datetime.time
117 118
118 119 Return:
119 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
120 121 fecha especificado, de lo contrario retorna False.
121 122
122 123 Excepciones:
123 124 Si el archivo no existe o no puede ser abierto
124 125 Si la cabecera no puede ser leida.
125 126
126 127 """
127 128
128 129 try:
129 130 fp = open(filename, 'rb')
130 131 except IOError:
131 132 print "The file %s can't be opened" % (filename)
132 133 return None
133 134
134 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
135 136 systemHeaderObj = SystemHeader()
136 137 radarControllerHeaderObj = RadarControllerHeader()
137 138 processingHeaderObj = ProcessingHeader()
138 139
139 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
140 141
141 142 sts = firstBasicHeaderObj.read(fp)
142 143
143 144 if not(sts):
144 145 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
145 146 return None
146 147
147 148 if not systemHeaderObj.read(fp):
148 149 return None
149 150
150 151 if not radarControllerHeaderObj.read(fp):
151 152 return None
152 153
153 154 if not processingHeaderObj.read(fp):
154 155 return None
155 156
156 157 filesize = os.path.getsize(filename)
157 158
158 159 offset = processingHeaderObj.blockSize + 24 # header size
159 160
160 161 if filesize <= offset:
161 162 print "[Reading] %s: This file has not enough data" % filename
162 163 return None
163 164
164 165 fp.seek(-offset, 2)
165 166
166 167 sts = lastBasicHeaderObj.read(fp)
167 168
168 169 fp.close()
169 170
170 171 thisDatetime = lastBasicHeaderObj.datatime
171 172 thisTime_last_block = thisDatetime.time()
172 173
173 174 thisDatetime = firstBasicHeaderObj.datatime
174 175 thisDate = thisDatetime.date()
175 176 thisTime_first_block = thisDatetime.time()
176 177
177 178 # General case
178 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
179 180 #-----------o----------------------------o-----------
180 181 # startTime endTime
181 182
182 183 if endTime >= startTime:
183 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
184 185 return None
185 186
186 187 return thisDatetime
187 188
188 189 # If endTime < startTime then endTime belongs to the next day
189 190
190 191 #<<<<<<<<<<<o o>>>>>>>>>>>
191 192 #-----------o----------------------------o-----------
192 193 # endTime startTime
193 194
194 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
195 196 return None
196 197
197 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
198 199 return None
199 200
200 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
201 202 return None
202 203
203 204 return thisDatetime
204 205
205 206
206 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
207 208 """
208 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
209 210
210 211 Inputs:
211 212 folder : nombre completo del directorio.
212 213 Su formato deberia ser "/path_root/?YYYYDDD"
213 214
214 215 siendo:
215 216 YYYY : Anio (ejemplo 2015)
216 217 DDD : Dia del anio (ejemplo 305)
217 218
218 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
219 220
220 221 endDate : fecha final del rango seleccionado en formato datetime.date
221 222
222 223 Return:
223 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
224 225 fecha especificado, de lo contrario retorna False.
225 226 Excepciones:
226 227 Si el directorio no tiene el formato adecuado
227 228 """
228 229
229 230 basename = os.path.basename(folder)
230 231
231 232 if not isRadarFolder(basename):
232 233 print "The folder %s has not the rigth format" % folder
233 234 return 0
234 235
235 236 if startDate and endDate:
236 237 thisDate = getDateFromRadarFolder(basename)
237 238
238 239 if thisDate < startDate:
239 240 return 0
240 241
241 242 if thisDate > endDate:
242 243 return 0
243 244
244 245 return 1
245 246
246 247
247 248 def isFileInDateRange(filename, startDate=None, endDate=None):
248 249 """
249 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
250 251
251 252 Inputs:
252 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
253 254
254 255 Su formato deberia ser "?YYYYDDDsss"
255 256
256 257 siendo:
257 258 YYYY : Anio (ejemplo 2015)
258 259 DDD : Dia del anio (ejemplo 305)
259 260 sss : set
260 261
261 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
262 263
263 264 endDate : fecha final del rango seleccionado en formato datetime.date
264 265
265 266 Return:
266 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
267 268 fecha especificado, de lo contrario retorna False.
268 269 Excepciones:
269 270 Si el archivo no tiene el formato adecuado
270 271 """
271 272
272 273 basename = os.path.basename(filename)
273 274
274 275 if not isRadarFile(basename):
275 276 print "The filename %s has not the rigth format" % filename
276 277 return 0
277 278
278 279 if startDate and endDate:
279 280 thisDate = getDateFromRadarFile(basename)
280 281
281 282 if thisDate < startDate:
282 283 return 0
283 284
284 285 if thisDate > endDate:
285 286 return 0
286 287
287 288 return 1
288 289
289 290
290 291 def getFileFromSet(path, ext, set):
291 292 validFilelist = []
292 293 fileList = os.listdir(path)
293 294
294 295 # 0 1234 567 89A BCDE
295 296 # H YYYY DDD SSS .ext
296 297
297 298 for thisFile in fileList:
298 299 try:
299 300 year = int(thisFile[1:5])
300 301 doy = int(thisFile[5:8])
301 302 except:
302 303 continue
303 304
304 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 306 continue
306 307
307 308 validFilelist.append(thisFile)
308 309
309 310 myfile = fnmatch.filter(
310 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
311 312
312 313 if len(myfile) != 0:
313 314 return myfile[0]
314 315 else:
315 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
316 317 print 'the filename %s does not exist' % filename
317 318 print '...going to the last file: '
318 319
319 320 if validFilelist:
320 321 validFilelist = sorted(validFilelist, key=str.lower)
321 322 return validFilelist[-1]
322 323
323 324 return None
324 325
325 326
326 327 def getlastFileFromPath(path, ext):
327 328 """
328 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
329 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
330 331
331 332 Input:
332 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
333 334 ext : extension de los files contenidos en una carpeta
334 335
335 336 Return:
336 337 El ultimo file de una determinada carpeta, no se considera el path.
337 338 """
338 339 validFilelist = []
339 340 fileList = os.listdir(path)
340 341
341 342 # 0 1234 567 89A BCDE
342 343 # H YYYY DDD SSS .ext
343 344
344 345 for thisFile in fileList:
345 346
346 347 year = thisFile[1:5]
347 348 if not isNumber(year):
348 349 continue
349 350
350 351 doy = thisFile[5:8]
351 352 if not isNumber(doy):
352 353 continue
353 354
354 355 year = int(year)
355 356 doy = int(doy)
356 357
357 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
358 359 continue
359 360
360 361 validFilelist.append(thisFile)
361 362
362 363 if validFilelist:
363 364 validFilelist = sorted(validFilelist, key=str.lower)
364 365 return validFilelist[-1]
365 366
366 367 return None
367 368
368 369
369 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
370 371 """
371 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
372 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
373 374 el path exacto de un determinado file.
374 375
375 376 Example :
376 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
377 378
378 379 Entonces la funcion prueba con las siguientes combinaciones
379 380 .../.../y2009307367.ext
380 381 .../.../Y2009307367.ext
381 382 .../.../x2009307/y2009307367.ext
382 383 .../.../x2009307/Y2009307367.ext
383 384 .../.../X2009307/y2009307367.ext
384 385 .../.../X2009307/Y2009307367.ext
385 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
386 387
387 388 Return:
388 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
389 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
390 391 para el filename
391 392 """
392 393 fullfilename = None
393 394 find_flag = False
394 395 filename = None
395 396
396 397 prefixDirList = [None, 'd', 'D']
397 398 if ext.lower() == ".r": # voltage
398 399 prefixFileList = ['d', 'D']
399 400 elif ext.lower() == ".pdata": # spectra
400 401 prefixFileList = ['p', 'P']
401 402 else:
402 403 return None, filename
403 404
404 405 # barrido por las combinaciones posibles
405 406 for prefixDir in prefixDirList:
406 407 thispath = path
407 408 if prefixDir != None:
408 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
409 410 if foldercounter == 0:
410 411 thispath = os.path.join(path, "%s%04d%03d" %
411 412 (prefixDir, year, doy))
412 413 else:
413 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
414 415 prefixDir, year, doy, foldercounter))
415 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
416 417 # formo el nombre del file xYYYYDDDSSS.ext
417 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
418 419 fullfilename = os.path.join(
419 420 thispath, filename) # formo el path completo
420 421
421 422 if os.path.exists(fullfilename): # verifico que exista
422 423 find_flag = True
423 424 break
424 425 if find_flag:
425 426 break
426 427
427 428 if not(find_flag):
428 429 return None, filename
429 430
430 431 return fullfilename, filename
431 432
432 433
433 434 def isRadarFolder(folder):
434 435 try:
435 436 year = int(folder[1:5])
436 437 doy = int(folder[5:8])
437 438 except:
438 439 return 0
439 440
440 441 return 1
441 442
442 443
443 444 def isRadarFile(file):
444 445 try:
445 446 year = int(file[1:5])
446 447 doy = int(file[5:8])
447 448 set = int(file[8:11])
448 449 except:
449 450 return 0
450 451
451 452 return 1
452 453
453 454
454 455 def getDateFromRadarFile(file):
455 456 try:
456 457 year = int(file[1:5])
457 458 doy = int(file[5:8])
458 459 set = int(file[8:11])
459 460 except:
460 461 return None
461 462
462 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
463 464 return thisDate
464 465
465 466
466 467 def getDateFromRadarFolder(folder):
467 468 try:
468 469 year = int(folder[1:5])
469 470 doy = int(folder[5:8])
470 471 except:
471 472 return None
472 473
473 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
474 475 return thisDate
475 476
476 477
477 478 class JRODataIO:
478 479
479 480 c = 3E8
480 481
481 482 isConfig = False
482 483
483 484 basicHeaderObj = None
484 485
485 486 systemHeaderObj = None
486 487
487 488 radarControllerHeaderObj = None
488 489
489 490 processingHeaderObj = None
490 491
491 492 dtype = None
492 493
493 494 pathList = []
494 495
495 496 filenameList = []
496 497
497 498 filename = None
498 499
499 500 ext = None
500 501
501 502 flagIsNewFile = 1
502 503
503 504 flagDiscontinuousBlock = 0
504 505
505 506 flagIsNewBlock = 0
506 507
507 508 fp = None
508 509
509 510 firstHeaderSize = 0
510 511
511 512 basicHeaderSize = 24
512 513
513 514 versionFile = 1103
514 515
515 516 fileSize = None
516 517
517 518 # ippSeconds = None
518 519
519 520 fileSizeByHeader = None
520 521
521 522 fileIndex = None
522 523
523 524 profileIndex = None
524 525
525 526 blockIndex = None
526 527
527 528 nTotalBlocks = None
528 529
529 530 maxTimeStep = 30
530 531
531 532 lastUTTime = None
532 533
533 534 datablock = None
534 535
535 536 dataOut = None
536 537
537 538 blocksize = None
538 539
539 540 getByBlock = False
540 541
541 542 def __init__(self):
542 543
543 544 raise NotImplementedError
544 545
545 546 def run(self):
546 547
547 548 raise NotImplementedError
548 549
549 550 def getDtypeWidth(self):
550 551
551 552 dtype_index = get_dtype_index(self.dtype)
552 553 dtype_width = get_dtype_width(dtype_index)
553 554
554 555 return dtype_width
555 556
556 557 def getAllowedArgs(self):
557 558 if hasattr(self, '__attrs__'):
558 559 return self.__attrs__
559 560 else:
560 561 return inspect.getargspec(self.run).args
561 562
562 563
563 564 class JRODataReader(JRODataIO):
564 565
565 566 online = 0
566 567
567 568 realtime = 0
568 569
569 570 nReadBlocks = 0
570 571
571 572 delay = 10 # number of seconds waiting a new file
572 573
573 574 nTries = 3 # quantity tries
574 575
575 576 nFiles = 3 # number of files for searching
576 577
577 578 path = None
578 579
579 580 foldercounter = 0
580 581
581 582 flagNoMoreFiles = 0
582 583
583 584 datetimeList = []
584 585
585 586 __isFirstTimeOnline = 1
586 587
587 588 __printInfo = True
588 589
589 590 profileIndex = None
590 591
591 592 nTxs = 1
592 593
593 594 txIndex = None
594 595
595 596 # Added--------------------
596 597
597 598 selBlocksize = None
598 599
599 600 selBlocktime = None
600 601
601 602 def __init__(self):
602 603 """
603 604 This class is used to find data files
604 605
605 606 Example:
606 607 reader = JRODataReader()
607 608 fileList = reader.findDataFiles()
608 609
609 610 """
610 611 pass
611 612
612 613 def createObjByDefault(self):
613 614 """
614 615
615 616 """
616 617 raise NotImplementedError
617 618
618 619 def getBlockDimension(self):
619 620
620 621 raise NotImplementedError
621 622
622 623 def searchFilesOffLine(self,
623 624 path,
624 625 startDate=None,
625 626 endDate=None,
626 627 startTime=datetime.time(0, 0, 0),
627 628 endTime=datetime.time(23, 59, 59),
628 629 set=None,
629 630 expLabel='',
630 631 ext='.r',
631 632 cursor=None,
632 633 skip=None,
633 634 walk=True):
634 635
635 636 self.filenameList = []
636 637 self.datetimeList = []
637 638
638 639 pathList = []
639 640
640 641 dateList, pathList = self.findDatafiles(
641 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
642 643
643 644 if dateList == []:
644 645 return [], []
645 646
646 647 if len(dateList) > 1:
647 648 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
648 649 else:
649 650 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
650 651
651 652 filenameList = []
652 653 datetimeList = []
653 654
654 655 for thisPath in pathList:
655 656
656 657 fileList = glob.glob1(thisPath, "*%s" % ext)
657 658 fileList.sort()
658 659
659 660 for file in fileList:
660 661
661 662 filename = os.path.join(thisPath, file)
662 663
663 664 if not isFileInDateRange(filename, startDate, endDate):
664 665 continue
665 666
666 667 thisDatetime = isFileInTimeRange(
667 668 filename, startDate, endDate, startTime, endTime)
668 669
669 670 if not(thisDatetime):
670 671 continue
671 672
672 673 filenameList.append(filename)
673 674 datetimeList.append(thisDatetime)
674 675
675 676 if cursor is not None and skip is not None:
676 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
677 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
678 679
679 680 if not(filenameList):
680 681 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
681 682 return [], []
682 683
683 684 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
684 685
685 686 # for i in range(len(filenameList)):
686 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
687 688
688 689 self.filenameList = filenameList
689 690 self.datetimeList = datetimeList
690 691
691 692 return pathList, filenameList
692 693
693 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
694 695 """
695 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
696 697 devuelve el archivo encontrado ademas de otros datos.
697 698
698 699 Input:
699 700 path : carpeta donde estan contenidos los files que contiene data
700 701
701 702 expLabel : Nombre del subexperimento (subfolder)
702 703
703 704 ext : extension de los files
704 705
705 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
706 707
707 708 Return:
708 709 directory : eL directorio donde esta el file encontrado
709 710 filename : el ultimo file de una determinada carpeta
710 711 year : el anho
711 712 doy : el numero de dia del anho
712 713 set : el set del archivo
713 714
714 715
715 716 """
716 717 if not os.path.isdir(path):
717 718 return None, None, None, None, None, None
718 719
719 720 dirList = []
720 721
721 722 if not walk:
722 723 fullpath = path
723 724 foldercounter = 0
724 725 else:
725 726 # Filtra solo los directorios
726 727 for thisPath in os.listdir(path):
727 728 if not os.path.isdir(os.path.join(path, thisPath)):
728 729 continue
729 730 if not isRadarFolder(thisPath):
730 731 continue
731 732
732 733 dirList.append(thisPath)
733 734
734 735 if not(dirList):
735 736 return None, None, None, None, None, None
736 737
737 738 dirList = sorted(dirList, key=str.lower)
738 739
739 740 doypath = dirList[-1]
740 741 foldercounter = int(doypath.split('_')[1]) if len(
741 742 doypath.split('_')) > 1 else 0
742 743 fullpath = os.path.join(path, doypath, expLabel)
743 744
744 745 print "[Reading] %s folder was found: " % (fullpath)
745 746
746 747 if set == None:
747 748 filename = getlastFileFromPath(fullpath, ext)
748 749 else:
749 750 filename = getFileFromSet(fullpath, ext, set)
750 751
751 752 if not(filename):
752 753 return None, None, None, None, None, None
753 754
754 755 print "[Reading] %s file was found" % (filename)
755 756
756 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
757 758 return None, None, None, None, None, None
758 759
759 760 year = int(filename[1:5])
760 761 doy = int(filename[5:8])
761 762 set = int(filename[8:11])
762 763
763 764 return fullpath, foldercounter, filename, year, doy, set
764 765
765 766 def __setNextFileOffline(self):
766 767
767 768 idFile = self.fileIndex
768 769
769 770 while (True):
770 771 idFile += 1
771 772 if not(idFile < len(self.filenameList)):
772 773 self.flagNoMoreFiles = 1
773 774 # print "[Reading] No more Files"
774 775 return 0
775 776
776 777 filename = self.filenameList[idFile]
777 778
778 779 if not(self.__verifyFile(filename)):
779 780 continue
780 781
781 782 fileSize = os.path.getsize(filename)
782 783 fp = open(filename, 'rb')
783 784 break
784 785
785 786 self.flagIsNewFile = 1
786 787 self.fileIndex = idFile
787 788 self.filename = filename
788 789 self.fileSize = fileSize
789 790 self.fp = fp
790 791
791 792 # print "[Reading] Setting the file: %s"%self.filename
792 793
793 794 return 1
794 795
795 796 def __setNextFileOnline(self):
796 797 """
797 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
798 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
799 800 siguientes.
800 801
801 802 Affected:
802 803 self.flagIsNewFile
803 804 self.filename
804 805 self.fileSize
805 806 self.fp
806 807 self.set
807 808 self.flagNoMoreFiles
808 809
809 810 Return:
810 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
811 812 1 : si el file fue abierto con exito y esta listo a ser leido
812 813
813 814 Excepciones:
814 815 Si un determinado file no puede ser abierto
815 816 """
816 817 nFiles = 0
817 818 fileOk_flag = False
818 819 firstTime_flag = True
819 820
820 821 self.set += 1
821 822
822 823 if self.set > 999:
823 824 self.set = 0
824 825 self.foldercounter += 1
825 826
826 827 # busca el 1er file disponible
827 828 fullfilename, filename = checkForRealPath(
828 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
829 830 if fullfilename:
830 831 if self.__verifyFile(fullfilename, False):
831 832 fileOk_flag = True
832 833
833 834 # si no encuentra un file entonces espera y vuelve a buscar
834 835 if not(fileOk_flag):
835 836 # busco en los siguientes self.nFiles+1 files posibles
836 837 for nFiles in range(self.nFiles + 1):
837 838
838 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
839 840 tries = self.nTries
840 841 else:
841 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
842 843
843 844 for nTries in range(tries):
844 845 if firstTime_flag:
845 846 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
846 847 sleep(self.delay)
847 848 else:
848 849 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
849 850
850 851 fullfilename, filename = checkForRealPath(
851 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
852 853 if fullfilename:
853 854 if self.__verifyFile(fullfilename):
854 855 fileOk_flag = True
855 856 break
856 857
857 858 if fileOk_flag:
858 859 break
859 860
860 861 firstTime_flag = False
861 862
862 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
863 864 self.set += 1
864 865
865 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
866 867 if nFiles == (self.nFiles - 1):
867 868 self.set = 0
868 869 self.doy += 1
869 870 self.foldercounter = 0
870 871
871 872 if fileOk_flag:
872 873 self.fileSize = os.path.getsize(fullfilename)
873 874 self.filename = fullfilename
874 875 self.flagIsNewFile = 1
875 876 if self.fp != None:
876 877 self.fp.close()
877 878 self.fp = open(fullfilename, 'rb')
878 879 self.flagNoMoreFiles = 0
879 880 # print '[Reading] Setting the file: %s' % fullfilename
880 881 else:
881 882 self.fileSize = 0
882 883 self.filename = None
883 884 self.flagIsNewFile = 0
884 885 self.fp = None
885 886 self.flagNoMoreFiles = 1
886 # print '[Reading] No more files to read'
887 887
888 888 return fileOk_flag
889 889
890 890 def setNextFile(self):
891 891 if self.fp != None:
892 892 self.fp.close()
893 893
894 894 if self.online:
895 895 newFile = self.__setNextFileOnline()
896 896 else:
897 897 newFile = self.__setNextFileOffline()
898 898
899 if not(newFile):
900 print '[Reading] No more files to read'
899 if not(newFile):
900 raise schainpy.admin.SchainWarning('No more files to read')
901 901 return 0
902 902
903 903 if self.verbose:
904 904 print '[Reading] Setting the file: %s' % self.filename
905 905
906 906 self.__readFirstHeader()
907 907 self.nReadBlocks = 0
908 908 return 1
909 909
910 910 def __waitNewBlock(self):
911 911 """
912 912 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
913 913
914 914 Si el modo de lectura es OffLine siempre retorn 0
915 915 """
916 916 if not self.online:
917 917 return 0
918 918
919 919 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
920 920 return 0
921 921
922 922 currentPointer = self.fp.tell()
923 923
924 924 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
925 925
926 926 for nTries in range(self.nTries):
927 927
928 928 self.fp.close()
929 929 self.fp = open(self.filename, 'rb')
930 930 self.fp.seek(currentPointer)
931 931
932 932 self.fileSize = os.path.getsize(self.filename)
933 933 currentSize = self.fileSize - currentPointer
934 934
935 935 if (currentSize >= neededSize):
936 936 self.basicHeaderObj.read(self.fp)
937 937 return 1
938 938
939 939 if self.fileSize == self.fileSizeByHeader:
940 940 # self.flagEoF = True
941 941 return 0
942 942
943 943 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
944 944 sleep(self.delay)
945 945
946 946 return 0
947 947
948 948 def waitDataBlock(self, pointer_location):
949 949
950 950 currentPointer = pointer_location
951 951
952 952 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
953 953
954 954 for nTries in range(self.nTries):
955 955 self.fp.close()
956 956 self.fp = open(self.filename, 'rb')
957 957 self.fp.seek(currentPointer)
958 958
959 959 self.fileSize = os.path.getsize(self.filename)
960 960 currentSize = self.fileSize - currentPointer
961 961
962 962 if (currentSize >= neededSize):
963 963 return 1
964 964
965 965 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
966 966 sleep(self.delay)
967 967
968 968 return 0
969 969
970 970 def __jumpToLastBlock(self):
971 971
972 972 if not(self.__isFirstTimeOnline):
973 973 return
974 974
975 975 csize = self.fileSize - self.fp.tell()
976 976 blocksize = self.processingHeaderObj.blockSize
977 977
978 978 # salta el primer bloque de datos
979 979 if csize > self.processingHeaderObj.blockSize:
980 980 self.fp.seek(self.fp.tell() + blocksize)
981 981 else:
982 982 return
983 983
984 984 csize = self.fileSize - self.fp.tell()
985 985 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
986 986 while True:
987 987
988 988 if self.fp.tell() < self.fileSize:
989 989 self.fp.seek(self.fp.tell() + neededsize)
990 990 else:
991 991 self.fp.seek(self.fp.tell() - neededsize)
992 992 break
993 993
994 994 # csize = self.fileSize - self.fp.tell()
995 995 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
996 996 # factor = int(csize/neededsize)
997 997 # if factor > 0:
998 998 # self.fp.seek(self.fp.tell() + factor*neededsize)
999 999
1000 1000 self.flagIsNewFile = 0
1001 1001 self.__isFirstTimeOnline = 0
1002 1002
1003 1003 def __setNewBlock(self):
1004 1004 # if self.server is None:
1005 1005 if self.fp == None:
1006 1006 return 0
1007 1007
1008 1008 # if self.online:
1009 1009 # self.__jumpToLastBlock()
1010 1010
1011 1011 if self.flagIsNewFile:
1012 1012 self.lastUTTime = self.basicHeaderObj.utc
1013 1013 return 1
1014 1014
1015 1015 if self.realtime:
1016 1016 self.flagDiscontinuousBlock = 1
1017 1017 if not(self.setNextFile()):
1018 1018 return 0
1019 1019 else:
1020 1020 return 1
1021 1021 # if self.server is None:
1022 1022 currentSize = self.fileSize - self.fp.tell()
1023 1023 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1024 1024 if (currentSize >= neededSize):
1025 1025 self.basicHeaderObj.read(self.fp)
1026 1026 self.lastUTTime = self.basicHeaderObj.utc
1027 1027 return 1
1028 1028 # else:
1029 1029 # self.basicHeaderObj.read(self.zHeader)
1030 1030 # self.lastUTTime = self.basicHeaderObj.utc
1031 1031 # return 1
1032 1032 if self.__waitNewBlock():
1033 1033 self.lastUTTime = self.basicHeaderObj.utc
1034 1034 return 1
1035 1035 # if self.server is None:
1036 1036 if not(self.setNextFile()):
1037 1037 return 0
1038 1038
1039 1039 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1040 1040 self.lastUTTime = self.basicHeaderObj.utc
1041 1041
1042 1042 self.flagDiscontinuousBlock = 0
1043 1043
1044 1044 if deltaTime > self.maxTimeStep:
1045 1045 self.flagDiscontinuousBlock = 1
1046 1046
1047 1047 return 1
1048 1048
1049 1049 def readNextBlock(self):
1050 1050
1051 1051 # Skip block out of startTime and endTime
1052 1052 while True:
1053 1053 if not(self.__setNewBlock()):
1054 1054 return 0
1055 1055
1056 1056 if not(self.readBlock()):
1057 1057 return 0
1058 1058
1059 1059 self.getBasicHeader()
1060 1060 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1061 1061 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1062 1062 self.processingHeaderObj.dataBlocksPerFile,
1063 1063 self.dataOut.datatime.ctime())
1064 1064 continue
1065 1065
1066 1066 break
1067 1067
1068 1068 if self.verbose:
1069 1069 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1070 1070 self.processingHeaderObj.dataBlocksPerFile,
1071 1071 self.dataOut.datatime.ctime())
1072 1072 return 1
1073 1073
1074 1074 def __readFirstHeader(self):
1075 1075
1076 1076 self.basicHeaderObj.read(self.fp)
1077 1077 self.systemHeaderObj.read(self.fp)
1078 1078 self.radarControllerHeaderObj.read(self.fp)
1079 1079 self.processingHeaderObj.read(self.fp)
1080 1080
1081 1081 self.firstHeaderSize = self.basicHeaderObj.size
1082 1082
1083 1083 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1084 1084 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1085 1085 if datatype == 0:
1086 1086 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1087 1087 elif datatype == 1:
1088 1088 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1089 1089 elif datatype == 2:
1090 1090 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1091 1091 elif datatype == 3:
1092 1092 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1093 1093 elif datatype == 4:
1094 1094 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1095 1095 elif datatype == 5:
1096 1096 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1097 1097 else:
1098 1098 raise ValueError, 'Data type was not defined'
1099 1099
1100 1100 self.dtype = datatype_str
1101 1101 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1102 1102 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1103 1103 self.firstHeaderSize + self.basicHeaderSize * \
1104 1104 (self.processingHeaderObj.dataBlocksPerFile - 1)
1105 1105 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1106 1106 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1107 1107 self.getBlockDimension()
1108 1108
1109 1109 def __verifyFile(self, filename, msgFlag=True):
1110 1110
1111 1111 msg = None
1112 1112
1113 1113 try:
1114 1114 fp = open(filename, 'rb')
1115 1115 except IOError:
1116 1116
1117 1117 if msgFlag:
1118 1118 print "[Reading] File %s can't be opened" % (filename)
1119 1119
1120 1120 return False
1121 1121
1122 1122 currentPosition = fp.tell()
1123 1123 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1124 1124
1125 1125 if neededSize == 0:
1126 1126 basicHeaderObj = BasicHeader(LOCALTIME)
1127 1127 systemHeaderObj = SystemHeader()
1128 1128 radarControllerHeaderObj = RadarControllerHeader()
1129 1129 processingHeaderObj = ProcessingHeader()
1130 1130
1131 1131 if not(basicHeaderObj.read(fp)):
1132 1132 fp.close()
1133 1133 return False
1134 1134
1135 1135 if not(systemHeaderObj.read(fp)):
1136 1136 fp.close()
1137 1137 return False
1138 1138
1139 1139 if not(radarControllerHeaderObj.read(fp)):
1140 1140 fp.close()
1141 1141 return False
1142 1142
1143 1143 if not(processingHeaderObj.read(fp)):
1144 1144 fp.close()
1145 1145 return False
1146 1146
1147 1147 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1148 1148 else:
1149 1149 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1150 1150
1151 1151 fp.close()
1152 1152
1153 1153 fileSize = os.path.getsize(filename)
1154 1154 currentSize = fileSize - currentPosition
1155 1155
1156 1156 if currentSize < neededSize:
1157 1157 if msgFlag and (msg != None):
1158 1158 print msg
1159 1159 return False
1160 1160
1161 1161 return True
1162 1162
1163 1163 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1164 1164
1165 1165 path_empty = True
1166 1166
1167 1167 dateList = []
1168 1168 pathList = []
1169 1169
1170 1170 multi_path = path.split(',')
1171 1171
1172 1172 if not walk:
1173 1173
1174 1174 for single_path in multi_path:
1175 1175
1176 1176 if not os.path.isdir(single_path):
1177 1177 continue
1178 1178
1179 1179 fileList = glob.glob1(single_path, "*" + ext)
1180 1180
1181 1181 if not fileList:
1182 1182 continue
1183 1183
1184 1184 path_empty = False
1185 1185
1186 1186 fileList.sort()
1187 1187
1188 1188 for thisFile in fileList:
1189 1189
1190 1190 if not os.path.isfile(os.path.join(single_path, thisFile)):
1191 1191 continue
1192 1192
1193 1193 if not isRadarFile(thisFile):
1194 1194 continue
1195 1195
1196 1196 if not isFileInDateRange(thisFile, startDate, endDate):
1197 1197 continue
1198 1198
1199 1199 thisDate = getDateFromRadarFile(thisFile)
1200 1200
1201 1201 if thisDate in dateList:
1202 1202 continue
1203 1203
1204 1204 dateList.append(thisDate)
1205 1205 pathList.append(single_path)
1206 1206
1207 1207 else:
1208 1208 for single_path in multi_path:
1209 1209
1210 1210 if not os.path.isdir(single_path):
1211 1211 continue
1212 1212
1213 1213 dirList = []
1214 1214
1215 1215 for thisPath in os.listdir(single_path):
1216 1216
1217 1217 if not os.path.isdir(os.path.join(single_path, thisPath)):
1218 1218 continue
1219 1219
1220 1220 if not isRadarFolder(thisPath):
1221 1221 continue
1222 1222
1223 1223 if not isFolderInDateRange(thisPath, startDate, endDate):
1224 1224 continue
1225 1225
1226 1226 dirList.append(thisPath)
1227 1227
1228 1228 if not dirList:
1229 1229 continue
1230 1230
1231 1231 dirList.sort()
1232 1232
1233 1233 for thisDir in dirList:
1234 1234
1235 1235 datapath = os.path.join(single_path, thisDir, expLabel)
1236 1236 fileList = glob.glob1(datapath, "*" + ext)
1237 1237
1238 1238 if not fileList:
1239 1239 continue
1240 1240
1241 1241 path_empty = False
1242 1242
1243 1243 thisDate = getDateFromRadarFolder(thisDir)
1244 1244
1245 1245 pathList.append(datapath)
1246 1246 dateList.append(thisDate)
1247 1247
1248 1248 dateList.sort()
1249 1249
1250 1250 if walk:
1251 1251 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1252 1252 else:
1253 1253 pattern_path = multi_path[0]
1254 1254
1255 1255 if path_empty:
1256 1256 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1257 1257 else:
1258 1258 if not dateList:
1259 1259 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1260 1260
1261 1261 if include_path:
1262 1262 return dateList, pathList
1263 1263
1264 1264 return dateList
1265 1265
1266 1266 def setup(self,
1267 1267 path=None,
1268 1268 startDate=None,
1269 1269 endDate=None,
1270 1270 startTime=datetime.time(0, 0, 0),
1271 1271 endTime=datetime.time(23, 59, 59),
1272 1272 set=None,
1273 1273 expLabel="",
1274 1274 ext=None,
1275 1275 online=False,
1276 1276 delay=60,
1277 1277 walk=True,
1278 1278 getblock=False,
1279 1279 nTxs=1,
1280 1280 realtime=False,
1281 1281 blocksize=None,
1282 1282 blocktime=None,
1283 1283 skip=None,
1284 1284 cursor=None,
1285 1285 warnings=True,
1286 1286 verbose=True,
1287 1287 server=None,
1288 1288 format=None,
1289 1289 oneDDict=None,
1290 1290 twoDDict=None,
1291 1291 ind2DList=None):
1292 1292 if server is not None:
1293 1293 if 'tcp://' in server:
1294 1294 address = server
1295 1295 else:
1296 1296 address = 'ipc:///tmp/%s' % server
1297 1297 self.server = address
1298 1298 self.context = zmq.Context()
1299 1299 self.receiver = self.context.socket(zmq.PULL)
1300 1300 self.receiver.connect(self.server)
1301 1301 time.sleep(0.5)
1302 1302 print '[Starting] ReceiverData from {}'.format(self.server)
1303 1303 else:
1304 1304 self.server = None
1305 1305 if path == None:
1306 1306 raise ValueError, "[Reading] The path is not valid"
1307 1307
1308 1308 if ext == None:
1309 1309 ext = self.ext
1310 1310
1311 1311 if online:
1312 1312 print "[Reading] Searching files in online mode..."
1313 1313
1314 1314 for nTries in range(self.nTries):
1315 1315 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1316 1316 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1317 1317
1318 1318 if fullpath:
1319 1319 break
1320 1320
1321 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1322 sleep(self.delay)
1321 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (delay, path, nTries + 1)
1322 sleep(delay)
1323 1323
1324 if not(fullpath):
1325 print "[Reading] There 'isn't any valid file in %s" % path
1324 if not(fullpath):
1325 raise schainpy.admin.SchainWarning('There isn\'t any valid file in {}'.format(path))
1326 1326 return
1327 1327
1328 1328 self.year = year
1329 1329 self.doy = doy
1330 1330 self.set = set - 1
1331 1331 self.path = path
1332 1332 self.foldercounter = foldercounter
1333 1333 last_set = None
1334 1334 else:
1335 1335 print "[Reading] Searching files in offline mode ..."
1336 1336 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1337 1337 startTime=startTime, endTime=endTime,
1338 1338 set=set, expLabel=expLabel, ext=ext,
1339 1339 walk=walk, cursor=cursor,
1340 1340 skip=skip)
1341 1341
1342 1342 if not(pathList):
1343 1343 self.fileIndex = -1
1344 1344 self.pathList = []
1345 1345 self.filenameList = []
1346 1346 return
1347 1347
1348 1348 self.fileIndex = -1
1349 1349 self.pathList = pathList
1350 1350 self.filenameList = filenameList
1351 1351 file_name = os.path.basename(filenameList[-1])
1352 1352 basename, ext = os.path.splitext(file_name)
1353 1353 last_set = int(basename[-3:])
1354 1354
1355 1355 self.online = online
1356 1356 self.realtime = realtime
1357 1357 self.delay = delay
1358 1358 ext = ext.lower()
1359 1359 self.ext = ext
1360 1360 self.getByBlock = getblock
1361 1361 self.nTxs = nTxs
1362 1362 self.startTime = startTime
1363 1363 self.endTime = endTime
1364 1364 self.endDate = endDate
1365 1365 self.startDate = startDate
1366 1366 # Added-----------------
1367 1367 self.selBlocksize = blocksize
1368 1368 self.selBlocktime = blocktime
1369 1369
1370 1370 # Verbose-----------
1371 1371 self.verbose = verbose
1372 1372 self.warnings = warnings
1373 1373
1374 1374 if not(self.setNextFile()):
1375 1375 if (startDate != None) and (endDate != None):
1376 1376 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1377 1377 elif startDate != None:
1378 1378 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1379 1379 else:
1380 1380 print "[Reading] No files"
1381 1381
1382 1382 self.fileIndex = -1
1383 1383 self.pathList = []
1384 1384 self.filenameList = []
1385 1385 return
1386 1386
1387 1387 # self.getBasicHeader()
1388 1388
1389 1389 if last_set != None:
1390 1390 self.dataOut.last_block = last_set * \
1391 1391 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1392 1392 return
1393 1393
1394 1394 def getBasicHeader(self):
1395 1395
1396 1396 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1397 1397 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1398 1398
1399 1399 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1400 1400
1401 1401 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1402 1402
1403 1403 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1404 1404
1405 1405 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1406 1406
1407 1407 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1408 1408
1409 1409 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1410 1410
1411 1411 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1412 1412
1413 1413 def getFirstHeader(self):
1414 1414
1415 1415 raise NotImplementedError
1416 1416
1417 1417 def getData(self):
1418 1418
1419 1419 raise NotImplementedError
1420 1420
1421 1421 def hasNotDataInBuffer(self):
1422 1422
1423 1423 raise NotImplementedError
1424 1424
1425 1425 def readBlock(self):
1426 1426
1427 1427 raise NotImplementedError
1428 1428
1429 1429 def isEndProcess(self):
1430 1430
1431 1431 return self.flagNoMoreFiles
1432 1432
1433 1433 def printReadBlocks(self):
1434 1434
1435 1435 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1436 1436
1437 1437 def printTotalBlocks(self):
1438 1438
1439 1439 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1440 1440
1441 1441 def printNumberOfBlock(self):
1442 1442 'SPAM!'
1443 1443
1444 1444 # if self.flagIsNewBlock:
1445 1445 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1446 1446 # self.processingHeaderObj.dataBlocksPerFile,
1447 1447 # self.dataOut.datatime.ctime())
1448 1448
1449 1449 def printInfo(self):
1450 1450
1451 1451 if self.__printInfo == False:
1452 1452 return
1453 1453
1454 1454 self.basicHeaderObj.printInfo()
1455 1455 self.systemHeaderObj.printInfo()
1456 1456 self.radarControllerHeaderObj.printInfo()
1457 1457 self.processingHeaderObj.printInfo()
1458 1458
1459 1459 self.__printInfo = False
1460 1460
1461 1461 def run(self,
1462 1462 path=None,
1463 1463 startDate=None,
1464 1464 endDate=None,
1465 1465 startTime=datetime.time(0, 0, 0),
1466 1466 endTime=datetime.time(23, 59, 59),
1467 1467 set=None,
1468 1468 expLabel="",
1469 1469 ext=None,
1470 1470 online=False,
1471 1471 delay=60,
1472 1472 walk=True,
1473 1473 getblock=False,
1474 1474 nTxs=1,
1475 1475 realtime=False,
1476 1476 blocksize=None,
1477 1477 blocktime=None,
1478 1478 skip=None,
1479 1479 cursor=None,
1480 1480 warnings=True,
1481 1481 server=None,
1482 1482 verbose=True,
1483 1483 format=None,
1484 1484 oneDDict=None,
1485 1485 twoDDict=None,
1486 1486 ind2DList=None, **kwargs):
1487 1487
1488 1488 if not(self.isConfig):
1489 1489 self.setup(path=path,
1490 1490 startDate=startDate,
1491 1491 endDate=endDate,
1492 1492 startTime=startTime,
1493 1493 endTime=endTime,
1494 1494 set=set,
1495 1495 expLabel=expLabel,
1496 1496 ext=ext,
1497 1497 online=online,
1498 1498 delay=delay,
1499 1499 walk=walk,
1500 1500 getblock=getblock,
1501 1501 nTxs=nTxs,
1502 1502 realtime=realtime,
1503 1503 blocksize=blocksize,
1504 1504 blocktime=blocktime,
1505 1505 skip=skip,
1506 1506 cursor=cursor,
1507 1507 warnings=warnings,
1508 1508 server=server,
1509 1509 verbose=verbose,
1510 1510 format=format,
1511 1511 oneDDict=oneDDict,
1512 1512 twoDDict=twoDDict,
1513 1513 ind2DList=ind2DList)
1514 1514 self.isConfig = True
1515 1515 if server is None:
1516 1516 self.getData()
1517 1517 else:
1518 1518 self.getFromServer()
1519 1519
1520 1520
1521 1521 class JRODataWriter(JRODataIO):
1522 1522
1523 1523 """
1524 1524 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1525 1525 de los datos siempre se realiza por bloques.
1526 1526 """
1527 1527
1528 1528 blockIndex = 0
1529 1529
1530 1530 path = None
1531 1531
1532 1532 setFile = None
1533 1533
1534 1534 profilesPerBlock = None
1535 1535
1536 1536 blocksPerFile = None
1537 1537
1538 1538 nWriteBlocks = 0
1539 1539
1540 1540 fileDate = None
1541 1541
1542 1542 def __init__(self, dataOut=None):
1543 1543 raise NotImplementedError
1544 1544
1545 1545 def hasAllDataInBuffer(self):
1546 1546 raise NotImplementedError
1547 1547
1548 1548 def setBlockDimension(self):
1549 1549 raise NotImplementedError
1550 1550
1551 1551 def writeBlock(self):
1552 1552 raise NotImplementedError
1553 1553
1554 1554 def putData(self):
1555 1555 raise NotImplementedError
1556 1556
1557 1557 def getProcessFlags(self):
1558 1558
1559 1559 processFlags = 0
1560 1560
1561 1561 dtype_index = get_dtype_index(self.dtype)
1562 1562 procflag_dtype = get_procflag_dtype(dtype_index)
1563 1563
1564 1564 processFlags += procflag_dtype
1565 1565
1566 1566 if self.dataOut.flagDecodeData:
1567 1567 processFlags += PROCFLAG.DECODE_DATA
1568 1568
1569 1569 if self.dataOut.flagDeflipData:
1570 1570 processFlags += PROCFLAG.DEFLIP_DATA
1571 1571
1572 1572 if self.dataOut.code is not None:
1573 1573 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1574 1574
1575 1575 if self.dataOut.nCohInt > 1:
1576 1576 processFlags += PROCFLAG.COHERENT_INTEGRATION
1577 1577
1578 1578 if self.dataOut.type == "Spectra":
1579 1579 if self.dataOut.nIncohInt > 1:
1580 1580 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1581 1581
1582 1582 if self.dataOut.data_dc is not None:
1583 1583 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1584 1584
1585 1585 if self.dataOut.flagShiftFFT:
1586 1586 processFlags += PROCFLAG.SHIFT_FFT_DATA
1587 1587
1588 1588 return processFlags
1589 1589
1590 1590 def setBasicHeader(self):
1591 1591
1592 1592 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1593 1593 self.basicHeaderObj.version = self.versionFile
1594 1594 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1595 1595
1596 1596 utc = numpy.floor(self.dataOut.utctime)
1597 1597 milisecond = (self.dataOut.utctime - utc) * 1000.0
1598 1598
1599 1599 self.basicHeaderObj.utc = utc
1600 1600 self.basicHeaderObj.miliSecond = milisecond
1601 1601 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1602 1602 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1603 1603 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1604 1604
1605 1605 def setFirstHeader(self):
1606 1606 """
1607 1607 Obtiene una copia del First Header
1608 1608
1609 1609 Affected:
1610 1610
1611 1611 self.basicHeaderObj
1612 1612 self.systemHeaderObj
1613 1613 self.radarControllerHeaderObj
1614 1614 self.processingHeaderObj self.
1615 1615
1616 1616 Return:
1617 1617 None
1618 1618 """
1619 1619
1620 1620 raise NotImplementedError
1621 1621
1622 1622 def __writeFirstHeader(self):
1623 1623 """
1624 1624 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1625 1625
1626 1626 Affected:
1627 1627 __dataType
1628 1628
1629 1629 Return:
1630 1630 None
1631 1631 """
1632 1632
1633 1633 # CALCULAR PARAMETROS
1634 1634
1635 1635 sizeLongHeader = self.systemHeaderObj.size + \
1636 1636 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1637 1637 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1638 1638
1639 1639 self.basicHeaderObj.write(self.fp)
1640 1640 self.systemHeaderObj.write(self.fp)
1641 1641 self.radarControllerHeaderObj.write(self.fp)
1642 1642 self.processingHeaderObj.write(self.fp)
1643 1643
1644 1644 def __setNewBlock(self):
1645 1645 """
1646 1646 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1647 1647
1648 1648 Return:
1649 1649 0 : si no pudo escribir nada
1650 1650 1 : Si escribio el Basic el First Header
1651 1651 """
1652 1652 if self.fp == None:
1653 1653 self.setNextFile()
1654 1654
1655 1655 if self.flagIsNewFile:
1656 1656 return 1
1657 1657
1658 1658 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1659 1659 self.basicHeaderObj.write(self.fp)
1660 1660 return 1
1661 1661
1662 1662 if not(self.setNextFile()):
1663 1663 return 0
1664 1664
1665 1665 return 1
1666 1666
1667 1667 def writeNextBlock(self):
1668 1668 """
1669 1669 Selecciona el bloque siguiente de datos y los escribe en un file
1670 1670
1671 1671 Return:
1672 1672 0 : Si no hizo pudo escribir el bloque de datos
1673 1673 1 : Si no pudo escribir el bloque de datos
1674 1674 """
1675 1675 if not(self.__setNewBlock()):
1676 1676 return 0
1677 1677
1678 1678 self.writeBlock()
1679 1679
1680 1680 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1681 1681 self.processingHeaderObj.dataBlocksPerFile)
1682 1682
1683 1683 return 1
1684 1684
1685 1685 def setNextFile(self):
1686 1686 """
1687 1687 Determina el siguiente file que sera escrito
1688 1688
1689 1689 Affected:
1690 1690 self.filename
1691 1691 self.subfolder
1692 1692 self.fp
1693 1693 self.setFile
1694 1694 self.flagIsNewFile
1695 1695
1696 1696 Return:
1697 1697 0 : Si el archivo no puede ser escrito
1698 1698 1 : Si el archivo esta listo para ser escrito
1699 1699 """
1700 1700 ext = self.ext
1701 1701 path = self.path
1702 1702
1703 1703 if self.fp != None:
1704 1704 self.fp.close()
1705 1705
1706 1706 timeTuple = time.localtime(self.dataOut.utctime)
1707 1707 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1708 1708
1709 1709 fullpath = os.path.join(path, subfolder)
1710 1710 setFile = self.setFile
1711 1711
1712 1712 if not(os.path.exists(fullpath)):
1713 1713 os.mkdir(fullpath)
1714 1714 setFile = -1 # inicializo mi contador de seteo
1715 1715 else:
1716 1716 filesList = os.listdir(fullpath)
1717 1717 if len(filesList) > 0:
1718 1718 filesList = sorted(filesList, key=str.lower)
1719 1719 filen = filesList[-1]
1720 1720 # el filename debera tener el siguiente formato
1721 1721 # 0 1234 567 89A BCDE (hex)
1722 1722 # x YYYY DDD SSS .ext
1723 1723 if isNumber(filen[8:11]):
1724 1724 # inicializo mi contador de seteo al seteo del ultimo file
1725 1725 setFile = int(filen[8:11])
1726 1726 else:
1727 1727 setFile = -1
1728 1728 else:
1729 1729 setFile = -1 # inicializo mi contador de seteo
1730 1730
1731 1731 setFile += 1
1732 1732
1733 1733 # If this is a new day it resets some values
1734 1734 if self.dataOut.datatime.date() > self.fileDate:
1735 1735 setFile = 0
1736 1736 self.nTotalBlocks = 0
1737 1737
1738 1738 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1739 1739 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1740 1740
1741 1741 filename = os.path.join(path, subfolder, filen)
1742 1742
1743 1743 fp = open(filename, 'wb')
1744 1744
1745 1745 self.blockIndex = 0
1746 1746
1747 1747 # guardando atributos
1748 1748 self.filename = filename
1749 1749 self.subfolder = subfolder
1750 1750 self.fp = fp
1751 1751 self.setFile = setFile
1752 1752 self.flagIsNewFile = 1
1753 1753 self.fileDate = self.dataOut.datatime.date()
1754 1754
1755 1755 self.setFirstHeader()
1756 1756
1757 1757 print '[Writing] Opening file: %s' % self.filename
1758 1758
1759 1759 self.__writeFirstHeader()
1760 1760
1761 1761 return 1
1762 1762
1763 1763 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1764 1764 """
1765 1765 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1766 1766
1767 1767 Inputs:
1768 1768 path : directory where data will be saved
1769 1769 profilesPerBlock : number of profiles per block
1770 1770 set : initial file set
1771 1771 datatype : An integer number that defines data type:
1772 1772 0 : int8 (1 byte)
1773 1773 1 : int16 (2 bytes)
1774 1774 2 : int32 (4 bytes)
1775 1775 3 : int64 (8 bytes)
1776 1776 4 : float32 (4 bytes)
1777 1777 5 : double64 (8 bytes)
1778 1778
1779 1779 Return:
1780 1780 0 : Si no realizo un buen seteo
1781 1781 1 : Si realizo un buen seteo
1782 1782 """
1783 1783
1784 1784 if ext == None:
1785 1785 ext = self.ext
1786 1786
1787 1787 self.ext = ext.lower()
1788 1788
1789 1789 self.path = path
1790 1790
1791 1791 if set is None:
1792 1792 self.setFile = -1
1793 1793 else:
1794 1794 self.setFile = set - 1
1795 1795
1796 1796 self.blocksPerFile = blocksPerFile
1797 1797
1798 1798 self.profilesPerBlock = profilesPerBlock
1799 1799
1800 1800 self.dataOut = dataOut
1801 1801 self.fileDate = self.dataOut.datatime.date()
1802 1802 # By default
1803 1803 self.dtype = self.dataOut.dtype
1804 1804
1805 1805 if datatype is not None:
1806 1806 self.dtype = get_numpy_dtype(datatype)
1807 1807
1808 1808 if not(self.setNextFile()):
1809 1809 print "[Writing] There isn't a next file"
1810 1810 return 0
1811 1811
1812 1812 self.setBlockDimension()
1813 1813
1814 1814 return 1
1815 1815
1816 1816 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1817 1817
1818 1818 if not(self.isConfig):
1819 1819
1820 1820 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1821 1821 set=set, ext=ext, datatype=datatype, **kwargs)
1822 1822 self.isConfig = True
1823 1823
1824 1824 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now