##// END OF EJS Templates
schain-jc
RSS
Merge branch 'schain_alarm' into v2.3
Juan C. Espinoza -
r1159:d0480f11c356 merge
parent child
Show More
Side by Side Unified
Context file:
r1159:d0480f11c356 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,498 +1,503
1 """The admin module contains all administrative classes relating to the schain python api.
1 """
2 The admin module contains all administrative classes relating to the schain python api.
2 3
3 4 The main role of this module is to send some reports. It contains a
4 5 notification class and a standard error handing class.
5 6
6 7 $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
7 8 """
8 9 import os
9 10 import sys
10 11 import time
11 12 import traceback
12 13 import smtplib
13 14 import ConfigParser
14 15 import StringIO
15 16 from threading import Thread
16 17 from multiprocessing import Process
17 18 from email.mime.text import MIMEText
18 19 from email.mime.application import MIMEApplication
19 20 from email.mime.multipart import MIMEMultipart
20 21
22 import schainpy
21 23 from schainpy.utils import log
22 24 from schainpy.model.graphics.jroplot_data import popup
23 25
24 26 def get_path():
25 27 '''
26 28 Return schainpy path
27 29 '''
28 30
29 31 try:
30 32 root = __file__
31 33 if os.path.islink(root):
32 34 root = os.path.realpath(root)
33 35
34 36 return os.path.dirname(os.path.abspath(root))
35 37 except:
36 38 log.error('I am sorry, but something is wrong... __file__ not found')
37 39
38 40 class Alarm(Process):
39 41 '''
40 42 modes:
41 43 0 - All
42 1 - Sound alarm
43 2 - Send email
44 3 - Popup message
44 1 - Send email
45 2 - Popup message
46 3 - Sound alarm
45 47 4 - Send to alarm system TODO
46 48 '''
47 49
48 def __init__(self, modes=[1], **kwargs):
50 def __init__(self, modes=[], **kwargs):
49 51 Process.__init__(self)
50 52 self.modes = modes
51 53 self.kwargs = kwargs
52 54
53 55 @staticmethod
54 56 def play_sound():
55 57 sound = os.path.join(get_path(), 'alarm1.oga')
56 58 if os.path.exists(sound):
57 59 for __ in range(2):
58 60 os.system('paplay {}'.format(sound))
59 61 time.sleep(0.5)
60 62 else:
61 63 log.warning('Unable to play alarm, sound file not found', 'ADMIN')
62 64
63 65 @staticmethod
64 66 def send_email(**kwargs):
65 67 notifier = SchainNotify()
68 print kwargs
66 69 notifier.notify(**kwargs)
67 70
68 71 @staticmethod
69 def show_popup(message='Error'):
72 def show_popup(message):
73 if isinstance(message, list):
74 message = message[-1]
70 75 popup(message)
71 76
72 77 @staticmethod
73 78 def send_alarm():
74 79 pass
75 80
76 81 @staticmethod
77 82 def get_kwargs(kwargs, keys):
78 83 ret = {}
79 84 for key in keys:
80 85 ret[key] = kwargs[key]
81 86 return ret
82 87
83 88 def run(self):
84 89 tasks = {
85 90 1 : self.send_email,
86 2 : self.play_sound,
87 3 : self.show_popup,
91 2 : self.show_popup,
92 3 : self.play_sound,
88 93 4 : self.send_alarm,
89 94 }
90 95
91 96 tasks_args = {
92 97 1: ['email', 'message', 'subject', 'subtitle', 'filename'],
93 2: [],
94 3: ['message'],
98 2: ['message'],
99 3: [],
95 100 4: [],
96 101 }
97 102 procs = []
98 103 for mode in self.modes:
99 104 if 0 in self.modes:
100 105 for x in tasks:
101 106 t = Thread(target=tasks[x], kwargs=self.get_kwargs(self.kwargs, tasks_args[x]))
102 107 t.start()
103 108 procs.append(t)
104 109 break
105 110 else:
106 111 t = Thread(target=tasks[mode], kwargs=self.get_kwargs(self.kwargs, tasks_args[mode]))
107 112 t.start()
108 113 procs.append(t)
109 114 for t in procs:
110 115 t.join()
111 116
112 117
113 118 class SchainConfigure():
114 119
115 120 __DEFAULT_ADMINISTRATOR_EMAIL = "juan.espinoza@jro.igp.gob.pe"
116 121 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
117 122 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
118 123 __DEFAULT_SENDER_PASS = ""
119 124
120 125 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
121 126 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
122 127 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
123 128 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
124 129
125 130 def __init__(self, initFile = None):
126 131
127 132 # Set configuration file
128 133 if (initFile == None):
129 134 self.__confFilePath = "/etc/schain.conf"
130 135 else:
131 136 self.__confFilePath = initFile
132 137
133 138 # open configuration file
134 139 try:
135 140 self.__confFile = open(self.__confFilePath, "r")
136 141 except IOError:
137 142 # can't read from file - use all hard-coded values
138 143 self.__initFromHardCode()
139 144 return
140 145
141 146 # create Parser using standard module ConfigParser
142 147 self.__parser = ConfigParser.ConfigParser()
143 148
144 149 # read conf file into a StringIO with "[madrigal]\n" section heading prepended
145 150 strConfFile = StringIO.StringIO("[schain]\n" + self.__confFile.read())
146 151
147 152 # parse StringIO configuration file
148 153 self.__parser.readfp(strConfFile)
149 154
150 155 # read information from configuration file
151 156 self.__readConfFile()
152 157
153 158 # close conf file
154 159 self.__confFile.close()
155 160
156 161
157 162 def __initFromHardCode(self):
158 163
159 164 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
160 165 self.__sender_pass = self.__DEFAULT_SENDER_PASS
161 166 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
162 167 self.__email_server = self.__DEFAULT_EMAIL_SERVER
163 168
164 169 def __readConfFile(self):
165 170 """__readConfFile is a private helper function that reads information from the parsed config file.
166 171
167 172 Inputs: None
168 173
169 174 Returns: Void.
170 175
171 176 Affects: Initializes class member variables that are found in the config file.
172 177
173 178 Exceptions: MadrigalError thrown if any key not found.
174 179 """
175 180
176 181 # get the sender email
177 182 try:
178 183 self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
179 184 except:
180 185 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
181 186
182 187 # get the sender password
183 188 try:
184 189 self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
185 190 except:
186 191 self.__sender_pass = self.__DEFAULT_SENDER_PASS
187 192
188 193 # get the administrator email
189 194 try:
190 195 self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
191 196 except:
192 197 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
193 198
194 199 # get the server email
195 200 try:
196 201 self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
197 202 except:
198 203 self.__email_server = self.__DEFAULT_EMAIL_SERVER
199 204
200 205 def getEmailServer(self):
201 206
202 207 return self.__email_server
203 208
204 209 def getSenderEmail(self):
205 210
206 211 return self.__sender_email
207 212
208 213 def getSenderPass(self):
209 214
210 215 return self.__sender_pass
211 216
212 217 def getAdminEmail(self):
213 218
214 219 return self.__admin_email
215 220
216 221 class SchainNotify:
217 222 """SchainNotify is an object used to send messages to an administrator about a Schain software.
218 223
219 224 This object provides functions needed to send messages to an administrator about a Schain , for now
220 225 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
221 226
222 227 Usage example:
223 228
224 229 import schainpy.admin
225 230
226 231 try:
227 232
228 233 adminObj = schainpy.admin.SchainNotify()
229 234 adminObj.sendAlert('This is important!', 'Important Message')
230 235
231 236 except schainpy.admin.SchainError, e:
232 237
233 238 print e.getExceptionStr()
234 239
235 240
236 241 Non-standard Python modules used:
237 242 None
238 243
239 244 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
240 245 throwing exceptions, since this is the class that will generally be called when there is a problem.
241 246
242 247 Change history:
243 248
244 249 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Dec. 1, 2015
245 250 """
246 251
247 252 #constants
248 253
249 254 def __init__(self):
250 255 """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
251 256
252 257 Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
253 258 this is the class that will generally be called when there is a problem.
254 259
255 260 Inputs: Existing SchainDB object, by default = None.
256 261
257 262 Returns: void
258 263
259 264 Affects: Initializes self.__binDir.
260 265
261 266 Exceptions: None.
262 267 """
263 268
264 269 # note that the main configuration file is unavailable
265 270 # the best that can be done is send an email to root using localhost mailserver
266 271 confObj = SchainConfigure()
267 272
268 273 self.__emailFromAddress = confObj.getSenderEmail()
269 274 self.__emailPass = confObj.getSenderPass()
270 275 self.__emailToAddress = confObj.getAdminEmail()
271 276 self.__emailServer = confObj.getEmailServer()
272 277
273 278 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
274 279
275 280 if not email_to:
276 281 return 0
277 282
278 283 if not self.__emailServer:
279 284 return 0
280 285
281 286 log.success('Sending email to {}...'.format(email_to), 'System')
282 287
283 288 msg = MIMEMultipart()
284 289 msg['Subject'] = subject
285 290 msg['From'] = "(Python SChain API): " + email_from
286 291 msg['Reply-to'] = email_from
287 292 msg['To'] = email_to
288 293
289 294 # That is what u see if dont have an email reader:
290 295 msg.preamble = 'SChainPy'
291 296
292 297 if html_format:
293 298 message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
294 299 message = "<html>\n" + message + '</html>'
295 300
296 301 # This is the textual part:
297 302 part = MIMEText(message, "html")
298 303 else:
299 304 message = subject + "\n" + subtitle + "\n" + message
300 305 part = MIMEText(message)
301 306
302 307 msg.attach(part)
303 308
304 309 if filename and os.path.isfile(filename):
305 310 # This is the binary part(The Attachment):
306 311 part = MIMEApplication(open(filename,"rb").read())
307 312 part.add_header('Content-Disposition',
308 313 'attachment',
309 314 filename=os.path.basename(filename))
310 315 msg.attach(part)
311 316
312 317 # Create an instance in SMTP server
313 318 try:
314 319 smtp = smtplib.SMTP(self.__emailServer)
315 320 except:
316 321 log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
317 322 return 0
318 323
319 324 # Start the server:
320 325 # smtp.ehlo()
321 326 if self.__emailPass:
322 327 smtp.login(self.__emailFromAddress, self.__emailPass)
323 328
324 329 # Send the email
325 330 try:
326 331 smtp.sendmail(msg['From'], msg['To'], msg.as_string())
327 332 except:
328 333 log.error('Could not send the email to {}'.format(msg['To']), 'System')
329 334 smtp.quit()
330 335 return 0
331 336
332 337 smtp.quit()
333 338
334 339 log.success('Email sent ', 'System')
335 340
336 341 return 1
337 342
338 343 def sendAlert(self, message, subject = "", subtitle="", filename=""):
339 344 """sendAlert sends an email with the given message and optional title.
340 345
341 346 Inputs: message (string), and optional title (string)
342 347
343 348 Returns: void
344 349
345 350 Affects: none
346 351
347 352 Exceptions: None.
348 353 """
349 354
350 355 if not self.__emailToAddress:
351 356 return 0
352 357
353 358 print "***** Sending alert to %s *****" %self.__emailToAddress
354 359 # set up message
355 360
356 361 sent=self.sendEmail(email_from=self.__emailFromAddress,
357 362 email_to=self.__emailToAddress,
358 363 subject=subject,
359 364 message=message,
360 365 subtitle=subtitle,
361 366 filename=filename)
362 367
363 368 if not sent:
364 369 return 0
365 370
366 371 return 1
367 372
368 373 def notify(self, email, message, subject = "", subtitle="", filename=""):
369 374 """notify sends an email with the given message and title to email.
370 375
371 376 Inputs: email (string), message (string), and subject (string)
372 377
373 378 Returns: void
374 379
375 380 Affects: none
376 381
377 382 Exceptions: None.
378 383 """
379 384
380 385 if email is None:
381 386 email = self.__emailToAddress
382 387
383 388 self.sendEmail(
384 389 email_from=self.__emailFromAddress,
385 390 email_to=email,
386 391 subject=subject,
387 392 message=message,
388 393 subtitle=subtitle,
389 394 filename=filename
390 395 )
391 396
392 397
393 398 class SchainError(Exception):
394 399 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
395 400
396 401 Usage example:
397 402
398 403 import sys, traceback
399 404 import schainpy.admin
400 405
401 406 try:
402 407
403 408 test = open('ImportantFile.txt', 'r')
404 409
405 410 except:
406 411
407 412 raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
408 413 traceback.format_exception(sys.exc_info()[0],
409 414 sys.exc_info()[1],
410 415 sys.exc_info()[2]))
411 416 """
412 417
413 418
414 419 def __init__(self, strInterpretation, exceptionList=None):
415 420 """ __init__ gathers the interpretation string along with all information from sys.exc_info().
416 421
417 422 Inputs:
418 423 strIntepretation - A string representing the programmer's interpretation of
419 424 why the exception occurred
420 425
421 426 exceptionList - a list of strings completely describing the exception.
422 427 Generated by traceback.format_exception(sys.exc_info()[0],
423 428 sys.exc_info()[1],
424 429 sys.exc_info()[2])
425 430
426 431 Returns: Void.
427 432
428 433 Affects: Initializes class member variables _strInterp, _strExcList.
429 434
430 435 Exceptions: None.
431 436 """
432 437
433 438 if not exceptionList:
434 439 exceptionList = traceback.format_exception(sys.exc_info()[0],
435 440 sys.exc_info()[1],
436 441 sys.exc_info()[2])
437 442
438 443 self._strInterp = strInterpretation
439 444 self._strExcList = exceptionList
440 445
441 446
442 447 def getExceptionStr(self):
443 448 """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
444 449
445 450 Inputs: None
446 451
447 452 Returns: A formatted string ready for printing completely describing the exception.
448 453
449 454 Affects: None
450 455
451 456 Exceptions: None.
452 457 """
453 458 excStr = ''
454 459 excStr = excStr + self._strInterp + '\n\n'
455 460
456 461 if self._strExcList != None:
457 462 for item in self._strExcList:
458 463 excStr = excStr + str(item) + '\n'
459 464
460 465 return excStr
461 466
462 467 def __str__(self):
463 468
464 469 return(self.getExceptionStr())
465 470
466 471
467 472 def getExceptionHtml(self):
468 473 """ getExceptionHtml returns an Html formatted string completely describing the exception.
469 474
470 475 Inputs: None
471 476
472 477 Returns: A formatted string ready for printing completely describing the exception.
473 478
474 479 Affects: None
475 480
476 481 Exceptions: None.
477 482 """
478 483
479 484 excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
480 485 excStr = excStr + self._strInterp + '\n<BR>\n'
481 486
482 487 if self._strExcList != None:
483 488 for item in self._strExcList:
484 489 excStr = excStr + str(item) + '\n<BR>'
485 490
486 491 return excStr
487 492
488 493 class SchainWarning(Exception):
489 494 pass
490 495
491 496
492 497 if __name__ == '__main__':
493 498
494 499 test = SchainNotify()
495 500
496 501 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
497 502
498 503 print 'Hopefully message sent - check.'
Show file before | Show file after | Hide comments
@@ -1,1330 +1,1337
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, cpu_count
13 13
14 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 15 from xml.dom import minidom
16 16
17 17 import schainpy
18 18 from schainpy.admin import Alarm, SchainWarning
19 19 from schainpy.model import *
20 20 from schainpy.utils import log
21 21
22 22 DTYPES = {
23 23 'Voltage': '.r',
24 24 'Spectra': '.pdata'
25 25 }
26 26
27 27
28 28 def MPProject(project, n=cpu_count()):
29 29 '''
30 30 Project wrapper to run schain in n processes
31 31 '''
32 32
33 33 rconf = project.getReadUnitObj()
34 34 op = rconf.getOperationObj('run')
35 35 dt1 = op.getParameterValue('startDate')
36 36 dt2 = op.getParameterValue('endDate')
37 37 tm1 = op.getParameterValue('startTime')
38 38 tm2 = op.getParameterValue('endTime')
39 39 days = (dt2 - dt1).days
40 40
41 41 for day in range(days + 1):
42 42 skip = 0
43 43 cursor = 0
44 44 processes = []
45 45 dt = dt1 + datetime.timedelta(day)
46 46 dt_str = dt.strftime('%Y/%m/%d')
47 47 reader = JRODataReader()
48 48 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 49 startDate=dt,
50 50 endDate=dt,
51 51 startTime=tm1,
52 52 endTime=tm2,
53 53 ext=DTYPES[rconf.datatype])
54 54 nFiles = len(files)
55 55 if nFiles == 0:
56 56 continue
57 57 skip = int(math.ceil(nFiles / n))
58 58 while nFiles > cursor * skip:
59 59 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 60 skip=skip)
61 61 p = project.clone()
62 62 p.start()
63 63 processes.append(p)
64 64 cursor += 1
65 65
66 66 def beforeExit(exctype, value, trace):
67 67 for process in processes:
68 68 process.terminate()
69 69 process.join()
70 70 print traceback.print_tb(trace)
71 71
72 72 sys.excepthook = beforeExit
73 73
74 74 for process in processes:
75 75 process.join()
76 76 process.terminate()
77 77
78 78 time.sleep(3)
79 79
80 80
81 81 class ParameterConf():
82 82
83 83 id = None
84 84 name = None
85 85 value = None
86 86 format = None
87 87
88 88 __formated_value = None
89 89
90 90 ELEMENTNAME = 'Parameter'
91 91
92 92 def __init__(self):
93 93
94 94 self.format = 'str'
95 95
96 96 def getElementName(self):
97 97
98 98 return self.ELEMENTNAME
99 99
100 100 def getValue(self):
101 101
102 102 value = self.value
103 103 format = self.format
104 104
105 105 if self.__formated_value != None:
106 106
107 107 return self.__formated_value
108 108
109 109 if format == 'obj':
110 110 return value
111 111
112 112 if format == 'str':
113 113 self.__formated_value = str(value)
114 114 return self.__formated_value
115 115
116 116 if value == '':
117 117 raise ValueError, '%s: This parameter value is empty' % self.name
118 118
119 119 if format == 'list':
120 120 strList = value.split(',')
121 121
122 122 self.__formated_value = strList
123 123
124 124 return self.__formated_value
125 125
126 126 if format == 'intlist':
127 127 '''
128 128 Example:
129 129 value = (0,1,2)
130 130 '''
131 131
132 132 new_value = ast.literal_eval(value)
133 133
134 134 if type(new_value) not in (tuple, list):
135 135 new_value = [int(new_value)]
136 136
137 137 self.__formated_value = new_value
138 138
139 139 return self.__formated_value
140 140
141 141 if format == 'floatlist':
142 142 '''
143 143 Example:
144 144 value = (0.5, 1.4, 2.7)
145 145 '''
146 146
147 147 new_value = ast.literal_eval(value)
148 148
149 149 if type(new_value) not in (tuple, list):
150 150 new_value = [float(new_value)]
151 151
152 152 self.__formated_value = new_value
153 153
154 154 return self.__formated_value
155 155
156 156 if format == 'date':
157 157 strList = value.split('/')
158 158 intList = [int(x) for x in strList]
159 159 date = datetime.date(intList[0], intList[1], intList[2])
160 160
161 161 self.__formated_value = date
162 162
163 163 return self.__formated_value
164 164
165 165 if format == 'time':
166 166 strList = value.split(':')
167 167 intList = [int(x) for x in strList]
168 168 time = datetime.time(intList[0], intList[1], intList[2])
169 169
170 170 self.__formated_value = time
171 171
172 172 return self.__formated_value
173 173
174 174 if format == 'pairslist':
175 175 '''
176 176 Example:
177 177 value = (0,1),(1,2)
178 178 '''
179 179
180 180 new_value = ast.literal_eval(value)
181 181
182 182 if type(new_value) not in (tuple, list):
183 183 raise ValueError, '%s has to be a tuple or list of pairs' % value
184 184
185 185 if type(new_value[0]) not in (tuple, list):
186 186 if len(new_value) != 2:
187 187 raise ValueError, '%s has to be a tuple or list of pairs' % value
188 188 new_value = [new_value]
189 189
190 190 for thisPair in new_value:
191 191 if len(thisPair) != 2:
192 192 raise ValueError, '%s has to be a tuple or list of pairs' % value
193 193
194 194 self.__formated_value = new_value
195 195
196 196 return self.__formated_value
197 197
198 198 if format == 'multilist':
199 199 '''
200 200 Example:
201 201 value = (0,1,2),(3,4,5)
202 202 '''
203 203 multiList = ast.literal_eval(value)
204 204
205 205 if type(multiList[0]) == int:
206 206 multiList = ast.literal_eval('(' + value + ')')
207 207
208 208 self.__formated_value = multiList
209 209
210 210 return self.__formated_value
211 211
212 212 if format == 'bool':
213 213 value = int(value)
214 214
215 215 if format == 'int':
216 216 value = float(value)
217 217
218 218 format_func = eval(format)
219 219
220 220 self.__formated_value = format_func(value)
221 221
222 222 return self.__formated_value
223 223
224 224 def updateId(self, new_id):
225 225
226 226 self.id = str(new_id)
227 227
228 228 def setup(self, id, name, value, format='str'):
229 229 self.id = str(id)
230 230 self.name = name
231 231 if format == 'obj':
232 232 self.value = value
233 233 else:
234 234 self.value = str(value)
235 235 self.format = str.lower(format)
236 236
237 237 self.getValue()
238 238
239 239 return 1
240 240
241 241 def update(self, name, value, format='str'):
242 242
243 243 self.name = name
244 244 self.value = str(value)
245 245 self.format = format
246 246
247 247 def makeXml(self, opElement):
248 248 if self.name not in ('queue',):
249 249 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 250 parmElement.set('id', str(self.id))
251 251 parmElement.set('name', self.name)
252 252 parmElement.set('value', self.value)
253 253 parmElement.set('format', self.format)
254 254
255 255 def readXml(self, parmElement):
256 256
257 257 self.id = parmElement.get('id')
258 258 self.name = parmElement.get('name')
259 259 self.value = parmElement.get('value')
260 260 self.format = str.lower(parmElement.get('format'))
261 261
262 262 # Compatible with old signal chain version
263 263 if self.format == 'int' and self.name == 'idfigure':
264 264 self.name = 'id'
265 265
266 266 def printattr(self):
267 267
268 268 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
269 269
270 270
271 271 class OperationConf():
272 272
273 273 id = None
274 274 name = None
275 275 priority = None
276 276 type = None
277 277
278 278 parmConfObjList = []
279 279
280 280 ELEMENTNAME = 'Operation'
281 281
282 282 def __init__(self):
283 283
284 284 self.id = '0'
285 285 self.name = None
286 286 self.priority = None
287 287 self.type = 'self'
288 288
289 289 def __getNewId(self):
290 290
291 291 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292 292
293 293 def updateId(self, new_id):
294 294
295 295 self.id = str(new_id)
296 296
297 297 n = 1
298 298 for parmObj in self.parmConfObjList:
299 299
300 300 idParm = str(int(new_id) * 10 + n)
301 301 parmObj.updateId(idParm)
302 302
303 303 n += 1
304 304
305 305 def getElementName(self):
306 306
307 307 return self.ELEMENTNAME
308 308
309 309 def getParameterObjList(self):
310 310
311 311 return self.parmConfObjList
312 312
313 313 def getParameterObj(self, parameterName):
314 314
315 315 for parmConfObj in self.parmConfObjList:
316 316
317 317 if parmConfObj.name != parameterName:
318 318 continue
319 319
320 320 return parmConfObj
321 321
322 322 return None
323 323
324 324 def getParameterObjfromValue(self, parameterValue):
325 325
326 326 for parmConfObj in self.parmConfObjList:
327 327
328 328 if parmConfObj.getValue() != parameterValue:
329 329 continue
330 330
331 331 return parmConfObj.getValue()
332 332
333 333 return None
334 334
335 335 def getParameterValue(self, parameterName):
336 336
337 337 parameterObj = self.getParameterObj(parameterName)
338 338
339 339 # if not parameterObj:
340 340 # return None
341 341
342 342 value = parameterObj.getValue()
343 343
344 344 return value
345 345
346 346 def getKwargs(self):
347 347
348 348 kwargs = {}
349 349
350 350 for parmConfObj in self.parmConfObjList:
351 351 if self.name == 'run' and parmConfObj.name == 'datatype':
352 352 continue
353 353
354 354 kwargs[parmConfObj.name] = parmConfObj.getValue()
355 355
356 356 return kwargs
357 357
358 358 def setup(self, id, name, priority, type):
359 359
360 360 self.id = str(id)
361 361 self.name = name
362 362 self.type = type
363 363 self.priority = priority
364 364
365 365 self.parmConfObjList = []
366 366
367 367 def removeParameters(self):
368 368
369 369 for obj in self.parmConfObjList:
370 370 del obj
371 371
372 372 self.parmConfObjList = []
373 373
374 374 def addParameter(self, name, value, format='str'):
375 375
376 376 if value is None:
377 377 return None
378 378 id = self.__getNewId()
379 379
380 380 parmConfObj = ParameterConf()
381 381 if not parmConfObj.setup(id, name, value, format):
382 382 return None
383 383
384 384 self.parmConfObjList.append(parmConfObj)
385 385
386 386 return parmConfObj
387 387
388 388 def changeParameter(self, name, value, format='str'):
389 389
390 390 parmConfObj = self.getParameterObj(name)
391 391 parmConfObj.update(name, value, format)
392 392
393 393 return parmConfObj
394 394
395 395 def makeXml(self, procUnitElement):
396 396
397 397 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 398 opElement.set('id', str(self.id))
399 399 opElement.set('name', self.name)
400 400 opElement.set('type', self.type)
401 401 opElement.set('priority', str(self.priority))
402 402
403 403 for parmConfObj in self.parmConfObjList:
404 404 parmConfObj.makeXml(opElement)
405 405
406 406 def readXml(self, opElement):
407 407
408 408 self.id = opElement.get('id')
409 409 self.name = opElement.get('name')
410 410 self.type = opElement.get('type')
411 411 self.priority = opElement.get('priority')
412 412
413 413 # Compatible with old signal chain version
414 414 # Use of 'run' method instead 'init'
415 415 if self.type == 'self' and self.name == 'init':
416 416 self.name = 'run'
417 417
418 418 self.parmConfObjList = []
419 419
420 420 parmElementList = opElement.iter(ParameterConf().getElementName())
421 421
422 422 for parmElement in parmElementList:
423 423 parmConfObj = ParameterConf()
424 424 parmConfObj.readXml(parmElement)
425 425
426 426 # Compatible with old signal chain version
427 427 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 428 if self.type != 'self' and self.name == 'Plot':
429 429 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 430 self.name = parmConfObj.value
431 431 continue
432 432
433 433 self.parmConfObjList.append(parmConfObj)
434 434
435 435 def printattr(self):
436 436
437 437 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 438 self.id,
439 439 self.name,
440 440 self.type,
441 441 self.priority)
442 442
443 443 for parmConfObj in self.parmConfObjList:
444 444 parmConfObj.printattr()
445 445
446 446 def createObject(self, plotter_queue=None):
447 447
448 448 if self.type == 'self':
449 449 raise ValueError, 'This operation type cannot be created'
450 450
451 451 if self.type == 'plotter':
452 452 if not plotter_queue:
453 453 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
454 454
455 455 opObj = Plotter(self.name, plotter_queue)
456 456
457 457 if self.type == 'external' or self.type == 'other':
458 458
459 459 className = eval(self.name)
460 460 kwargs = self.getKwargs()
461 461
462 462 opObj = className(**kwargs)
463 463
464 464 return opObj
465 465
466 466
467 467 class ProcUnitConf():
468 468
469 469 id = None
470 470 name = None
471 471 datatype = None
472 472 inputId = None
473 473 parentId = None
474 474
475 475 opConfObjList = []
476 476
477 477 procUnitObj = None
478 478 opObjList = []
479 479
480 480 ELEMENTNAME = 'ProcUnit'
481 481
482 482 def __init__(self):
483 483
484 484 self.id = None
485 485 self.datatype = None
486 486 self.name = None
487 487 self.inputId = None
488 488
489 489 self.opConfObjList = []
490 490
491 491 self.procUnitObj = None
492 492 self.opObjDict = {}
493 493
494 494 def __getPriority(self):
495 495
496 496 return len(self.opConfObjList) + 1
497 497
498 498 def __getNewId(self):
499 499
500 500 return int(self.id) * 10 + len(self.opConfObjList) + 1
501 501
502 502 def getElementName(self):
503 503
504 504 return self.ELEMENTNAME
505 505
506 506 def getId(self):
507 507
508 508 return self.id
509 509
510 510 def updateId(self, new_id, parentId=parentId):
511 511
512 512 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 513 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514 514
515 515 # If this proc unit has not inputs
516 516 if self.inputId == '0':
517 517 new_inputId = 0
518 518
519 519 n = 1
520 520 for opConfObj in self.opConfObjList:
521 521
522 522 idOp = str(int(new_id) * 10 + n)
523 523 opConfObj.updateId(idOp)
524 524
525 525 n += 1
526 526
527 527 self.parentId = str(parentId)
528 528 self.id = str(new_id)
529 529 self.inputId = str(new_inputId)
530 530
531 531 def getInputId(self):
532 532
533 533 return self.inputId
534 534
535 535 def getOperationObjList(self):
536 536
537 537 return self.opConfObjList
538 538
539 539 def getOperationObj(self, name=None):
540 540
541 541 for opConfObj in self.opConfObjList:
542 542
543 543 if opConfObj.name != name:
544 544 continue
545 545
546 546 return opConfObj
547 547
548 548 return None
549 549
550 550 def getOpObjfromParamValue(self, value=None):
551 551
552 552 for opConfObj in self.opConfObjList:
553 553 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 554 continue
555 555 return opConfObj
556 556 return None
557 557
558 558 def getProcUnitObj(self):
559 559
560 560 return self.procUnitObj
561 561
562 562 def setup(self, id, name, datatype, inputId, parentId=None):
563 563
564 564 # Compatible with old signal chain version
565 565 if datatype == None and name == None:
566 566 raise ValueError, 'datatype or name should be defined'
567 567
568 568 if name == None:
569 569 if 'Proc' in datatype:
570 570 name = datatype
571 571 else:
572 572 name = '%sProc' % (datatype)
573 573
574 574 if datatype == None:
575 575 datatype = name.replace('Proc', '')
576 576
577 577 self.id = str(id)
578 578 self.name = name
579 579 self.datatype = datatype
580 580 self.inputId = inputId
581 581 self.parentId = parentId
582 582
583 583 self.opConfObjList = []
584 584
585 585 self.addOperation(name='run', optype='self')
586 586
587 587 def removeOperations(self):
588 588
589 589 for obj in self.opConfObjList:
590 590 del obj
591 591
592 592 self.opConfObjList = []
593 593 self.addOperation(name='run')
594 594
595 595 def addParameter(self, **kwargs):
596 596 '''
597 597 Add parameters to 'run' operation
598 598 '''
599 599 opObj = self.opConfObjList[0]
600 600
601 601 opObj.addParameter(**kwargs)
602 602
603 603 return opObj
604 604
605 605 def addOperation(self, name, optype='self'):
606 606
607 607 id = self.__getNewId()
608 608 priority = self.__getPriority()
609 609
610 610 opConfObj = OperationConf()
611 611 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 612
613 613 self.opConfObjList.append(opConfObj)
614 614
615 615 return opConfObj
616 616
617 617 def makeXml(self, projectElement):
618 618
619 619 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 620 procUnitElement.set('id', str(self.id))
621 621 procUnitElement.set('name', self.name)
622 622 procUnitElement.set('datatype', self.datatype)
623 623 procUnitElement.set('inputId', str(self.inputId))
624 624
625 625 for opConfObj in self.opConfObjList:
626 626 opConfObj.makeXml(procUnitElement)
627 627
628 628 def readXml(self, upElement):
629 629
630 630 self.id = upElement.get('id')
631 631 self.name = upElement.get('name')
632 632 self.datatype = upElement.get('datatype')
633 633 self.inputId = upElement.get('inputId')
634 634
635 635 if self.ELEMENTNAME == 'ReadUnit':
636 636 self.datatype = self.datatype.replace('Reader', '')
637 637
638 638 if self.ELEMENTNAME == 'ProcUnit':
639 639 self.datatype = self.datatype.replace('Proc', '')
640 640
641 641 if self.inputId == 'None':
642 642 self.inputId = '0'
643 643
644 644 self.opConfObjList = []
645 645
646 646 opElementList = upElement.iter(OperationConf().getElementName())
647 647
648 648 for opElement in opElementList:
649 649 opConfObj = OperationConf()
650 650 opConfObj.readXml(opElement)
651 651 self.opConfObjList.append(opConfObj)
652 652
653 653 def printattr(self):
654 654
655 655 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 656 self.id,
657 657 self.name,
658 658 self.datatype,
659 659 self.inputId)
660 660
661 661 for opConfObj in self.opConfObjList:
662 662 opConfObj.printattr()
663 663
664 664 def getKwargs(self):
665 665
666 666 opObj = self.opConfObjList[0]
667 667 kwargs = opObj.getKwargs()
668 668
669 669 return kwargs
670 670
671 671 def createObjects(self, plotter_queue=None):
672 672
673 673 className = eval(self.name)
674 674 kwargs = self.getKwargs()
675 675 procUnitObj = className(**kwargs)
676 676
677 677 for opConfObj in self.opConfObjList:
678 678
679 679 if opConfObj.type == 'self' and self.name == 'run':
680 680 continue
681 681 elif opConfObj.type == 'self':
682 682 procUnitObj.addOperationKwargs(
683 683 opConfObj.id, **opConfObj.getKwargs())
684 684 continue
685 685
686 686 opObj = opConfObj.createObject(plotter_queue)
687 687
688 688 self.opObjDict[opConfObj.id] = opObj
689 689
690 690 procUnitObj.addOperation(opObj, opConfObj.id)
691 691
692 692 self.procUnitObj = procUnitObj
693 693
694 694 return procUnitObj
695 695
696 696 def run(self):
697 697
698 698 is_ok = False
699 699
700 700 for opConfObj in self.opConfObjList:
701 701
702 702 kwargs = {}
703 703 for parmConfObj in opConfObj.getParameterObjList():
704 704 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 705 continue
706 706
707 707 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 708
709 709 sts = self.procUnitObj.call(opType=opConfObj.type,
710 710 opName=opConfObj.name,
711 711 opId=opConfObj.id)
712 712
713 713 is_ok = is_ok or sts
714 714
715 715 return is_ok
716 716
717 717 def close(self):
718 718
719 719 for opConfObj in self.opConfObjList:
720 720 if opConfObj.type == 'self':
721 721 continue
722 722
723 723 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 724 opObj.close()
725 725
726 726 self.procUnitObj.close()
727 727
728 728 return
729 729
730 730
731 731 class ReadUnitConf(ProcUnitConf):
732 732
733 733 path = None
734 734 startDate = None
735 735 endDate = None
736 736 startTime = None
737 737 endTime = None
738 738
739 739 ELEMENTNAME = 'ReadUnit'
740 740
741 741 def __init__(self):
742 742
743 743 self.id = None
744 744 self.datatype = None
745 745 self.name = None
746 746 self.inputId = None
747 747
748 748 self.parentId = None
749 749
750 750 self.opConfObjList = []
751 751 self.opObjList = []
752 752
753 753 def getElementName(self):
754 754
755 755 return self.ELEMENTNAME
756 756
757 757 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 758 startTime='', endTime='', parentId=None, server=None, **kwargs):
759 759
760 760 # Compatible with old signal chain version
761 761 if datatype == None and name == None:
762 762 raise ValueError, 'datatype or name should be defined'
763 763 if name == None:
764 764 if 'Reader' in datatype:
765 765 name = datatype
766 766 datatype = name.replace('Reader','')
767 767 else:
768 768 name = '{}Reader'.format(datatype)
769 769 if datatype == None:
770 770 if 'Reader' in name:
771 771 datatype = name.replace('Reader','')
772 772 else:
773 773 datatype = name
774 774 name = '{}Reader'.format(name)
775 775
776 776 self.id = id
777 777 self.name = name
778 778 self.datatype = datatype
779 779 if path != '':
780 780 self.path = os.path.abspath(path)
781 781 self.startDate = startDate
782 782 self.endDate = endDate
783 783 self.startTime = startTime
784 784 self.endTime = endTime
785 785 self.inputId = '0'
786 786 self.parentId = parentId
787 787 self.server = server
788 788 self.addRunOperation(**kwargs)
789 789
790 790 def update(self, **kwargs):
791 791
792 792 if 'datatype' in kwargs:
793 793 datatype = kwargs.pop('datatype')
794 794 if 'Reader' in datatype:
795 795 self.name = datatype
796 796 else:
797 797 self.name = '%sReader' % (datatype)
798 798 self.datatype = self.name.replace('Reader', '')
799 799
800 800 attrs = ('path', 'startDate', 'endDate',
801 801 'startTime', 'endTime', 'parentId')
802 802
803 803 for attr in attrs:
804 804 if attr in kwargs:
805 805 setattr(self, attr, kwargs.pop(attr))
806 806
807 807 self.inputId = '0'
808 808 self.updateRunOperation(**kwargs)
809 809
810 810 def removeOperations(self):
811 811
812 812 for obj in self.opConfObjList:
813 813 del obj
814 814
815 815 self.opConfObjList = []
816 816
817 817 def addRunOperation(self, **kwargs):
818 818
819 819 opObj = self.addOperation(name='run', optype='self')
820 820
821 821 if self.server is None:
822 822 opObj.addParameter(
823 823 name='datatype', value=self.datatype, format='str')
824 824 opObj.addParameter(name='path', value=self.path, format='str')
825 825 opObj.addParameter(
826 826 name='startDate', value=self.startDate, format='date')
827 827 opObj.addParameter(
828 828 name='endDate', value=self.endDate, format='date')
829 829 opObj.addParameter(
830 830 name='startTime', value=self.startTime, format='time')
831 831 opObj.addParameter(
832 832 name='endTime', value=self.endTime, format='time')
833 833
834 834 for key, value in kwargs.items():
835 835 opObj.addParameter(name=key, value=value,
836 836 format=type(value).__name__)
837 837 else:
838 838 opObj.addParameter(name='server', value=self.server, format='str')
839 839
840 840 return opObj
841 841
842 842 def updateRunOperation(self, **kwargs):
843 843
844 844 opObj = self.getOperationObj(name='run')
845 845 opObj.removeParameters()
846 846
847 847 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 848 opObj.addParameter(name='path', value=self.path, format='str')
849 849 opObj.addParameter(
850 850 name='startDate', value=self.startDate, format='date')
851 851 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 852 opObj.addParameter(
853 853 name='startTime', value=self.startTime, format='time')
854 854 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855 855
856 856 for key, value in kwargs.items():
857 857 opObj.addParameter(name=key, value=value,
858 858 format=type(value).__name__)
859 859
860 860 return opObj
861 861
862 862 def readXml(self, upElement):
863 863
864 864 self.id = upElement.get('id')
865 865 self.name = upElement.get('name')
866 866 self.datatype = upElement.get('datatype')
867 867 self.inputId = upElement.get('inputId')
868 868
869 869 if self.ELEMENTNAME == 'ReadUnit':
870 870 self.datatype = self.datatype.replace('Reader', '')
871 871
872 872 if self.inputId == 'None':
873 873 self.inputId = '0'
874 874
875 875 self.opConfObjList = []
876 876
877 877 opElementList = upElement.iter(OperationConf().getElementName())
878 878
879 879 for opElement in opElementList:
880 880 opConfObj = OperationConf()
881 881 opConfObj.readXml(opElement)
882 882 self.opConfObjList.append(opConfObj)
883 883
884 884 if opConfObj.name == 'run':
885 885 self.path = opConfObj.getParameterValue('path')
886 886 self.startDate = opConfObj.getParameterValue('startDate')
887 887 self.endDate = opConfObj.getParameterValue('endDate')
888 888 self.startTime = opConfObj.getParameterValue('startTime')
889 889 self.endTime = opConfObj.getParameterValue('endTime')
890 890
891 891
892 892 class Project(Process):
893 893
894 894 id = None
895 895 # name = None
896 896 description = None
897 897 filename = None
898 898
899 899 procUnitConfObjDict = None
900 900
901 901 ELEMENTNAME = 'Project'
902 902
903 903 plotterQueue = None
904 904
905 905 def __init__(self, plotter_queue=None):
906 906
907 907 Process.__init__(self)
908 908 self.id = None
909 909 self.description = None
910 910 self.email = None
911 self.alarm = [0]
911 self.alarm = None
912 912 self.plotterQueue = plotter_queue
913 913 self.procUnitConfObjDict = {}
914 914
915 915 def __getNewId(self):
916 916
917 917 idList = self.procUnitConfObjDict.keys()
918 918
919 919 id = int(self.id) * 10
920 920
921 921 while True:
922 922 id += 1
923 923
924 924 if str(id) in idList:
925 925 continue
926 926
927 927 break
928 928
929 929 return str(id)
930 930
931 931 def getElementName(self):
932 932
933 933 return self.ELEMENTNAME
934 934
935 935 def getId(self):
936 936
937 937 return self.id
938 938
939 939 def updateId(self, new_id):
940 940
941 941 self.id = str(new_id)
942 942
943 943 keyList = self.procUnitConfObjDict.keys()
944 944 keyList.sort()
945 945
946 946 n = 1
947 947 newProcUnitConfObjDict = {}
948 948
949 949 for procKey in keyList:
950 950
951 951 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 952 idProcUnit = str(int(self.id) * 10 + n)
953 953 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 954 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
955 955 n += 1
956 956
957 957 self.procUnitConfObjDict = newProcUnitConfObjDict
958 958
959 def setup(self, id, name='', description='', email=None, alarm=[0]):
959 def setup(self, id, name='', description='', email=None, alarm=[3]):
960 960
961 961 print
962 962 print '*' * 60
963 963 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
964 964 print '*' * 60
965 965 print
966 966 self.id = str(id)
967 967 self.description = description
968 968 self.email = email
969 969 self.alarm = alarm
970 970
971 971 def update(self, **kwargs):
972 972
973 973 for key, value in kwargs.items():
974 974 setattr(self, key, value)
975 975
976 976 def clone(self):
977 977
978 978 p = Project()
979 979 p.procUnitConfObjDict = self.procUnitConfObjDict
980 980 return p
981 981
982 982 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
983 983
984 984 if id is None:
985 985 idReadUnit = self.__getNewId()
986 986 else:
987 987 idReadUnit = str(id)
988 988
989 989 readUnitConfObj = ReadUnitConf()
990 990 readUnitConfObj.setup(idReadUnit, name, datatype,
991 991 parentId=self.id, **kwargs)
992 992
993 993 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
994 994
995 995 return readUnitConfObj
996 996
997 997 def addProcUnit(self, inputId='0', datatype=None, name=None):
998 998
999 999 idProcUnit = self.__getNewId()
1000 1000
1001 1001 procUnitConfObj = ProcUnitConf()
1002 1002 procUnitConfObj.setup(idProcUnit, name, datatype,
1003 1003 inputId, parentId=self.id)
1004 1004
1005 1005 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 1006
1007 1007 return procUnitConfObj
1008 1008
1009 1009 def removeProcUnit(self, id):
1010 1010
1011 1011 if id in self.procUnitConfObjDict.keys():
1012 1012 self.procUnitConfObjDict.pop(id)
1013 1013
1014 1014 def getReadUnitId(self):
1015 1015
1016 1016 readUnitConfObj = self.getReadUnitObj()
1017 1017
1018 1018 return readUnitConfObj.id
1019 1019
1020 1020 def getReadUnitObj(self):
1021 1021
1022 1022 for obj in self.procUnitConfObjDict.values():
1023 1023 if obj.getElementName() == 'ReadUnit':
1024 1024 return obj
1025 1025
1026 1026 return None
1027 1027
1028 1028 def getProcUnitObj(self, id=None, name=None):
1029 1029
1030 1030 if id != None:
1031 1031 return self.procUnitConfObjDict[id]
1032 1032
1033 1033 if name != None:
1034 1034 return self.getProcUnitObjByName(name)
1035 1035
1036 1036 return None
1037 1037
1038 1038 def getProcUnitObjByName(self, name):
1039 1039
1040 1040 for obj in self.procUnitConfObjDict.values():
1041 1041 if obj.name == name:
1042 1042 return obj
1043 1043
1044 1044 return None
1045 1045
1046 1046 def procUnitItems(self):
1047 1047
1048 1048 return self.procUnitConfObjDict.items()
1049 1049
1050 1050 def makeXml(self):
1051 1051
1052 1052 projectElement = Element('Project')
1053 1053 projectElement.set('id', str(self.id))
1054 1054 projectElement.set('name', self.name)
1055 1055 projectElement.set('description', self.description)
1056 1056
1057 1057 for procUnitConfObj in self.procUnitConfObjDict.values():
1058 1058 procUnitConfObj.makeXml(projectElement)
1059 1059
1060 1060 self.projectElement = projectElement
1061 1061
1062 1062 def writeXml(self, filename=None):
1063 1063
1064 1064 if filename == None:
1065 1065 if self.filename:
1066 1066 filename = self.filename
1067 1067 else:
1068 1068 filename = 'schain.xml'
1069 1069
1070 1070 if not filename:
1071 1071 print 'filename has not been defined. Use setFilename(filename) for do it.'
1072 1072 return 0
1073 1073
1074 1074 abs_file = os.path.abspath(filename)
1075 1075
1076 1076 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 1077 print 'No write permission on %s' % os.path.dirname(abs_file)
1078 1078 return 0
1079 1079
1080 1080 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 1081 print 'File %s already exists and it could not be overwriten' % abs_file
1082 1082 return 0
1083 1083
1084 1084 self.makeXml()
1085 1085
1086 1086 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 1087
1088 1088 self.filename = abs_file
1089 1089
1090 1090 return 1
1091 1091
1092 1092 def readXml(self, filename=None):
1093 1093
1094 1094 if not filename:
1095 1095 print 'filename is not defined'
1096 1096 return 0
1097 1097
1098 1098 abs_file = os.path.abspath(filename)
1099 1099
1100 1100 if not os.path.isfile(abs_file):
1101 1101 print '%s file does not exist' % abs_file
1102 1102 return 0
1103 1103
1104 1104 self.projectElement = None
1105 1105 self.procUnitConfObjDict = {}
1106 1106
1107 1107 try:
1108 1108 self.projectElement = ElementTree().parse(abs_file)
1109 1109 except:
1110 1110 print 'Error reading %s, verify file format' % filename
1111 1111 return 0
1112 1112
1113 1113 self.project = self.projectElement.tag
1114 1114
1115 1115 self.id = self.projectElement.get('id')
1116 1116 self.name = self.projectElement.get('name')
1117 1117 self.description = self.projectElement.get('description')
1118 1118
1119 1119 readUnitElementList = self.projectElement.iter(
1120 1120 ReadUnitConf().getElementName())
1121 1121
1122 1122 for readUnitElement in readUnitElementList:
1123 1123 readUnitConfObj = ReadUnitConf()
1124 1124 readUnitConfObj.readXml(readUnitElement)
1125 1125
1126 1126 if readUnitConfObj.parentId == None:
1127 1127 readUnitConfObj.parentId = self.id
1128 1128
1129 1129 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130 1130
1131 1131 procUnitElementList = self.projectElement.iter(
1132 1132 ProcUnitConf().getElementName())
1133 1133
1134 1134 for procUnitElement in procUnitElementList:
1135 1135 procUnitConfObj = ProcUnitConf()
1136 1136 procUnitConfObj.readXml(procUnitElement)
1137 1137
1138 1138 if procUnitConfObj.parentId == None:
1139 1139 procUnitConfObj.parentId = self.id
1140 1140
1141 1141 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1142 1142
1143 1143 self.filename = abs_file
1144 1144
1145 1145 return 1
1146 1146
1147 1147 def printattr(self):
1148 1148
1149 1149 print 'Project[%s]: name = %s, description = %s' % (self.id,
1150 1150 self.name,
1151 1151 self.description)
1152 1152
1153 1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1154 1154 procUnitConfObj.printattr()
1155 1155
1156 1156 def createObjects(self):
1157 1157
1158 1158 for procUnitConfObj in self.procUnitConfObjDict.values():
1159 1159 procUnitConfObj.createObjects(self.plotterQueue)
1160 1160
1161 1161 def __connect(self, objIN, thisObj):
1162 1162
1163 1163 thisObj.setInput(objIN.getOutputObj())
1164 1164
1165 1165 def connectObjects(self):
1166 1166
1167 1167 for thisPUConfObj in self.procUnitConfObjDict.values():
1168 1168
1169 1169 inputId = thisPUConfObj.getInputId()
1170 1170
1171 1171 if int(inputId) == 0:
1172 1172 continue
1173 1173
1174 1174 # Get input object
1175 1175 puConfINObj = self.procUnitConfObjDict[inputId]
1176 1176 puObjIN = puConfINObj.getProcUnitObj()
1177 1177
1178 1178 # Get current object
1179 1179 thisPUObj = thisPUConfObj.getProcUnitObj()
1180 1180
1181 1181 self.__connect(puObjIN, thisPUObj)
1182 1182
1183 def __handleError(self, procUnitConfObj, modes=None):
1183 def __handleError(self, procUnitConfObj, modes=None, stdout=True):
1184 1184
1185 1185 import socket
1186 1186
1187 1187 if modes is None:
1188 1188 modes = self.alarm
1189 1189
1190 1190 err = traceback.format_exception(sys.exc_info()[0],
1191 1191 sys.exc_info()[1],
1192 1192 sys.exc_info()[2])
1193 1193
1194 1194 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1195 1195
1196 1196 message = ''.join(err)
1197 1197
1198 sys.stderr.write(message)
1198 if stdout:
1199 sys.stderr.write(message)
1199 1200
1200 1201 subject = 'SChain v%s: Error running %s\n' % (
1201 1202 schainpy.__version__, procUnitConfObj.name)
1202 1203
1203 1204 subtitle = '%s: %s\n' % (
1204 1205 procUnitConfObj.getElementName(), procUnitConfObj.name)
1205 1206 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1206 1207 socket.gethostname())
1207 1208 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1208 1209 subtitle += 'Configuration file: %s\n' % self.filename
1209 1210 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1210 1211
1211 1212 readUnitConfObj = self.getReadUnitObj()
1212 1213 if readUnitConfObj:
1213 1214 subtitle += '\nInput parameters:\n'
1214 1215 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1215 1216 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1216 1217 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1217 1218 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1218 1219 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1219 1220 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1220 1221
1221 1222 a = Alarm(
1222 1223 modes=modes,
1223 1224 email=self.email,
1224 1225 message=message,
1225 1226 subject=subject,
1226 1227 subtitle=subtitle,
1227 1228 filename=self.filename
1228 1229 )
1229 1230
1230 a.start()
1231 return a
1231 1232
1232 1233 def isPaused(self):
1233 1234 return 0
1234 1235
1235 1236 def isStopped(self):
1236 1237 return 0
1237 1238
1238 1239 def runController(self):
1239 1240 '''
1240 1241 returns 0 when this process has been stopped, 1 otherwise
1241 1242 '''
1242 1243
1243 1244 if self.isPaused():
1244 1245 print 'Process suspended'
1245 1246
1246 1247 while True:
1247 1248 time.sleep(0.1)
1248 1249
1249 1250 if not self.isPaused():
1250 1251 break
1251 1252
1252 1253 if self.isStopped():
1253 1254 break
1254 1255
1255 1256 print 'Process reinitialized'
1256 1257
1257 1258 if self.isStopped():
1258 1259 print 'Process stopped'
1259 1260 return 0
1260 1261
1261 1262 return 1
1262 1263
1263 1264 def setFilename(self, filename):
1264 1265
1265 1266 self.filename = filename
1266 1267
1267 1268 def setPlotterQueue(self, plotter_queue):
1268 1269
1269 1270 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270 1271
1271 1272 def getPlotterQueue(self):
1272 1273
1273 1274 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1274 1275
1275 1276 def useExternalPlotter(self):
1276 1277
1277 1278 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1278 1279
1279 1280 def run(self):
1280 1281
1281 1282 log.success('Starting {}'.format(self.name))
1282 1283 self.start_time = time.time()
1283 1284 self.createObjects()
1284 1285 self.connectObjects()
1285 1286
1286 1287 keyList = self.procUnitConfObjDict.keys()
1287 1288 keyList.sort()
1288 1289
1290 err = None
1291
1289 1292 while(True):
1290 1293
1291 1294 is_ok = False
1292 1295
1293 1296 for procKey in keyList:
1294 1297
1295 1298 procUnitConfObj = self.procUnitConfObjDict[procKey]
1296 1299
1297 1300 try:
1298 1301 sts = procUnitConfObj.run()
1299 1302 is_ok = is_ok or sts
1300 1303 except SchainWarning:
1301 self.__handleError(procUnitConfObj, modes=[2, 3])
1304 err = self.__handleError(procUnitConfObj, modes=[2, 3], stdout=False)
1302 1305 except KeyboardInterrupt:
1303 1306 is_ok = False
1304 1307 break
1305 1308 except ValueError, e:
1306 1309 time.sleep(0.5)
1307 self.__handleError(procUnitConfObj)
1310 err = self.__handleError(procUnitConfObj)
1308 1311 is_ok = False
1309 1312 break
1310 1313 except:
1311 1314 time.sleep(0.5)
1312 self.__handleError(procUnitConfObj)
1315 err = self.__handleError(procUnitConfObj)
1313 1316 is_ok = False
1314 1317 break
1315 1318
1316 1319 # If every process unit finished so end process
1317 1320 if not(is_ok):
1318 1321 break
1319 1322
1320 1323 if not self.runController():
1321 1324 break
1322 1325
1323 1326 # Closing every process
1324 1327 for procKey in keyList:
1325 1328 procUnitConfObj = self.procUnitConfObjDict[procKey]
1326 1329 procUnitConfObj.close()
1327 1330
1331 if err is not None:
1332 err.start()
1333 # err.join()
1334
1328 1335 log.success('{} finished (time: {}s)'.format(
1329 1336 self.name,
1330 1337 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,971 +1,978
1 1
2 2 import os
3 3 import time
4 4 import glob
5 5 import datetime
6 6 from multiprocessing import Process
7 7
8 8 import zmq
9 9 import numpy
10 10 import matplotlib
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
14 14
15 15 from schainpy.model.proc.jroproc_base import Operation
16 16 from schainpy.utils import log
17 17
18 18 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
19 19 blu_values = matplotlib.pyplot.get_cmap(
20 20 'seismic_r', 20)(numpy.arange(20))[10:15]
21 21 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
22 22 'jro', numpy.vstack((blu_values, jet_values)))
23 23 matplotlib.pyplot.register_cmap(cmap=ncmap)
24 24
25 25 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis', 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
26 26
27 27
28 28 def figpause(interval):
29 29 backend = plt.rcParams['backend']
30 30 if backend in matplotlib.rcsetup.interactive_bk:
31 31 figManager = matplotlib._pylab_helpers.Gcf.get_active()
32 32 if figManager is not None:
33 33 canvas = figManager.canvas
34 34 if canvas.figure.stale:
35 35 canvas.draw()
36 canvas.start_event_loop(interval)
36 try:
37 canvas.start_event_loop(interval)
38 except:
39 pass
37 40 return
38 41
39 42 def popup(message):
40 fig = plt.figure(figsize=(12, 8), facecolor='r')
41 fig.text(0.5, 0.5, message, ha='center', va='center', size='20', weight='heavy', color='w')
43 '''
44 '''
45
46 fig = plt.figure(figsize=(12, 8), facecolor='r')
47 text = '\n'.join([s.strip() for s in message.split(':')])
48 fig.text(0.01, 0.5, text, ha='left', va='center', size='20', weight='heavy', color='w')
42 49 fig.show()
43 50 figpause(1000)
44 51
45 52
46 53 class PlotData(Operation, Process):
47 54 '''
48 55 Base class for Schain plotting operations
49 56 '''
50 57
51 58 CODE = 'Figure'
52 59 colormap = 'jro'
53 60 bgcolor = 'white'
54 61 CONFLATE = False
55 62 __missing = 1E30
56 63
57 64 __attrs__ = ['show', 'save', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
58 65 'zlimits', 'xlabel', 'ylabel', 'xaxis','cb_label', 'title',
59 66 'colorbar', 'bgcolor', 'width', 'height', 'localtime', 'oneFigure',
60 67 'showprofile', 'decimation']
61 68
62 69 def __init__(self, **kwargs):
63 70
64 71 Operation.__init__(self, plot=True, **kwargs)
65 72 Process.__init__(self)
66 73
67 74 self.kwargs['code'] = self.CODE
68 75 self.mp = False
69 76 self.data = None
70 77 self.isConfig = False
71 78 self.figures = []
72 79 self.axes = []
73 80 self.cb_axes = []
74 81 self.localtime = kwargs.pop('localtime', True)
75 82 self.show = kwargs.get('show', True)
76 83 self.save = kwargs.get('save', False)
77 84 self.colormap = kwargs.get('colormap', self.colormap)
78 85 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
79 86 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
80 87 self.colormaps = kwargs.get('colormaps', None)
81 88 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
82 89 self.showprofile = kwargs.get('showprofile', False)
83 90 self.title = kwargs.get('wintitle', self.CODE.upper())
84 91 self.cb_label = kwargs.get('cb_label', None)
85 92 self.cb_labels = kwargs.get('cb_labels', None)
86 93 self.xaxis = kwargs.get('xaxis', 'frequency')
87 94 self.zmin = kwargs.get('zmin', None)
88 95 self.zmax = kwargs.get('zmax', None)
89 96 self.zlimits = kwargs.get('zlimits', None)
90 97 self.xmin = kwargs.get('xmin', None)
91 98 self.xmax = kwargs.get('xmax', None)
92 99 self.xrange = kwargs.get('xrange', 24)
93 100 self.ymin = kwargs.get('ymin', None)
94 101 self.ymax = kwargs.get('ymax', None)
95 102 self.xlabel = kwargs.get('xlabel', None)
96 103 self.decimation = kwargs.get('decimation', None)
97 104 self.showSNR = kwargs.get('showSNR', False)
98 105 self.oneFigure = kwargs.get('oneFigure', True)
99 106 self.width = kwargs.get('width', None)
100 107 self.height = kwargs.get('height', None)
101 108 self.colorbar = kwargs.get('colorbar', True)
102 109 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
103 110 self.titles = kwargs.get('titles', [])
104 111 self.polar = False
105 112
106 113 def __fmtTime(self, x, pos):
107 114 '''
108 115 '''
109 116
110 117 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
111 118
112 119 def __setup(self):
113 120 '''
114 121 Common setup for all figures, here figures and axes are created
115 122 '''
116 123
117 124 if self.CODE not in self.data:
118 125 raise ValueError(log.error('Missing data for {}'.format(self.CODE),
119 126 self.name))
120 127
121 128 self.setup()
122 129
123 130 self.time_label = 'LT' if self.localtime else 'UTC'
124 131 if self.data.localtime:
125 132 self.getDateTime = datetime.datetime.fromtimestamp
126 133 else:
127 134 self.getDateTime = datetime.datetime.utcfromtimestamp
128 135
129 136 if self.width is None:
130 137 self.width = 8
131 138
132 139 self.figures = []
133 140 self.axes = []
134 141 self.cb_axes = []
135 142 self.pf_axes = []
136 143 self.cmaps = []
137 144
138 145 size = '15%' if self.ncols == 1 else '30%'
139 146 pad = '4%' if self.ncols == 1 else '8%'
140 147
141 148 if self.oneFigure:
142 149 if self.height is None:
143 150 self.height = 1.4 * self.nrows + 1
144 151 fig = plt.figure(figsize=(self.width, self.height),
145 152 edgecolor='k',
146 153 facecolor='w')
147 154 self.figures.append(fig)
148 155 for n in range(self.nplots):
149 156 ax = fig.add_subplot(self.nrows, self.ncols,
150 157 n + 1, polar=self.polar)
151 158 ax.tick_params(labelsize=8)
152 159 ax.firsttime = True
153 160 ax.index = 0
154 161 ax.press = None
155 162 self.axes.append(ax)
156 163 if self.showprofile:
157 164 cax = self.__add_axes(ax, size=size, pad=pad)
158 165 cax.tick_params(labelsize=8)
159 166 self.pf_axes.append(cax)
160 167 else:
161 168 if self.height is None:
162 169 self.height = 3
163 170 for n in range(self.nplots):
164 171 fig = plt.figure(figsize=(self.width, self.height),
165 172 edgecolor='k',
166 173 facecolor='w')
167 174 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
168 175 ax.tick_params(labelsize=8)
169 176 ax.firsttime = True
170 177 ax.index = 0
171 178 ax.press = None
172 179 self.figures.append(fig)
173 180 self.axes.append(ax)
174 181 if self.showprofile:
175 182 cax = self.__add_axes(ax, size=size, pad=pad)
176 183 cax.tick_params(labelsize=8)
177 184 self.pf_axes.append(cax)
178 185
179 186 for n in range(self.nrows):
180 187 if self.colormaps is not None:
181 188 cmap = plt.get_cmap(self.colormaps[n])
182 189 else:
183 190 cmap = plt.get_cmap(self.colormap)
184 191 cmap.set_bad(self.bgcolor, 1.)
185 192 self.cmaps.append(cmap)
186 193
187 194 for fig in self.figures:
188 195 fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
189 196 fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
190 197 fig.canvas.mpl_connect('button_press_event', self.onBtnPress)
191 198 fig.canvas.mpl_connect('motion_notify_event', self.onMotion)
192 199 fig.canvas.mpl_connect('button_release_event', self.onBtnRelease)
193 200 if self.show:
194 201 fig.show()
195 202
196 203 def OnKeyPress(self, event):
197 204 '''
198 205 Event for pressing keys (up, down) change colormap
199 206 '''
200 207 ax = event.inaxes
201 208 if ax in self.axes:
202 209 if event.key == 'down':
203 210 ax.index += 1
204 211 elif event.key == 'up':
205 212 ax.index -= 1
206 213 if ax.index < 0:
207 214 ax.index = len(CMAPS) - 1
208 215 elif ax.index == len(CMAPS):
209 216 ax.index = 0
210 217 cmap = CMAPS[ax.index]
211 218 ax.cbar.set_cmap(cmap)
212 219 ax.cbar.draw_all()
213 220 ax.plt.set_cmap(cmap)
214 221 ax.cbar.patch.figure.canvas.draw()
215 222 self.colormap = cmap.name
216 223
217 224 def OnBtnScroll(self, event):
218 225 '''
219 226 Event for scrolling, scale figure
220 227 '''
221 228 cb_ax = event.inaxes
222 229 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
223 230 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
224 231 pt = ax.cbar.ax.bbox.get_points()[:, 1]
225 232 nrm = ax.cbar.norm
226 233 vmin, vmax, p0, p1, pS = (
227 234 nrm.vmin, nrm.vmax, pt[0], pt[1], event.y)
228 235 scale = 2 if event.step == 1 else 0.5
229 236 point = vmin + (vmax - vmin) / (p1 - p0) * (pS - p0)
230 237 ax.cbar.norm.vmin = point - scale * (point - vmin)
231 238 ax.cbar.norm.vmax = point - scale * (point - vmax)
232 239 ax.plt.set_norm(ax.cbar.norm)
233 240 ax.cbar.draw_all()
234 241 ax.cbar.patch.figure.canvas.draw()
235 242
236 243 def onBtnPress(self, event):
237 244 '''
238 245 Event for mouse button press
239 246 '''
240 247 cb_ax = event.inaxes
241 248 if cb_ax is None:
242 249 return
243 250
244 251 if cb_ax in [ax.cbar.ax for ax in self.axes if ax.cbar]:
245 252 cb_ax.press = event.x, event.y
246 253 else:
247 254 cb_ax.press = None
248 255
249 256 def onMotion(self, event):
250 257 '''
251 258 Event for move inside colorbar
252 259 '''
253 260 cb_ax = event.inaxes
254 261 if cb_ax is None:
255 262 return
256 263 if cb_ax not in [ax.cbar.ax for ax in self.axes if ax.cbar]:
257 264 return
258 265 if cb_ax.press is None:
259 266 return
260 267
261 268 ax = [ax for ax in self.axes if cb_ax == ax.cbar.ax][0]
262 269 xprev, yprev = cb_ax.press
263 270 dx = event.x - xprev
264 271 dy = event.y - yprev
265 272 cb_ax.press = event.x, event.y
266 273 scale = ax.cbar.norm.vmax - ax.cbar.norm.vmin
267 274 perc = 0.03
268 275
269 276 if event.button == 1:
270 277 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
271 278 ax.cbar.norm.vmax -= (perc * scale) * numpy.sign(dy)
272 279 elif event.button == 3:
273 280 ax.cbar.norm.vmin -= (perc * scale) * numpy.sign(dy)
274 281 ax.cbar.norm.vmax += (perc * scale) * numpy.sign(dy)
275 282
276 283 ax.cbar.draw_all()
277 284 ax.plt.set_norm(ax.cbar.norm)
278 285 ax.cbar.patch.figure.canvas.draw()
279 286
280 287 def onBtnRelease(self, event):
281 288 '''
282 289 Event for mouse button release
283 290 '''
284 291 cb_ax = event.inaxes
285 292 if cb_ax is not None:
286 293 cb_ax.press = None
287 294
288 295 def __add_axes(self, ax, size='30%', pad='8%'):
289 296 '''
290 297 Add new axes to the given figure
291 298 '''
292 299 divider = make_axes_locatable(ax)
293 300 nax = divider.new_horizontal(size=size, pad=pad)
294 301 ax.figure.add_axes(nax)
295 302 return nax
296 303
297 304 self.setup()
298 305
299 306 def setup(self):
300 307 '''
301 308 This method should be implemented in the child class, the following
302 309 attributes should be set:
303 310
304 311 self.nrows: number of rows
305 312 self.ncols: number of cols
306 313 self.nplots: number of plots (channels or pairs)
307 314 self.ylabel: label for Y axes
308 315 self.titles: list of axes title
309 316
310 317 '''
311 318 raise(NotImplementedError, 'Implement this method in child class')
312 319
313 320 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
314 321 '''
315 322 Create a masked array for missing data
316 323 '''
317 324 if x_buffer.shape[0] < 2:
318 325 return x_buffer, y_buffer, z_buffer
319 326
320 327 deltas = x_buffer[1:] - x_buffer[0:-1]
321 328 x_median = numpy.median(deltas)
322 329
323 330 index = numpy.where(deltas > 5 * x_median)
324 331
325 332 if len(index[0]) != 0:
326 333 z_buffer[::, index[0], ::] = self.__missing
327 334 z_buffer = numpy.ma.masked_inside(z_buffer,
328 335 0.99 * self.__missing,
329 336 1.01 * self.__missing)
330 337
331 338 return x_buffer, y_buffer, z_buffer
332 339
333 340 def decimate(self):
334 341
335 342 # dx = int(len(self.x)/self.__MAXNUMX) + 1
336 343 dy = int(len(self.y) / self.decimation) + 1
337 344
338 345 # x = self.x[::dx]
339 346 x = self.x
340 347 y = self.y[::dy]
341 348 z = self.z[::, ::, ::dy]
342 349
343 350 return x, y, z
344 351
345 352 def format(self):
346 353 '''
347 354 Set min and max values, labels, ticks and titles
348 355 '''
349 356
350 357 if self.xmin is None:
351 358 xmin = self.min_time
352 359 else:
353 360 if self.xaxis is 'time':
354 361 dt = self.getDateTime(self.min_time)
355 362 xmin = (dt.replace(hour=int(self.xmin), minute=0, second=0) -
356 363 datetime.datetime(1970, 1, 1)).total_seconds()
357 364 if self.data.localtime:
358 365 xmin += time.timezone
359 366 else:
360 367 xmin = self.xmin
361 368
362 369 if self.xmax is None:
363 370 xmax = xmin + self.xrange * 60 * 60
364 371 else:
365 372 if self.xaxis is 'time':
366 373 dt = self.getDateTime(self.max_time)
367 374 xmax = (dt.replace(hour=int(self.xmax), minute=59, second=59) -
368 375 datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=1)).total_seconds()
369 376 if self.data.localtime:
370 377 xmax += time.timezone
371 378 else:
372 379 xmax = self.xmax
373 380
374 381 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
375 382 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
376 383
377 384 Y = numpy.array([5, 10, 20, 50, 100, 200, 500, 1000, 2000])
378 385 i = 1 if numpy.where(ymax-ymin < Y)[0][0] < 0 else numpy.where(ymax-ymin < Y)[0][0]
379 386 ystep = Y[i] / 5
380 387
381 388 for n, ax in enumerate(self.axes):
382 389 if ax.firsttime:
383 390 ax.set_facecolor(self.bgcolor)
384 391 ax.yaxis.set_major_locator(MultipleLocator(ystep))
385 392 if self.xaxis is 'time':
386 393 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
387 394 ax.xaxis.set_major_locator(LinearLocator(9))
388 395 if self.xlabel is not None:
389 396 ax.set_xlabel(self.xlabel)
390 397 ax.set_ylabel(self.ylabel)
391 398 ax.firsttime = False
392 399 if self.showprofile:
393 400 self.pf_axes[n].set_ylim(ymin, ymax)
394 401 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
395 402 self.pf_axes[n].set_xlabel('dB')
396 403 self.pf_axes[n].grid(b=True, axis='x')
397 404 [tick.set_visible(False)
398 405 for tick in self.pf_axes[n].get_yticklabels()]
399 406 if self.colorbar:
400 407 ax.cbar = plt.colorbar(
401 408 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
402 409 ax.cbar.ax.tick_params(labelsize=8)
403 410 ax.cbar.ax.press = None
404 411 if self.cb_label:
405 412 ax.cbar.set_label(self.cb_label, size=8)
406 413 elif self.cb_labels:
407 414 ax.cbar.set_label(self.cb_labels[n], size=8)
408 415 else:
409 416 ax.cbar = None
410 417
411 418 if not self.polar:
412 419 ax.set_xlim(xmin, xmax)
413 420 ax.set_ylim(ymin, ymax)
414 421 ax.set_title('{} - {} {}'.format(
415 422 self.titles[n],
416 423 self.getDateTime(self.max_time).strftime('%H:%M:%S'),
417 424 self.time_label),
418 425 size=8)
419 426 else:
420 427 ax.set_title('{}'.format(self.titles[n]), size=8)
421 428 ax.set_ylim(0, 90)
422 429 ax.set_yticks(numpy.arange(0, 90, 20))
423 430 ax.yaxis.labelpad = 40
424 431
425 432 def __plot(self):
426 433 '''
427 434 '''
428 435 log.success('Plotting', self.name)
429 436
430 437 try:
431 438 self.plot()
432 439 self.format()
433 440 except:
434 441 log.warning('{} Plot could not be updated... check data'.format(self.CODE), self.name)
435 442
436 443 for n, fig in enumerate(self.figures):
437 444 if self.nrows == 0 or self.nplots == 0:
438 445 log.warning('No data', self.name)
439 446 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
440 447 fig.canvas.manager.set_window_title(self.CODE)
441 448 continue
442 449
443 450 fig.tight_layout()
444 451 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
445 452 self.getDateTime(self.max_time).strftime('%Y/%m/%d')))
446 453 fig.canvas.draw()
447 454
448 455 if self.save and self.data.ended:
449 456 channels = range(self.nrows)
450 457 if self.oneFigure:
451 458 label = ''
452 459 else:
453 460 label = '_{}'.format(channels[n])
454 461 figname = os.path.join(
455 462 self.save,
456 463 '{}{}_{}.png'.format(
457 464 self.CODE,
458 465 label,
459 466 self.getDateTime(self.saveTime).strftime(
460 467 '%Y%m%d_%H%M%S'),
461 468 )
462 469 )
463 470 log.log('Saving figure: {}'.format(figname), self.name)
464 471 fig.savefig(figname)
465 472
466 473 def plot(self):
467 474 '''
468 475 '''
469 476 raise(NotImplementedError, 'Implement this method in child class')
470 477
471 478 def run(self):
472 479
473 480 log.success('Starting', self.name)
474 481
475 482 context = zmq.Context()
476 483 receiver = context.socket(zmq.SUB)
477 484 receiver.setsockopt(zmq.SUBSCRIBE, '')
478 485 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
479 486
480 487 if 'server' in self.kwargs['parent']:
481 488 receiver.connect(
482 489 'ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
483 490 else:
484 491 receiver.connect("ipc:///tmp/zmq.plots")
485 492
486 493 while True:
487 494 try:
488 495 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
489 496 if self.data.localtime and self.localtime:
490 497 self.times = self.data.times
491 498 elif self.data.localtime and not self.localtime:
492 499 self.times = self.data.times + time.timezone
493 500 elif not self.data.localtime and self.localtime:
494 501 self.times = self.data.times - time.timezone
495 502 else:
496 503 self.times = self.data.times
497 504
498 505 self.min_time = self.times[0]
499 506 self.max_time = self.times[-1]
500 507
501 508 if self.isConfig is False:
502 509 self.__setup()
503 510 self.isConfig = True
504 511
505 512 self.__plot()
506 513
507 514 except zmq.Again as e:
508 515 log.log('Waiting for data...')
509 516 if self.data:
510 517 figpause(self.data.throttle)
511 518 else:
512 519 time.sleep(2)
513 520
514 521 def close(self):
515 522 if self.data:
516 523 self.__plot()
517 524
518 525
519 526 class PlotSpectraData(PlotData):
520 527 '''
521 528 Plot for Spectra data
522 529 '''
523 530
524 531 CODE = 'spc'
525 532 colormap = 'jro'
526 533
527 534 def setup(self):
528 535 self.nplots = len(self.data.channels)
529 536 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
530 537 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
531 538 self.width = 3.4 * self.ncols
532 539 self.height = 3 * self.nrows
533 540 self.cb_label = 'dB'
534 541 if self.showprofile:
535 542 self.width += 0.8 * self.ncols
536 543
537 544 self.ylabel = 'Range [km]'
538 545
539 546 def plot(self):
540 547 if self.xaxis == "frequency":
541 548 x = self.data.xrange[0]
542 549 self.xlabel = "Frequency (kHz)"
543 550 elif self.xaxis == "time":
544 551 x = self.data.xrange[1]
545 552 self.xlabel = "Time (ms)"
546 553 else:
547 554 x = self.data.xrange[2]
548 555 self.xlabel = "Velocity (m/s)"
549 556
550 557 if self.CODE == 'spc_mean':
551 558 x = self.data.xrange[2]
552 559 self.xlabel = "Velocity (m/s)"
553 560
554 561 self.titles = []
555 562
556 563 y = self.data.heights
557 564 self.y = y
558 565 z = self.data['spc']
559 566
560 567 for n, ax in enumerate(self.axes):
561 568 noise = self.data['noise'][n][-1]
562 569 if self.CODE == 'spc_mean':
563 570 mean = self.data['mean'][n][-1]
564 571 if ax.firsttime:
565 572 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
566 573 self.xmin = self.xmin if self.xmin else -self.xmax
567 574 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
568 575 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
569 576 ax.plt = ax.pcolormesh(x, y, z[n].T,
570 577 vmin=self.zmin,
571 578 vmax=self.zmax,
572 579 cmap=plt.get_cmap(self.colormap)
573 580 )
574 581
575 582 if self.showprofile:
576 583 ax.plt_profile = self.pf_axes[n].plot(
577 584 self.data['rti'][n][-1], y)[0]
578 585 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
579 586 color="k", linestyle="dashed", lw=1)[0]
580 587 if self.CODE == 'spc_mean':
581 588 ax.plt_mean = ax.plot(mean, y, color='k')[0]
582 589 else:
583 590 ax.plt.set_array(z[n].T.ravel())
584 591 if self.showprofile:
585 592 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
586 593 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
587 594 if self.CODE == 'spc_mean':
588 595 ax.plt_mean.set_data(mean, y)
589 596
590 597 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
591 598 self.saveTime = self.max_time
592 599
593 600
594 601 class PlotCrossSpectraData(PlotData):
595 602
596 603 CODE = 'cspc'
597 604 zmin_coh = None
598 605 zmax_coh = None
599 606 zmin_phase = None
600 607 zmax_phase = None
601 608
602 609 def setup(self):
603 610
604 611 self.ncols = 4
605 612 self.nrows = len(self.data.pairs)
606 613 self.nplots = self.nrows * 4
607 614 self.width = 3.4 * self.ncols
608 615 self.height = 3 * self.nrows
609 616 self.ylabel = 'Range [km]'
610 617 self.showprofile = False
611 618
612 619 def plot(self):
613 620
614 621 if self.xaxis == "frequency":
615 622 x = self.data.xrange[0]
616 623 self.xlabel = "Frequency (kHz)"
617 624 elif self.xaxis == "time":
618 625 x = self.data.xrange[1]
619 626 self.xlabel = "Time (ms)"
620 627 else:
621 628 x = self.data.xrange[2]
622 629 self.xlabel = "Velocity (m/s)"
623 630
624 631 self.titles = []
625 632
626 633 y = self.data.heights
627 634 self.y = y
628 635 spc = self.data['spc']
629 636 cspc = self.data['cspc']
630 637
631 638 for n in range(self.nrows):
632 639 noise = self.data['noise'][n][-1]
633 640 pair = self.data.pairs[n]
634 641 ax = self.axes[4 * n]
635 642 ax3 = self.axes[4 * n + 3]
636 643 if ax.firsttime:
637 644 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
638 645 self.xmin = self.xmin if self.xmin else -self.xmax
639 646 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
640 647 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
641 648 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
642 649 vmin=self.zmin,
643 650 vmax=self.zmax,
644 651 cmap=plt.get_cmap(self.colormap)
645 652 )
646 653 else:
647 654 ax.plt.set_array(spc[pair[0]].T.ravel())
648 655 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
649 656
650 657 ax = self.axes[4 * n + 1]
651 658 if ax.firsttime:
652 659 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
653 660 vmin=self.zmin,
654 661 vmax=self.zmax,
655 662 cmap=plt.get_cmap(self.colormap)
656 663 )
657 664 else:
658 665 ax.plt.set_array(spc[pair[1]].T.ravel())
659 666 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
660 667
661 668 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
662 669 coh = numpy.abs(out)
663 670 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
664 671
665 672 ax = self.axes[4 * n + 2]
666 673 if ax.firsttime:
667 674 ax.plt = ax.pcolormesh(x, y, coh.T,
668 675 vmin=0,
669 676 vmax=1,
670 677 cmap=plt.get_cmap(self.colormap_coh)
671 678 )
672 679 else:
673 680 ax.plt.set_array(coh.T.ravel())
674 681 self.titles.append(
675 682 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
676 683
677 684 ax = self.axes[4 * n + 3]
678 685 if ax.firsttime:
679 686 ax.plt = ax.pcolormesh(x, y, phase.T,
680 687 vmin=-180,
681 688 vmax=180,
682 689 cmap=plt.get_cmap(self.colormap_phase)
683 690 )
684 691 else:
685 692 ax.plt.set_array(phase.T.ravel())
686 693 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
687 694
688 695 self.saveTime = self.max_time
689 696
690 697
691 698 class PlotSpectraMeanData(PlotSpectraData):
692 699 '''
693 700 Plot for Spectra and Mean
694 701 '''
695 702 CODE = 'spc_mean'
696 703 colormap = 'jro'
697 704
698 705
699 706 class PlotRTIData(PlotData):
700 707 '''
701 708 Plot for RTI data
702 709 '''
703 710
704 711 CODE = 'rti'
705 712 colormap = 'jro'
706 713
707 714 def setup(self):
708 715 self.xaxis = 'time'
709 716 self.ncols = 1
710 717 self.nrows = len(self.data.channels)
711 718 self.nplots = len(self.data.channels)
712 719 self.ylabel = 'Range [km]'
713 720 self.cb_label = 'dB'
714 721 self.titles = ['{} Channel {}'.format(
715 722 self.CODE.upper(), x) for x in range(self.nrows)]
716 723
717 724 def plot(self):
718 725 self.x = self.times
719 726 self.y = self.data.heights
720 727 self.z = self.data[self.CODE]
721 728 self.z = numpy.ma.masked_invalid(self.z)
722 729
723 730 if self.decimation is None:
724 731 x, y, z = self.fill_gaps(self.x, self.y, self.z)
725 732 else:
726 733 x, y, z = self.fill_gaps(*self.decimate())
727 734
728 735 for n, ax in enumerate(self.axes):
729 736 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
730 737 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
731 738 if ax.firsttime:
732 739 ax.plt = ax.pcolormesh(x, y, z[n].T,
733 740 vmin=self.zmin,
734 741 vmax=self.zmax,
735 742 cmap=plt.get_cmap(self.colormap)
736 743 )
737 744 if self.showprofile:
738 745 ax.plot_profile = self.pf_axes[n].plot(
739 746 self.data['rti'][n][-1], self.y)[0]
740 747 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
741 748 color="k", linestyle="dashed", lw=1)[0]
742 749 else:
743 750 ax.collections.remove(ax.collections[0])
744 751 ax.plt = ax.pcolormesh(x, y, z[n].T,
745 752 vmin=self.zmin,
746 753 vmax=self.zmax,
747 754 cmap=plt.get_cmap(self.colormap)
748 755 )
749 756 if self.showprofile:
750 757 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
751 758 ax.plot_noise.set_data(numpy.repeat(
752 759 self.data['noise'][n][-1], len(self.y)), self.y)
753 760
754 761 self.saveTime = self.min_time
755 762
756 763
757 764 class PlotCOHData(PlotRTIData):
758 765 '''
759 766 Plot for Coherence data
760 767 '''
761 768
762 769 CODE = 'coh'
763 770
764 771 def setup(self):
765 772 self.xaxis = 'time'
766 773 self.ncols = 1
767 774 self.nrows = len(self.data.pairs)
768 775 self.nplots = len(self.data.pairs)
769 776 self.ylabel = 'Range [km]'
770 777 if self.CODE == 'coh':
771 778 self.cb_label = ''
772 779 self.titles = [
773 780 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
774 781 else:
775 782 self.cb_label = 'Degrees'
776 783 self.titles = [
777 784 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
778 785
779 786
780 787 class PlotPHASEData(PlotCOHData):
781 788 '''
782 789 Plot for Phase map data
783 790 '''
784 791
785 792 CODE = 'phase'
786 793 colormap = 'seismic'
787 794
788 795
789 796 class PlotNoiseData(PlotData):
790 797 '''
791 798 Plot for noise
792 799 '''
793 800
794 801 CODE = 'noise'
795 802
796 803 def setup(self):
797 804 self.xaxis = 'time'
798 805 self.ncols = 1
799 806 self.nrows = 1
800 807 self.nplots = 1
801 808 self.ylabel = 'Intensity [dB]'
802 809 self.titles = ['Noise']
803 810 self.colorbar = False
804 811
805 812 def plot(self):
806 813
807 814 x = self.times
808 815 xmin = self.min_time
809 816 xmax = xmin + self.xrange * 60 * 60
810 817 Y = self.data[self.CODE]
811 818
812 819 if self.axes[0].firsttime:
813 820 for ch in self.data.channels:
814 821 y = Y[ch]
815 822 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
816 823 plt.legend()
817 824 else:
818 825 for ch in self.data.channels:
819 826 y = Y[ch]
820 827 self.axes[0].lines[ch].set_data(x, y)
821 828
822 829 self.ymin = numpy.nanmin(Y) - 5
823 830 self.ymax = numpy.nanmax(Y) + 5
824 831 self.saveTime = self.min_time
825 832
826 833
827 834 class PlotSNRData(PlotRTIData):
828 835 '''
829 836 Plot for SNR Data
830 837 '''
831 838
832 839 CODE = 'snr'
833 840 colormap = 'jet'
834 841
835 842
836 843 class PlotDOPData(PlotRTIData):
837 844 '''
838 845 Plot for DOPPLER Data
839 846 '''
840 847
841 848 CODE = 'dop'
842 849 colormap = 'jet'
843 850
844 851
845 852 class PlotSkyMapData(PlotData):
846 853 '''
847 854 Plot for meteors detection data
848 855 '''
849 856
850 857 CODE = 'param'
851 858
852 859 def setup(self):
853 860
854 861 self.ncols = 1
855 862 self.nrows = 1
856 863 self.width = 7.2
857 864 self.height = 7.2
858 865 self.nplots = 1
859 866 self.xlabel = 'Zonal Zenith Angle (deg)'
860 867 self.ylabel = 'Meridional Zenith Angle (deg)'
861 868 self.polar = True
862 869 self.ymin = -180
863 870 self.ymax = 180
864 871 self.colorbar = False
865 872
866 873 def plot(self):
867 874
868 875 arrayParameters = numpy.concatenate(self.data['param'])
869 876 error = arrayParameters[:, -1]
870 877 indValid = numpy.where(error == 0)[0]
871 878 finalMeteor = arrayParameters[indValid, :]
872 879 finalAzimuth = finalMeteor[:, 3]
873 880 finalZenith = finalMeteor[:, 4]
874 881
875 882 x = finalAzimuth * numpy.pi / 180
876 883 y = finalZenith
877 884
878 885 ax = self.axes[0]
879 886
880 887 if ax.firsttime:
881 888 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
882 889 else:
883 890 ax.plot.set_data(x, y)
884 891
885 892 dt1 = self.getDateTime(self.min_time).strftime('%y/%m/%d %H:%M:%S')
886 893 dt2 = self.getDateTime(self.max_time).strftime('%y/%m/%d %H:%M:%S')
887 894 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
888 895 dt2,
889 896 len(x))
890 897 self.titles[0] = title
891 898 self.saveTime = self.max_time
892 899
893 900
894 901 class PlotParamData(PlotRTIData):
895 902 '''
896 903 Plot for data_param object
897 904 '''
898 905
899 906 CODE = 'param'
900 907 colormap = 'seismic'
901 908
902 909 def setup(self):
903 910 self.xaxis = 'time'
904 911 self.ncols = 1
905 912 self.nrows = self.data.shape(self.CODE)[0]
906 913 self.nplots = self.nrows
907 914 if self.showSNR:
908 915 self.nrows += 1
909 916 self.nplots += 1
910 917
911 918 self.ylabel = 'Height [km]'
912 919 if not self.titles:
913 920 self.titles = self.data.parameters \
914 921 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
915 922 if self.showSNR:
916 923 self.titles.append('SNR')
917 924
918 925 def plot(self):
919 926 self.data.normalize_heights()
920 927 self.x = self.times
921 928 self.y = self.data.heights
922 929 if self.showSNR:
923 930 self.z = numpy.concatenate(
924 931 (self.data[self.CODE], self.data['snr'])
925 932 )
926 933 else:
927 934 self.z = self.data[self.CODE]
928 935
929 936 self.z = numpy.ma.masked_invalid(self.z)
930 937
931 938 if self.decimation is None:
932 939 x, y, z = self.fill_gaps(self.x, self.y, self.z)
933 940 else:
934 941 x, y, z = self.fill_gaps(*self.decimate())
935 942
936 943 for n, ax in enumerate(self.axes):
937 944
938 945 self.zmax = self.zmax if self.zmax is not None else numpy.max(
939 946 self.z[n])
940 947 self.zmin = self.zmin if self.zmin is not None else numpy.min(
941 948 self.z[n])
942 949
943 950 if ax.firsttime:
944 951 if self.zlimits is not None:
945 952 self.zmin, self.zmax = self.zlimits[n]
946 953
947 954 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
948 955 vmin=self.zmin,
949 956 vmax=self.zmax,
950 957 cmap=self.cmaps[n]
951 958 )
952 959 else:
953 960 if self.zlimits is not None:
954 961 self.zmin, self.zmax = self.zlimits[n]
955 962 ax.collections.remove(ax.collections[0])
956 963 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
957 964 vmin=self.zmin,
958 965 vmax=self.zmax,
959 966 cmap=self.cmaps[n]
960 967 )
961 968
962 969 self.saveTime = self.min_time
963 970
964 971
965 972 class PlotOutputData(PlotParamData):
966 973 '''
967 974 Plot data_output object
968 975 '''
969 976
970 977 CODE = 'output'
971 978 colormap = 'seismic'
Show file before | Show file after | Hide comments
@@ -1,1826 +1,1826
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 import schainpy.admin
23 24 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 25 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 26 from schainpy.utils import log
26 27 import schainpy.admin
27 28
28 29 LOCALTIME = True
29 30
30 31
31 32 def isNumber(cad):
32 33 """
33 34 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34 35
35 36 Excepciones:
36 37 Si un determinado string no puede ser convertido a numero
37 38 Input:
38 39 str, string al cual se le analiza para determinar si convertible a un numero o no
39 40
40 41 Return:
41 42 True : si el string es uno numerico
42 43 False : no es un string numerico
43 44 """
44 45 try:
45 46 float(cad)
46 47 return True
47 48 except:
48 49 return False
49 50
50 51
51 52 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 53 """
53 54 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54 55
55 56 Inputs:
56 57 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57 58
58 59 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 60 segundos contados desde 01/01/1970.
60 61 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 62 segundos contados desde 01/01/1970.
62 63
63 64 Return:
64 65 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 66 fecha especificado, de lo contrario retorna False.
66 67
67 68 Excepciones:
68 69 Si el archivo no existe o no puede ser abierto
69 70 Si la cabecera no puede ser leida.
70 71
71 72 """
72 73 basicHeaderObj = BasicHeader(LOCALTIME)
73 74
74 75 try:
75 76 fp = open(filename, 'rb')
76 77 except IOError:
77 78 print "The file %s can't be opened" % (filename)
78 79 return 0
79 80
80 81 sts = basicHeaderObj.read(fp)
81 82 fp.close()
82 83
83 84 if not(sts):
84 85 print "Skipping the file %s because it has not a valid header" % (filename)
85 86 return 0
86 87
87 88 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 89 return 0
89 90
90 91 return 1
91 92
92 93
93 94 def isTimeInRange(thisTime, startTime, endTime):
94 95 if endTime >= startTime:
95 96 if (thisTime < startTime) or (thisTime > endTime):
96 97 return 0
97 98 return 1
98 99 else:
99 100 if (thisTime < startTime) and (thisTime > endTime):
100 101 return 0
101 102 return 1
102 103
103 104
104 105 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 106 """
106 107 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107 108
108 109 Inputs:
109 110 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110 111
111 112 startDate : fecha inicial del rango seleccionado en formato datetime.date
112 113
113 114 endDate : fecha final del rango seleccionado en formato datetime.date
114 115
115 116 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116 117
117 118 endTime : tiempo final del rango seleccionado en formato datetime.time
118 119
119 120 Return:
120 121 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 122 fecha especificado, de lo contrario retorna False.
122 123
123 124 Excepciones:
124 125 Si el archivo no existe o no puede ser abierto
125 126 Si la cabecera no puede ser leida.
126 127
127 128 """
128 129
129 130 try:
130 131 fp = open(filename, 'rb')
131 132 except IOError:
132 133 print "The file %s can't be opened" % (filename)
133 134 return None
134 135
135 136 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 137 systemHeaderObj = SystemHeader()
137 138 radarControllerHeaderObj = RadarControllerHeader()
138 139 processingHeaderObj = ProcessingHeader()
139 140
140 141 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141 142
142 143 sts = firstBasicHeaderObj.read(fp)
143 144
144 145 if not(sts):
145 146 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
146 147 return None
147 148
148 149 if not systemHeaderObj.read(fp):
149 150 return None
150 151
151 152 if not radarControllerHeaderObj.read(fp):
152 153 return None
153 154
154 155 if not processingHeaderObj.read(fp):
155 156 return None
156 157
157 158 filesize = os.path.getsize(filename)
158 159
159 160 offset = processingHeaderObj.blockSize + 24 # header size
160 161
161 162 if filesize <= offset:
162 163 print "[Reading] %s: This file has not enough data" % filename
163 164 return None
164 165
165 166 fp.seek(-offset, 2)
166 167
167 168 sts = lastBasicHeaderObj.read(fp)
168 169
169 170 fp.close()
170 171
171 172 thisDatetime = lastBasicHeaderObj.datatime
172 173 thisTime_last_block = thisDatetime.time()
173 174
174 175 thisDatetime = firstBasicHeaderObj.datatime
175 176 thisDate = thisDatetime.date()
176 177 thisTime_first_block = thisDatetime.time()
177 178
178 179 # General case
179 180 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 181 #-----------o----------------------------o-----------
181 182 # startTime endTime
182 183
183 184 if endTime >= startTime:
184 185 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 186 return None
186 187
187 188 return thisDatetime
188 189
189 190 # If endTime < startTime then endTime belongs to the next day
190 191
191 192 #<<<<<<<<<<<o o>>>>>>>>>>>
192 193 #-----------o----------------------------o-----------
193 194 # endTime startTime
194 195
195 196 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 197 return None
197 198
198 199 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 200 return None
200 201
201 202 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 203 return None
203 204
204 205 return thisDatetime
205 206
206 207
207 208 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 209 """
209 210 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210 211
211 212 Inputs:
212 213 folder : nombre completo del directorio.
213 214 Su formato deberia ser "/path_root/?YYYYDDD"
214 215
215 216 siendo:
216 217 YYYY : Anio (ejemplo 2015)
217 218 DDD : Dia del anio (ejemplo 305)
218 219
219 220 startDate : fecha inicial del rango seleccionado en formato datetime.date
220 221
221 222 endDate : fecha final del rango seleccionado en formato datetime.date
222 223
223 224 Return:
224 225 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 226 fecha especificado, de lo contrario retorna False.
226 227 Excepciones:
227 228 Si el directorio no tiene el formato adecuado
228 229 """
229 230
230 231 basename = os.path.basename(folder)
231 232
232 233 if not isRadarFolder(basename):
233 234 print "The folder %s has not the rigth format" % folder
234 235 return 0
235 236
236 237 if startDate and endDate:
237 238 thisDate = getDateFromRadarFolder(basename)
238 239
239 240 if thisDate < startDate:
240 241 return 0
241 242
242 243 if thisDate > endDate:
243 244 return 0
244 245
245 246 return 1
246 247
247 248
248 249 def isFileInDateRange(filename, startDate=None, endDate=None):
249 250 """
250 251 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251 252
252 253 Inputs:
253 254 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254 255
255 256 Su formato deberia ser "?YYYYDDDsss"
256 257
257 258 siendo:
258 259 YYYY : Anio (ejemplo 2015)
259 260 DDD : Dia del anio (ejemplo 305)
260 261 sss : set
261 262
262 263 startDate : fecha inicial del rango seleccionado en formato datetime.date
263 264
264 265 endDate : fecha final del rango seleccionado en formato datetime.date
265 266
266 267 Return:
267 268 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 269 fecha especificado, de lo contrario retorna False.
269 270 Excepciones:
270 271 Si el archivo no tiene el formato adecuado
271 272 """
272 273
273 274 basename = os.path.basename(filename)
274 275
275 276 if not isRadarFile(basename):
276 277 print "The filename %s has not the rigth format" % filename
277 278 return 0
278 279
279 280 if startDate and endDate:
280 281 thisDate = getDateFromRadarFile(basename)
281 282
282 283 if thisDate < startDate:
283 284 return 0
284 285
285 286 if thisDate > endDate:
286 287 return 0
287 288
288 289 return 1
289 290
290 291
291 292 def getFileFromSet(path, ext, set):
292 293 validFilelist = []
293 294 fileList = os.listdir(path)
294 295
295 296 # 0 1234 567 89A BCDE
296 297 # H YYYY DDD SSS .ext
297 298
298 299 for thisFile in fileList:
299 300 try:
300 301 year = int(thisFile[1:5])
301 302 doy = int(thisFile[5:8])
302 303 except:
303 304 continue
304 305
305 306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 307 continue
307 308
308 309 validFilelist.append(thisFile)
309 310
310 311 myfile = fnmatch.filter(
311 312 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312 313
313 314 if len(myfile) != 0:
314 315 return myfile[0]
315 316 else:
316 317 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 318 print 'the filename %s does not exist' % filename
318 319 print '...going to the last file: '
319 320
320 321 if validFilelist:
321 322 validFilelist = sorted(validFilelist, key=str.lower)
322 323 return validFilelist[-1]
323 324
324 325 return None
325 326
326 327
327 328 def getlastFileFromPath(path, ext):
328 329 """
329 330 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 331 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331 332
332 333 Input:
333 334 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 335 ext : extension de los files contenidos en una carpeta
335 336
336 337 Return:
337 338 El ultimo file de una determinada carpeta, no se considera el path.
338 339 """
339 340 validFilelist = []
340 341 fileList = os.listdir(path)
341 342
342 343 # 0 1234 567 89A BCDE
343 344 # H YYYY DDD SSS .ext
344 345
345 346 for thisFile in fileList:
346 347
347 348 year = thisFile[1:5]
348 349 if not isNumber(year):
349 350 continue
350 351
351 352 doy = thisFile[5:8]
352 353 if not isNumber(doy):
353 354 continue
354 355
355 356 year = int(year)
356 357 doy = int(doy)
357 358
358 359 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 360 continue
360 361
361 362 validFilelist.append(thisFile)
362 363
363 364 if validFilelist:
364 365 validFilelist = sorted(validFilelist, key=str.lower)
365 366 return validFilelist[-1]
366 367
367 368 return None
368 369
369 370
370 371 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 372 """
372 373 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 374 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 375 el path exacto de un determinado file.
375 376
376 377 Example :
377 378 nombre correcto del file es .../.../D2009307/P2009307367.ext
378 379
379 380 Entonces la funcion prueba con las siguientes combinaciones
380 381 .../.../y2009307367.ext
381 382 .../.../Y2009307367.ext
382 383 .../.../x2009307/y2009307367.ext
383 384 .../.../x2009307/Y2009307367.ext
384 385 .../.../X2009307/y2009307367.ext
385 386 .../.../X2009307/Y2009307367.ext
386 387 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387 388
388 389 Return:
389 390 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 391 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 392 para el filename
392 393 """
393 394 fullfilename = None
394 395 find_flag = False
395 396 filename = None
396 397
397 398 prefixDirList = [None, 'd', 'D']
398 399 if ext.lower() == ".r": # voltage
399 400 prefixFileList = ['d', 'D']
400 401 elif ext.lower() == ".pdata": # spectra
401 402 prefixFileList = ['p', 'P']
402 403 else:
403 404 return None, filename
404 405
405 406 # barrido por las combinaciones posibles
406 407 for prefixDir in prefixDirList:
407 408 thispath = path
408 409 if prefixDir != None:
409 410 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 411 if foldercounter == 0:
411 412 thispath = os.path.join(path, "%s%04d%03d" %
412 413 (prefixDir, year, doy))
413 414 else:
414 415 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 416 prefixDir, year, doy, foldercounter))
416 417 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 418 # formo el nombre del file xYYYYDDDSSS.ext
418 419 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 420 fullfilename = os.path.join(
420 421 thispath, filename) # formo el path completo
421 422
422 423 if os.path.exists(fullfilename): # verifico que exista
423 424 find_flag = True
424 425 break
425 426 if find_flag:
426 427 break
427 428
428 429 if not(find_flag):
429 430 return None, filename
430 431
431 432 return fullfilename, filename
432 433
433 434
434 435 def isRadarFolder(folder):
435 436 try:
436 437 year = int(folder[1:5])
437 438 doy = int(folder[5:8])
438 439 except:
439 440 return 0
440 441
441 442 return 1
442 443
443 444
444 445 def isRadarFile(file):
445 446 try:
446 447 year = int(file[1:5])
447 448 doy = int(file[5:8])
448 449 set = int(file[8:11])
449 450 except:
450 451 return 0
451 452
452 453 return 1
453 454
454 455
455 456 def getDateFromRadarFile(file):
456 457 try:
457 458 year = int(file[1:5])
458 459 doy = int(file[5:8])
459 460 set = int(file[8:11])
460 461 except:
461 462 return None
462 463
463 464 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 465 return thisDate
465 466
466 467
467 468 def getDateFromRadarFolder(folder):
468 469 try:
469 470 year = int(folder[1:5])
470 471 doy = int(folder[5:8])
471 472 except:
472 473 return None
473 474
474 475 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 476 return thisDate
476 477
477 478
478 479 class JRODataIO:
479 480
480 481 c = 3E8
481 482
482 483 isConfig = False
483 484
484 485 basicHeaderObj = None
485 486
486 487 systemHeaderObj = None
487 488
488 489 radarControllerHeaderObj = None
489 490
490 491 processingHeaderObj = None
491 492
492 493 dtype = None
493 494
494 495 pathList = []
495 496
496 497 filenameList = []
497 498
498 499 filename = None
499 500
500 501 ext = None
501 502
502 503 flagIsNewFile = 1
503 504
504 505 flagDiscontinuousBlock = 0
505 506
506 507 flagIsNewBlock = 0
507 508
508 509 fp = None
509 510
510 511 firstHeaderSize = 0
511 512
512 513 basicHeaderSize = 24
513 514
514 515 versionFile = 1103
515 516
516 517 fileSize = None
517 518
518 519 # ippSeconds = None
519 520
520 521 fileSizeByHeader = None
521 522
522 523 fileIndex = None
523 524
524 525 profileIndex = None
525 526
526 527 blockIndex = None
527 528
528 529 nTotalBlocks = None
529 530
530 531 maxTimeStep = 30
531 532
532 533 lastUTTime = None
533 534
534 535 datablock = None
535 536
536 537 dataOut = None
537 538
538 539 blocksize = None
539 540
540 541 getByBlock = False
541 542
542 543 def __init__(self):
543 544
544 545 raise NotImplementedError
545 546
546 547 def run(self):
547 548
548 549 raise NotImplementedError
549 550
550 551 def getDtypeWidth(self):
551 552
552 553 dtype_index = get_dtype_index(self.dtype)
553 554 dtype_width = get_dtype_width(dtype_index)
554 555
555 556 return dtype_width
556 557
557 558 def getAllowedArgs(self):
558 559 if hasattr(self, '__attrs__'):
559 560 return self.__attrs__
560 561 else:
561 562 return inspect.getargspec(self.run).args
562 563
563 564
564 565 class JRODataReader(JRODataIO):
565 566
566 567 online = 0
567 568
568 569 realtime = 0
569 570
570 571 nReadBlocks = 0
571 572
572 573 delay = 10 # number of seconds waiting a new file
573 574
574 575 nTries = 3 # quantity tries
575 576
576 577 nFiles = 3 # number of files for searching
577 578
578 579 path = None
579 580
580 581 foldercounter = 0
581 582
582 583 flagNoMoreFiles = 0
583 584
584 585 datetimeList = []
585 586
586 587 __isFirstTimeOnline = 1
587 588
588 589 __printInfo = True
589 590
590 591 profileIndex = None
591 592
592 593 nTxs = 1
593 594
594 595 txIndex = None
595 596
596 597 # Added--------------------
597 598
598 599 selBlocksize = None
599 600
600 601 selBlocktime = None
601 602
602 603 def __init__(self):
603 604 """
604 605 This class is used to find data files
605 606
606 607 Example:
607 608 reader = JRODataReader()
608 609 fileList = reader.findDataFiles()
609 610
610 611 """
611 612 pass
612 613
613 614 def createObjByDefault(self):
614 615 """
615 616
616 617 """
617 618 raise NotImplementedError
618 619
619 620 def getBlockDimension(self):
620 621
621 622 raise NotImplementedError
622 623
623 624 def searchFilesOffLine(self,
624 625 path,
625 626 startDate=None,
626 627 endDate=None,
627 628 startTime=datetime.time(0, 0, 0),
628 629 endTime=datetime.time(23, 59, 59),
629 630 set=None,
630 631 expLabel='',
631 632 ext='.r',
632 633 cursor=None,
633 634 skip=None,
634 635 walk=True):
635 636
636 637 self.filenameList = []
637 638 self.datetimeList = []
638 639
639 640 pathList = []
640 641
641 642 dateList, pathList = self.findDatafiles(
642 643 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643 644
644 645 if dateList == []:
645 646 return [], []
646 647
647 648 if len(dateList) > 1:
648 649 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
649 650 else:
650 651 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
651 652
652 653 filenameList = []
653 654 datetimeList = []
654 655
655 656 for thisPath in pathList:
656 657
657 658 fileList = glob.glob1(thisPath, "*%s" % ext)
658 659 fileList.sort()
659 660
660 661 for file in fileList:
661 662
662 663 filename = os.path.join(thisPath, file)
663 664
664 665 if not isFileInDateRange(filename, startDate, endDate):
665 666 continue
666 667
667 668 thisDatetime = isFileInTimeRange(
668 669 filename, startDate, endDate, startTime, endTime)
669 670
670 671 if not(thisDatetime):
671 672 continue
672 673
673 674 filenameList.append(filename)
674 675 datetimeList.append(thisDatetime)
675 676
676 677 if cursor is not None and skip is not None:
677 678 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 679 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679 680
680 681 if not(filenameList):
681 682 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
682 683 return [], []
683 684
684 685 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
685 686
686 687 # for i in range(len(filenameList)):
687 688 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688 689
689 690 self.filenameList = filenameList
690 691 self.datetimeList = datetimeList
691 692
692 693 return pathList, filenameList
693 694
694 695 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 696 """
696 697 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 698 devuelve el archivo encontrado ademas de otros datos.
698 699
699 700 Input:
700 701 path : carpeta donde estan contenidos los files que contiene data
701 702
702 703 expLabel : Nombre del subexperimento (subfolder)
703 704
704 705 ext : extension de los files
705 706
706 707 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707 708
708 709 Return:
709 710 directory : eL directorio donde esta el file encontrado
710 711 filename : el ultimo file de una determinada carpeta
711 712 year : el anho
712 713 doy : el numero de dia del anho
713 714 set : el set del archivo
714 715
715 716
716 717 """
717 718 if not os.path.isdir(path):
718 719 return None, None, None, None, None, None
719 720
720 721 dirList = []
721 722
722 723 if not walk:
723 724 fullpath = path
724 725 foldercounter = 0
725 726 else:
726 727 # Filtra solo los directorios
727 728 for thisPath in os.listdir(path):
728 729 if not os.path.isdir(os.path.join(path, thisPath)):
729 730 continue
730 731 if not isRadarFolder(thisPath):
731 732 continue
732 733
733 734 dirList.append(thisPath)
734 735
735 736 if not(dirList):
736 737 return None, None, None, None, None, None
737 738
738 739 dirList = sorted(dirList, key=str.lower)
739 740
740 741 doypath = dirList[-1]
741 742 foldercounter = int(doypath.split('_')[1]) if len(
742 743 doypath.split('_')) > 1 else 0
743 744 fullpath = os.path.join(path, doypath, expLabel)
744 745
745 746 print "[Reading] %s folder was found: " % (fullpath)
746 747
747 748 if set == None:
748 749 filename = getlastFileFromPath(fullpath, ext)
749 750 else:
750 751 filename = getFileFromSet(fullpath, ext, set)
751 752
752 753 if not(filename):
753 754 return None, None, None, None, None, None
754 755
755 756 print "[Reading] %s file was found" % (filename)
756 757
757 758 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 759 return None, None, None, None, None, None
759 760
760 761 year = int(filename[1:5])
761 762 doy = int(filename[5:8])
762 763 set = int(filename[8:11])
763 764
764 765 return fullpath, foldercounter, filename, year, doy, set
765 766
766 767 def __setNextFileOffline(self):
767 768
768 769 idFile = self.fileIndex
769 770
770 771 while (True):
771 772 idFile += 1
772 773 if not(idFile < len(self.filenameList)):
773 774 self.flagNoMoreFiles = 1
774 775 # print "[Reading] No more Files"
775 776 return 0
776 777
777 778 filename = self.filenameList[idFile]
778 779
779 780 if not(self.__verifyFile(filename)):
780 781 continue
781 782
782 783 fileSize = os.path.getsize(filename)
783 784 fp = open(filename, 'rb')
784 785 break
785 786
786 787 self.flagIsNewFile = 1
787 788 self.fileIndex = idFile
788 789 self.filename = filename
789 790 self.fileSize = fileSize
790 791 self.fp = fp
791 792
792 793 # print "[Reading] Setting the file: %s"%self.filename
793 794
794 795 return 1
795 796
796 797 def __setNextFileOnline(self):
797 798 """
798 799 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 800 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 801 siguientes.
801 802
802 803 Affected:
803 804 self.flagIsNewFile
804 805 self.filename
805 806 self.fileSize
806 807 self.fp
807 808 self.set
808 809 self.flagNoMoreFiles
809 810
810 811 Return:
811 812 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 813 1 : si el file fue abierto con exito y esta listo a ser leido
813 814
814 815 Excepciones:
815 816 Si un determinado file no puede ser abierto
816 817 """
817 818 nFiles = 0
818 819 fileOk_flag = False
819 820 firstTime_flag = True
820 821
821 822 self.set += 1
822 823
823 824 if self.set > 999:
824 825 self.set = 0
825 826 self.foldercounter += 1
826 827
827 828 # busca el 1er file disponible
828 829 fullfilename, filename = checkForRealPath(
829 830 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 831 if fullfilename:
831 832 if self.__verifyFile(fullfilename, False):
832 833 fileOk_flag = True
833 834
834 835 # si no encuentra un file entonces espera y vuelve a buscar
835 836 if not(fileOk_flag):
836 837 # busco en los siguientes self.nFiles+1 files posibles
837 838 for nFiles in range(self.nFiles + 1):
838 839
839 840 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 841 tries = self.nTries
841 842 else:
842 843 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843 844
844 845 for nTries in range(tries):
845 846 if firstTime_flag:
846 847 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
847 848 sleep(self.delay)
848 849 else:
849 850 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
850 851
851 852 fullfilename, filename = checkForRealPath(
852 853 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 854 if fullfilename:
854 855 if self.__verifyFile(fullfilename):
855 856 fileOk_flag = True
856 857 break
857 858
858 859 if fileOk_flag:
859 860 break
860 861
861 862 firstTime_flag = False
862 863
863 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
864 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
864 865 self.set += 1
865 866
866 867 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 868 if nFiles == (self.nFiles - 1):
868 869 self.set = 0
869 870 self.doy += 1
870 871 self.foldercounter = 0
871 872
872 873 if fileOk_flag:
873 874 self.fileSize = os.path.getsize(fullfilename)
874 875 self.filename = fullfilename
875 876 self.flagIsNewFile = 1
876 877 if self.fp != None:
877 878 self.fp.close()
878 879 self.fp = open(fullfilename, 'rb')
879 880 self.flagNoMoreFiles = 0
880 881 # print '[Reading] Setting the file: %s' % fullfilename
881 882 else:
882 883 self.fileSize = 0
883 884 self.filename = None
884 885 self.flagIsNewFile = 0
885 886 self.fp = None
886 887 self.flagNoMoreFiles = 1
887 # print '[Reading] No more files to read'
888 888
889 889 return fileOk_flag
890 890
891 891 def setNextFile(self):
892 892 if self.fp != None:
893 893 self.fp.close()
894 894
895 895 if self.online:
896 896 newFile = self.__setNextFileOnline()
897 897 else:
898 898 newFile = self.__setNextFileOffline()
899 899
900 if not(newFile):
901 print '[Reading] No more files to read'
900 if not(newFile):
901 raise schainpy.admin.SchainWarning('No more files to read')
902 902 return 0
903 903
904 904 if self.verbose:
905 905 print '[Reading] Setting the file: %s' % self.filename
906 906
907 907 self.__readFirstHeader()
908 908 self.nReadBlocks = 0
909 909 return 1
910 910
911 911 def __waitNewBlock(self):
912 912 """
913 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 914
915 915 Si el modo de lectura es OffLine siempre retorn 0
916 916 """
917 917 if not self.online:
918 918 return 0
919 919
920 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 921 return 0
922 922
923 923 currentPointer = self.fp.tell()
924 924
925 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 926
927 927 for nTries in range(self.nTries):
928 928
929 929 self.fp.close()
930 930 self.fp = open(self.filename, 'rb')
931 931 self.fp.seek(currentPointer)
932 932
933 933 self.fileSize = os.path.getsize(self.filename)
934 934 currentSize = self.fileSize - currentPointer
935 935
936 936 if (currentSize >= neededSize):
937 937 self.basicHeaderObj.read(self.fp)
938 938 return 1
939 939
940 940 if self.fileSize == self.fileSizeByHeader:
941 941 # self.flagEoF = True
942 942 return 0
943 943
944 944 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
945 945 sleep(self.delay)
946 946
947 947 return 0
948 948
949 949 def waitDataBlock(self, pointer_location):
950 950
951 951 currentPointer = pointer_location
952 952
953 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 954
955 955 for nTries in range(self.nTries):
956 956 self.fp.close()
957 957 self.fp = open(self.filename, 'rb')
958 958 self.fp.seek(currentPointer)
959 959
960 960 self.fileSize = os.path.getsize(self.filename)
961 961 currentSize = self.fileSize - currentPointer
962 962
963 963 if (currentSize >= neededSize):
964 964 return 1
965 965
966 966 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
967 967 sleep(self.delay)
968 968
969 969 return 0
970 970
971 971 def __jumpToLastBlock(self):
972 972
973 973 if not(self.__isFirstTimeOnline):
974 974 return
975 975
976 976 csize = self.fileSize - self.fp.tell()
977 977 blocksize = self.processingHeaderObj.blockSize
978 978
979 979 # salta el primer bloque de datos
980 980 if csize > self.processingHeaderObj.blockSize:
981 981 self.fp.seek(self.fp.tell() + blocksize)
982 982 else:
983 983 return
984 984
985 985 csize = self.fileSize - self.fp.tell()
986 986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 987 while True:
988 988
989 989 if self.fp.tell() < self.fileSize:
990 990 self.fp.seek(self.fp.tell() + neededsize)
991 991 else:
992 992 self.fp.seek(self.fp.tell() - neededsize)
993 993 break
994 994
995 995 # csize = self.fileSize - self.fp.tell()
996 996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 997 # factor = int(csize/neededsize)
998 998 # if factor > 0:
999 999 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000 1000
1001 1001 self.flagIsNewFile = 0
1002 1002 self.__isFirstTimeOnline = 0
1003 1003
1004 1004 def __setNewBlock(self):
1005 1005 # if self.server is None:
1006 1006 if self.fp == None:
1007 1007 return 0
1008 1008
1009 1009 # if self.online:
1010 1010 # self.__jumpToLastBlock()
1011 1011
1012 1012 if self.flagIsNewFile:
1013 1013 self.lastUTTime = self.basicHeaderObj.utc
1014 1014 return 1
1015 1015
1016 1016 if self.realtime:
1017 1017 self.flagDiscontinuousBlock = 1
1018 1018 if not(self.setNextFile()):
1019 1019 return 0
1020 1020 else:
1021 1021 return 1
1022 1022 # if self.server is None:
1023 1023 currentSize = self.fileSize - self.fp.tell()
1024 1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 1025 if (currentSize >= neededSize):
1026 1026 self.basicHeaderObj.read(self.fp)
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029 # else:
1030 1030 # self.basicHeaderObj.read(self.zHeader)
1031 1031 # self.lastUTTime = self.basicHeaderObj.utc
1032 1032 # return 1
1033 1033 if self.__waitNewBlock():
1034 1034 self.lastUTTime = self.basicHeaderObj.utc
1035 1035 return 1
1036 1036 # if self.server is None:
1037 1037 if not(self.setNextFile()):
1038 1038 return 0
1039 1039
1040 1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042
1043 1043 self.flagDiscontinuousBlock = 0
1044 1044
1045 1045 if deltaTime > self.maxTimeStep:
1046 1046 self.flagDiscontinuousBlock = 1
1047 1047
1048 1048 return 1
1049 1049
1050 1050 def readNextBlock(self):
1051 1051
1052 1052 # Skip block out of startTime and endTime
1053 1053 while True:
1054 1054 if not(self.__setNewBlock()):
1055 1055 raise(schainpy.admin.SchainWarning('No more files'))
1056 1056 return 0
1057 1057
1058 1058 if not(self.readBlock()):
1059 1059 return 0
1060 1060
1061 1061 self.getBasicHeader()
1062 1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 1063 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 1064 self.processingHeaderObj.dataBlocksPerFile,
1065 1065 self.dataOut.datatime.ctime())
1066 1066 continue
1067 1067
1068 1068 break
1069 1069
1070 1070 if self.verbose:
1071 1071 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 1072 self.processingHeaderObj.dataBlocksPerFile,
1073 1073 self.dataOut.datatime.ctime())
1074 1074 return 1
1075 1075
1076 1076 def __readFirstHeader(self):
1077 1077
1078 1078 self.basicHeaderObj.read(self.fp)
1079 1079 self.systemHeaderObj.read(self.fp)
1080 1080 self.radarControllerHeaderObj.read(self.fp)
1081 1081 self.processingHeaderObj.read(self.fp)
1082 1082
1083 1083 self.firstHeaderSize = self.basicHeaderObj.size
1084 1084
1085 1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 1087 if datatype == 0:
1088 1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 1089 elif datatype == 1:
1090 1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 1091 elif datatype == 2:
1092 1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 1093 elif datatype == 3:
1094 1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 1095 elif datatype == 4:
1096 1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 1097 elif datatype == 5:
1098 1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 1099 else:
1100 1100 raise ValueError, 'Data type was not defined'
1101 1101
1102 1102 self.dtype = datatype_str
1103 1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 1105 self.firstHeaderSize + self.basicHeaderSize * \
1106 1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 1109 self.getBlockDimension()
1110 1110
1111 1111 def __verifyFile(self, filename, msgFlag=True):
1112 1112
1113 1113 msg = None
1114 1114
1115 1115 try:
1116 1116 fp = open(filename, 'rb')
1117 1117 except IOError:
1118 1118
1119 1119 if msgFlag:
1120 1120 print "[Reading] File %s can't be opened" % (filename)
1121 1121
1122 1122 return False
1123 1123
1124 1124 currentPosition = fp.tell()
1125 1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126 1126
1127 1127 if neededSize == 0:
1128 1128 basicHeaderObj = BasicHeader(LOCALTIME)
1129 1129 systemHeaderObj = SystemHeader()
1130 1130 radarControllerHeaderObj = RadarControllerHeader()
1131 1131 processingHeaderObj = ProcessingHeader()
1132 1132
1133 1133 if not(basicHeaderObj.read(fp)):
1134 1134 fp.close()
1135 1135 return False
1136 1136
1137 1137 if not(systemHeaderObj.read(fp)):
1138 1138 fp.close()
1139 1139 return False
1140 1140
1141 1141 if not(radarControllerHeaderObj.read(fp)):
1142 1142 fp.close()
1143 1143 return False
1144 1144
1145 1145 if not(processingHeaderObj.read(fp)):
1146 1146 fp.close()
1147 1147 return False
1148 1148
1149 1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 1150 else:
1151 1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152 1152
1153 1153 fp.close()
1154 1154
1155 1155 fileSize = os.path.getsize(filename)
1156 1156 currentSize = fileSize - currentPosition
1157 1157
1158 1158 if currentSize < neededSize:
1159 1159 if msgFlag and (msg != None):
1160 1160 print msg
1161 1161 return False
1162 1162
1163 1163 return True
1164 1164
1165 1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166 1166
1167 1167 path_empty = True
1168 1168
1169 1169 dateList = []
1170 1170 pathList = []
1171 1171
1172 1172 multi_path = path.split(',')
1173 1173
1174 1174 if not walk:
1175 1175
1176 1176 for single_path in multi_path:
1177 1177
1178 1178 if not os.path.isdir(single_path):
1179 1179 continue
1180 1180
1181 1181 fileList = glob.glob1(single_path, "*" + ext)
1182 1182
1183 1183 if not fileList:
1184 1184 continue
1185 1185
1186 1186 path_empty = False
1187 1187
1188 1188 fileList.sort()
1189 1189
1190 1190 for thisFile in fileList:
1191 1191
1192 1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 1193 continue
1194 1194
1195 1195 if not isRadarFile(thisFile):
1196 1196 continue
1197 1197
1198 1198 if not isFileInDateRange(thisFile, startDate, endDate):
1199 1199 continue
1200 1200
1201 1201 thisDate = getDateFromRadarFile(thisFile)
1202 1202
1203 1203 if thisDate in dateList:
1204 1204 continue
1205 1205
1206 1206 dateList.append(thisDate)
1207 1207 pathList.append(single_path)
1208 1208
1209 1209 else:
1210 1210 for single_path in multi_path:
1211 1211
1212 1212 if not os.path.isdir(single_path):
1213 1213 continue
1214 1214
1215 1215 dirList = []
1216 1216
1217 1217 for thisPath in os.listdir(single_path):
1218 1218
1219 1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 1220 continue
1221 1221
1222 1222 if not isRadarFolder(thisPath):
1223 1223 continue
1224 1224
1225 1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 1226 continue
1227 1227
1228 1228 dirList.append(thisPath)
1229 1229
1230 1230 if not dirList:
1231 1231 continue
1232 1232
1233 1233 dirList.sort()
1234 1234
1235 1235 for thisDir in dirList:
1236 1236
1237 1237 datapath = os.path.join(single_path, thisDir, expLabel)
1238 1238 fileList = glob.glob1(datapath, "*" + ext)
1239 1239
1240 1240 if not fileList:
1241 1241 continue
1242 1242
1243 1243 path_empty = False
1244 1244
1245 1245 thisDate = getDateFromRadarFolder(thisDir)
1246 1246
1247 1247 pathList.append(datapath)
1248 1248 dateList.append(thisDate)
1249 1249
1250 1250 dateList.sort()
1251 1251
1252 1252 if walk:
1253 1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 1254 else:
1255 1255 pattern_path = multi_path[0]
1256 1256
1257 1257 if path_empty:
1258 1258 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1259 1259 else:
1260 1260 if not dateList:
1261 1261 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1262 1262
1263 1263 if include_path:
1264 1264 return dateList, pathList
1265 1265
1266 1266 return dateList
1267 1267
1268 1268 def setup(self,
1269 1269 path=None,
1270 1270 startDate=None,
1271 1271 endDate=None,
1272 1272 startTime=datetime.time(0, 0, 0),
1273 1273 endTime=datetime.time(23, 59, 59),
1274 1274 set=None,
1275 1275 expLabel="",
1276 1276 ext=None,
1277 1277 online=False,
1278 1278 delay=60,
1279 1279 walk=True,
1280 1280 getblock=False,
1281 1281 nTxs=1,
1282 1282 realtime=False,
1283 1283 blocksize=None,
1284 1284 blocktime=None,
1285 1285 skip=None,
1286 1286 cursor=None,
1287 1287 warnings=True,
1288 1288 verbose=True,
1289 1289 server=None,
1290 1290 format=None,
1291 1291 oneDDict=None,
1292 1292 twoDDict=None,
1293 1293 ind2DList=None):
1294 1294 if server is not None:
1295 1295 if 'tcp://' in server:
1296 1296 address = server
1297 1297 else:
1298 1298 address = 'ipc:///tmp/%s' % server
1299 1299 self.server = address
1300 1300 self.context = zmq.Context()
1301 1301 self.receiver = self.context.socket(zmq.PULL)
1302 1302 self.receiver.connect(self.server)
1303 1303 time.sleep(0.5)
1304 1304 print '[Starting] ReceiverData from {}'.format(self.server)
1305 1305 else:
1306 1306 self.server = None
1307 1307 if path == None:
1308 1308 raise ValueError, "[Reading] The path is not valid"
1309 1309
1310 1310 if ext == None:
1311 1311 ext = self.ext
1312 1312
1313 1313 if online:
1314 1314 print "[Reading] Searching files in online mode..."
1315 1315
1316 1316 for nTries in range(self.nTries):
1317 1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319 1319
1320 1320 if fullpath:
1321 1321 break
1322 1322
1323 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1324 sleep(self.delay)
1323 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (delay, path, nTries + 1)
1324 sleep(delay)
1325 1325
1326 if not(fullpath):
1327 print "[Reading] There 'isn't any valid file in %s" % path
1326 if not(fullpath):
1327 raise schainpy.admin.SchainWarning('There isn\'t any valid file in {}'.format(path))
1328 1328 return
1329 1329
1330 1330 self.year = year
1331 1331 self.doy = doy
1332 1332 self.set = set - 1
1333 1333 self.path = path
1334 1334 self.foldercounter = foldercounter
1335 1335 last_set = None
1336 1336 else:
1337 1337 print "[Reading] Searching files in offline mode ..."
1338 1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 1339 startTime=startTime, endTime=endTime,
1340 1340 set=set, expLabel=expLabel, ext=ext,
1341 1341 walk=walk, cursor=cursor,
1342 1342 skip=skip)
1343 1343
1344 1344 if not(pathList):
1345 1345 self.fileIndex = -1
1346 1346 self.pathList = []
1347 1347 self.filenameList = []
1348 1348 return
1349 1349
1350 1350 self.fileIndex = -1
1351 1351 self.pathList = pathList
1352 1352 self.filenameList = filenameList
1353 1353 file_name = os.path.basename(filenameList[-1])
1354 1354 basename, ext = os.path.splitext(file_name)
1355 1355 last_set = int(basename[-3:])
1356 1356
1357 1357 self.online = online
1358 1358 self.realtime = realtime
1359 1359 self.delay = delay
1360 1360 ext = ext.lower()
1361 1361 self.ext = ext
1362 1362 self.getByBlock = getblock
1363 1363 self.nTxs = nTxs
1364 1364 self.startTime = startTime
1365 1365 self.endTime = endTime
1366 1366 self.endDate = endDate
1367 1367 self.startDate = startDate
1368 1368 # Added-----------------
1369 1369 self.selBlocksize = blocksize
1370 1370 self.selBlocktime = blocktime
1371 1371
1372 1372 # Verbose-----------
1373 1373 self.verbose = verbose
1374 1374 self.warnings = warnings
1375 1375
1376 1376 if not(self.setNextFile()):
1377 1377 if (startDate != None) and (endDate != None):
1378 1378 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1379 1379 elif startDate != None:
1380 1380 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1381 1381 else:
1382 1382 print "[Reading] No files"
1383 1383
1384 1384 self.fileIndex = -1
1385 1385 self.pathList = []
1386 1386 self.filenameList = []
1387 1387 return
1388 1388
1389 1389 # self.getBasicHeader()
1390 1390
1391 1391 if last_set != None:
1392 1392 self.dataOut.last_block = last_set * \
1393 1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 1394 return
1395 1395
1396 1396 def getBasicHeader(self):
1397 1397
1398 1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400 1400
1401 1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402 1402
1403 1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404 1404
1405 1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406 1406
1407 1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408 1408
1409 1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410 1410
1411 1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412 1412
1413 1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414 1414
1415 1415 def getFirstHeader(self):
1416 1416
1417 1417 raise NotImplementedError
1418 1418
1419 1419 def getData(self):
1420 1420
1421 1421 raise NotImplementedError
1422 1422
1423 1423 def hasNotDataInBuffer(self):
1424 1424
1425 1425 raise NotImplementedError
1426 1426
1427 1427 def readBlock(self):
1428 1428
1429 1429 raise NotImplementedError
1430 1430
1431 1431 def isEndProcess(self):
1432 1432
1433 1433 return self.flagNoMoreFiles
1434 1434
1435 1435 def printReadBlocks(self):
1436 1436
1437 1437 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1438 1438
1439 1439 def printTotalBlocks(self):
1440 1440
1441 1441 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1442 1442
1443 1443 def printNumberOfBlock(self):
1444 1444 'SPAM!'
1445 1445
1446 1446 # if self.flagIsNewBlock:
1447 1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 1448 # self.processingHeaderObj.dataBlocksPerFile,
1449 1449 # self.dataOut.datatime.ctime())
1450 1450
1451 1451 def printInfo(self):
1452 1452
1453 1453 if self.__printInfo == False:
1454 1454 return
1455 1455
1456 1456 self.basicHeaderObj.printInfo()
1457 1457 self.systemHeaderObj.printInfo()
1458 1458 self.radarControllerHeaderObj.printInfo()
1459 1459 self.processingHeaderObj.printInfo()
1460 1460
1461 1461 self.__printInfo = False
1462 1462
1463 1463 def run(self,
1464 1464 path=None,
1465 1465 startDate=None,
1466 1466 endDate=None,
1467 1467 startTime=datetime.time(0, 0, 0),
1468 1468 endTime=datetime.time(23, 59, 59),
1469 1469 set=None,
1470 1470 expLabel="",
1471 1471 ext=None,
1472 1472 online=False,
1473 1473 delay=60,
1474 1474 walk=True,
1475 1475 getblock=False,
1476 1476 nTxs=1,
1477 1477 realtime=False,
1478 1478 blocksize=None,
1479 1479 blocktime=None,
1480 1480 skip=None,
1481 1481 cursor=None,
1482 1482 warnings=True,
1483 1483 server=None,
1484 1484 verbose=True,
1485 1485 format=None,
1486 1486 oneDDict=None,
1487 1487 twoDDict=None,
1488 1488 ind2DList=None, **kwargs):
1489 1489
1490 1490 if not(self.isConfig):
1491 1491 self.setup(path=path,
1492 1492 startDate=startDate,
1493 1493 endDate=endDate,
1494 1494 startTime=startTime,
1495 1495 endTime=endTime,
1496 1496 set=set,
1497 1497 expLabel=expLabel,
1498 1498 ext=ext,
1499 1499 online=online,
1500 1500 delay=delay,
1501 1501 walk=walk,
1502 1502 getblock=getblock,
1503 1503 nTxs=nTxs,
1504 1504 realtime=realtime,
1505 1505 blocksize=blocksize,
1506 1506 blocktime=blocktime,
1507 1507 skip=skip,
1508 1508 cursor=cursor,
1509 1509 warnings=warnings,
1510 1510 server=server,
1511 1511 verbose=verbose,
1512 1512 format=format,
1513 1513 oneDDict=oneDDict,
1514 1514 twoDDict=twoDDict,
1515 1515 ind2DList=ind2DList)
1516 1516 self.isConfig = True
1517 1517 if server is None:
1518 1518 self.getData()
1519 1519 else:
1520 1520 self.getFromServer()
1521 1521
1522 1522
1523 1523 class JRODataWriter(JRODataIO):
1524 1524
1525 1525 """
1526 1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 1527 de los datos siempre se realiza por bloques.
1528 1528 """
1529 1529
1530 1530 blockIndex = 0
1531 1531
1532 1532 path = None
1533 1533
1534 1534 setFile = None
1535 1535
1536 1536 profilesPerBlock = None
1537 1537
1538 1538 blocksPerFile = None
1539 1539
1540 1540 nWriteBlocks = 0
1541 1541
1542 1542 fileDate = None
1543 1543
1544 1544 def __init__(self, dataOut=None):
1545 1545 raise NotImplementedError
1546 1546
1547 1547 def hasAllDataInBuffer(self):
1548 1548 raise NotImplementedError
1549 1549
1550 1550 def setBlockDimension(self):
1551 1551 raise NotImplementedError
1552 1552
1553 1553 def writeBlock(self):
1554 1554 raise NotImplementedError
1555 1555
1556 1556 def putData(self):
1557 1557 raise NotImplementedError
1558 1558
1559 1559 def getProcessFlags(self):
1560 1560
1561 1561 processFlags = 0
1562 1562
1563 1563 dtype_index = get_dtype_index(self.dtype)
1564 1564 procflag_dtype = get_procflag_dtype(dtype_index)
1565 1565
1566 1566 processFlags += procflag_dtype
1567 1567
1568 1568 if self.dataOut.flagDecodeData:
1569 1569 processFlags += PROCFLAG.DECODE_DATA
1570 1570
1571 1571 if self.dataOut.flagDeflipData:
1572 1572 processFlags += PROCFLAG.DEFLIP_DATA
1573 1573
1574 1574 if self.dataOut.code is not None:
1575 1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576 1576
1577 1577 if self.dataOut.nCohInt > 1:
1578 1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579 1579
1580 1580 if self.dataOut.type == "Spectra":
1581 1581 if self.dataOut.nIncohInt > 1:
1582 1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583 1583
1584 1584 if self.dataOut.data_dc is not None:
1585 1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586 1586
1587 1587 if self.dataOut.flagShiftFFT:
1588 1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589 1589
1590 1590 return processFlags
1591 1591
1592 1592 def setBasicHeader(self):
1593 1593
1594 1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 1595 self.basicHeaderObj.version = self.versionFile
1596 1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597 1597
1598 1598 utc = numpy.floor(self.dataOut.utctime)
1599 1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600 1600
1601 1601 self.basicHeaderObj.utc = utc
1602 1602 self.basicHeaderObj.miliSecond = milisecond
1603 1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606 1606
1607 1607 def setFirstHeader(self):
1608 1608 """
1609 1609 Obtiene una copia del First Header
1610 1610
1611 1611 Affected:
1612 1612
1613 1613 self.basicHeaderObj
1614 1614 self.systemHeaderObj
1615 1615 self.radarControllerHeaderObj
1616 1616 self.processingHeaderObj self.
1617 1617
1618 1618 Return:
1619 1619 None
1620 1620 """
1621 1621
1622 1622 raise NotImplementedError
1623 1623
1624 1624 def __writeFirstHeader(self):
1625 1625 """
1626 1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627 1627
1628 1628 Affected:
1629 1629 __dataType
1630 1630
1631 1631 Return:
1632 1632 None
1633 1633 """
1634 1634
1635 1635 # CALCULAR PARAMETROS
1636 1636
1637 1637 sizeLongHeader = self.systemHeaderObj.size + \
1638 1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640 1640
1641 1641 self.basicHeaderObj.write(self.fp)
1642 1642 self.systemHeaderObj.write(self.fp)
1643 1643 self.radarControllerHeaderObj.write(self.fp)
1644 1644 self.processingHeaderObj.write(self.fp)
1645 1645
1646 1646 def __setNewBlock(self):
1647 1647 """
1648 1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649 1649
1650 1650 Return:
1651 1651 0 : si no pudo escribir nada
1652 1652 1 : Si escribio el Basic el First Header
1653 1653 """
1654 1654 if self.fp == None:
1655 1655 self.setNextFile()
1656 1656
1657 1657 if self.flagIsNewFile:
1658 1658 return 1
1659 1659
1660 1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 1661 self.basicHeaderObj.write(self.fp)
1662 1662 return 1
1663 1663
1664 1664 if not(self.setNextFile()):
1665 1665 return 0
1666 1666
1667 1667 return 1
1668 1668
1669 1669 def writeNextBlock(self):
1670 1670 """
1671 1671 Selecciona el bloque siguiente de datos y los escribe en un file
1672 1672
1673 1673 Return:
1674 1674 0 : Si no hizo pudo escribir el bloque de datos
1675 1675 1 : Si no pudo escribir el bloque de datos
1676 1676 """
1677 1677 if not(self.__setNewBlock()):
1678 1678 return 0
1679 1679
1680 1680 self.writeBlock()
1681 1681
1682 1682 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1683 1683 self.processingHeaderObj.dataBlocksPerFile)
1684 1684
1685 1685 return 1
1686 1686
1687 1687 def setNextFile(self):
1688 1688 """
1689 1689 Determina el siguiente file que sera escrito
1690 1690
1691 1691 Affected:
1692 1692 self.filename
1693 1693 self.subfolder
1694 1694 self.fp
1695 1695 self.setFile
1696 1696 self.flagIsNewFile
1697 1697
1698 1698 Return:
1699 1699 0 : Si el archivo no puede ser escrito
1700 1700 1 : Si el archivo esta listo para ser escrito
1701 1701 """
1702 1702 ext = self.ext
1703 1703 path = self.path
1704 1704
1705 1705 if self.fp != None:
1706 1706 self.fp.close()
1707 1707
1708 1708 timeTuple = time.localtime(self.dataOut.utctime)
1709 1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710 1710
1711 1711 fullpath = os.path.join(path, subfolder)
1712 1712 setFile = self.setFile
1713 1713
1714 1714 if not(os.path.exists(fullpath)):
1715 1715 os.mkdir(fullpath)
1716 1716 setFile = -1 # inicializo mi contador de seteo
1717 1717 else:
1718 1718 filesList = os.listdir(fullpath)
1719 1719 if len(filesList) > 0:
1720 1720 filesList = sorted(filesList, key=str.lower)
1721 1721 filen = filesList[-1]
1722 1722 # el filename debera tener el siguiente formato
1723 1723 # 0 1234 567 89A BCDE (hex)
1724 1724 # x YYYY DDD SSS .ext
1725 1725 if isNumber(filen[8:11]):
1726 1726 # inicializo mi contador de seteo al seteo del ultimo file
1727 1727 setFile = int(filen[8:11])
1728 1728 else:
1729 1729 setFile = -1
1730 1730 else:
1731 1731 setFile = -1 # inicializo mi contador de seteo
1732 1732
1733 1733 setFile += 1
1734 1734
1735 1735 # If this is a new day it resets some values
1736 1736 if self.dataOut.datatime.date() > self.fileDate:
1737 1737 setFile = 0
1738 1738 self.nTotalBlocks = 0
1739 1739
1740 1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742 1742
1743 1743 filename = os.path.join(path, subfolder, filen)
1744 1744
1745 1745 fp = open(filename, 'wb')
1746 1746
1747 1747 self.blockIndex = 0
1748 1748
1749 1749 # guardando atributos
1750 1750 self.filename = filename
1751 1751 self.subfolder = subfolder
1752 1752 self.fp = fp
1753 1753 self.setFile = setFile
1754 1754 self.flagIsNewFile = 1
1755 1755 self.fileDate = self.dataOut.datatime.date()
1756 1756
1757 1757 self.setFirstHeader()
1758 1758
1759 1759 print '[Writing] Opening file: %s' % self.filename
1760 1760
1761 1761 self.__writeFirstHeader()
1762 1762
1763 1763 return 1
1764 1764
1765 1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 1766 """
1767 1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768 1768
1769 1769 Inputs:
1770 1770 path : directory where data will be saved
1771 1771 profilesPerBlock : number of profiles per block
1772 1772 set : initial file set
1773 1773 datatype : An integer number that defines data type:
1774 1774 0 : int8 (1 byte)
1775 1775 1 : int16 (2 bytes)
1776 1776 2 : int32 (4 bytes)
1777 1777 3 : int64 (8 bytes)
1778 1778 4 : float32 (4 bytes)
1779 1779 5 : double64 (8 bytes)
1780 1780
1781 1781 Return:
1782 1782 0 : Si no realizo un buen seteo
1783 1783 1 : Si realizo un buen seteo
1784 1784 """
1785 1785
1786 1786 if ext == None:
1787 1787 ext = self.ext
1788 1788
1789 1789 self.ext = ext.lower()
1790 1790
1791 1791 self.path = path
1792 1792
1793 1793 if set is None:
1794 1794 self.setFile = -1
1795 1795 else:
1796 1796 self.setFile = set - 1
1797 1797
1798 1798 self.blocksPerFile = blocksPerFile
1799 1799
1800 1800 self.profilesPerBlock = profilesPerBlock
1801 1801
1802 1802 self.dataOut = dataOut
1803 1803 self.fileDate = self.dataOut.datatime.date()
1804 1804 # By default
1805 1805 self.dtype = self.dataOut.dtype
1806 1806
1807 1807 if datatype is not None:
1808 1808 self.dtype = get_numpy_dtype(datatype)
1809 1809
1810 1810 if not(self.setNextFile()):
1811 1811 print "[Writing] There isn't a next file"
1812 1812 return 0
1813 1813
1814 1814 self.setBlockDimension()
1815 1815
1816 1816 return 1
1817 1817
1818 1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819 1819
1820 1820 if not(self.isConfig):
1821 1821
1822 1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 1823 set=set, ext=ext, datatype=datatype, **kwargs)
1824 1824 self.isConfig = True
1825 1825
1826 1826 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now