##// END OF EJS Templates
schain
RSS
Update2 for EW-Drifts
Percy Condor -
r1383:3e971ac8dea1
parent child
Show More
Side by Side Unified
Context file:
r1383:3e971ac8dea1 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,505 +1,505
1 1 """
2 2 The admin module contains all administrative classes relating to the schain python api.
3 3
4 4 The main role of this module is to send some reports. It contains a
5 5 notification class and a standard error handing class.
6 6
7 7 $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
8 8 """
9 9 import os
10 10 import sys
11 11 import time
12 12 import traceback
13 13 import smtplib
14 14 if sys.version[0] == '3':
15 15 from configparser import ConfigParser
16 16 else:
17 17 from ConfigParser import ConfigParser
18 18 import io
19 19 from threading import Thread
20 20 from multiprocessing import Process
21 21 from email.mime.text import MIMEText
22 22 from email.mime.application import MIMEApplication
23 23 from email.mime.multipart import MIMEMultipart
24 24
25 25 import schainpy
26 26 from schainpy.utils import log
27 27 from schainpy.model.graphics.jroplot_base import popup
28 28
29 29 def get_path():
30 30 '''
31 31 Return schainpy path
32 32 '''
33 33
34 34 try:
35 35 root = __file__
36 36 if os.path.islink(root):
37 37 root = os.path.realpath(root)
38 38
39 39 return os.path.dirname(os.path.abspath(root))
40 40 except:
41 41 log.error('I am sorry, but something is wrong... __file__ not found')
42 42
43 43 class Alarm(Process):
44 44 '''
45 45 modes:
46 46 0 - All
47 47 1 - Send email
48 48 2 - Popup message
49 49 3 - Sound alarm
50 50 4 - Send to alarm system TODO
51 51 '''
52 52
53 53 def __init__(self, modes=[], **kwargs):
54 54 Process.__init__(self)
55 55 self.modes = modes
56 56 self.kwargs = kwargs
57 57
58 58 @staticmethod
59 59 def play_sound():
60 60 sound = os.path.join(get_path(), 'alarm1.oga')
61 61 if os.path.exists(sound):
62 62 for __ in range(2):
63 63 os.system('paplay {}'.format(sound))
64 64 time.sleep(0.5)
65 65 else:
66 66 log.warning('Unable to play alarm, sound file not found', 'ADMIN')
67 67
68 68 @staticmethod
69 69 def send_email(**kwargs):
70 70 notifier = SchainNotify()
71 71 notifier.notify(**kwargs)
72 72
73 73 @staticmethod
74 74 def show_popup(message):
75 75 if isinstance(message, list):
76 76 message = message[-1]
77 77 popup(message)
78 78
79 79 @staticmethod
80 80 def send_alarm():
81 81 pass
82 82
83 83 @staticmethod
84 84 def get_kwargs(kwargs, keys):
85 85 ret = {}
86 86 for key in keys:
87 87 ret[key] = kwargs[key]
88 88 return ret
89 89
90 90 def run(self):
91 91 tasks = {
92 92 1 : self.send_email,
93 93 2 : self.show_popup,
94 94 3 : self.play_sound,
95 95 4 : self.send_alarm,
96 96 }
97 97
98 98 tasks_args = {
99 99 1: ['email', 'message', 'subject', 'subtitle', 'filename'],
100 100 2: ['message'],
101 101 3: [],
102 102 4: [],
103 103 }
104 104 procs = []
105 105 for mode in self.modes:
106 106 if 0 in self.modes:
107 107 for x in tasks:
108 108 t = Thread(target=tasks[x], kwargs=self.get_kwargs(self.kwargs, tasks_args[x]))
109 109 t.start()
110 110 procs.append(t)
111 111 break
112 112 else:
113 113 t = Thread(target=tasks[mode], kwargs=self.get_kwargs(self.kwargs, tasks_args[mode]))
114 114 t.start()
115 115 procs.append(t)
116 116 for t in procs:
117 117 t.join()
118 118
119 119
120 120 class SchainConfigure():
121 121
122 122 __DEFAULT_ADMINISTRATOR_EMAIL = "juan.espinoza@jro.igp.gob.pe"
123 123 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
124 124 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
125 125 __DEFAULT_SENDER_PASS = ""
126 126
127 127 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
128 128 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
129 129 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
130 130 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
131 131
132 def __init__(self, initFile = None):
132 def __init__(self, initFile=None):
133 133
134 134 # Set configuration file
135 135 if (initFile == None):
136 136 self.__confFilePath = "/etc/schain.conf"
137 137 else:
138 138 self.__confFilePath = initFile
139 139
140 140 # open configuration file
141 141 try:
142 142 self.__confFile = open(self.__confFilePath, "r")
143 143 except IOError:
144 144 # can't read from file - use all hard-coded values
145 145 self.__initFromHardCode()
146 146 return
147 147
148 148 # create Parser using standard module ConfigParser
149 149 self.__parser = ConfigParser()
150 150
151 151 # read conf file into a StringIO with "[madrigal]\n" section heading prepended
152 152 strConfFile = io.StringIO("[schain]\n" + self.__confFile.read())
153 153
154 154 # parse StringIO configuration file
155 155 self.__parser.readfp(strConfFile)
156 156
157 157 # read information from configuration file
158 158 self.__readConfFile()
159 159
160 160 # close conf file
161 161 self.__confFile.close()
162 162
163 163
164 164 def __initFromHardCode(self):
165 165
166 166 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
167 167 self.__sender_pass = self.__DEFAULT_SENDER_PASS
168 168 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
169 169 self.__email_server = self.__DEFAULT_EMAIL_SERVER
170 170
171 171 def __readConfFile(self):
172 172 """__readConfFile is a private helper function that reads information from the parsed config file.
173 173
174 174 Inputs: None
175 175
176 176 Returns: Void.
177 177
178 178 Affects: Initializes class member variables that are found in the config file.
179 179
180 180 Exceptions: MadrigalError thrown if any key not found.
181 181 """
182 182
183 183 # get the sender email
184 184 try:
185 185 self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
186 186 except:
187 187 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
188 188
189 189 # get the sender password
190 190 try:
191 191 self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
192 192 except:
193 193 self.__sender_pass = self.__DEFAULT_SENDER_PASS
194 194
195 195 # get the administrator email
196 196 try:
197 197 self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
198 198 except:
199 199 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
200 200
201 201 # get the server email
202 202 try:
203 203 self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
204 204 except:
205 205 self.__email_server = self.__DEFAULT_EMAIL_SERVER
206 206
207 207 def getEmailServer(self):
208 208
209 209 return self.__email_server
210 210
211 211 def getSenderEmail(self):
212 212
213 213 return self.__sender_email
214 214
215 215 def getSenderPass(self):
216 216
217 217 return self.__sender_pass
218 218
219 219 def getAdminEmail(self):
220 220
221 221 return self.__admin_email
222 222
223 223 class SchainNotify:
224 224 """SchainNotify is an object used to send messages to an administrator about a Schain software.
225 225
226 226 This object provides functions needed to send messages to an administrator about a Schain , for now
227 227 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
228 228
229 229 Usage example:
230 230
231 231 import schainpy.admin
232 232
233 233 try:
234 234
235 235 adminObj = schainpy.admin.SchainNotify()
236 236 adminObj.sendAlert('This is important!', 'Important Message')
237 237
238 238 except schainpy.admin.SchainError, e:
239 239
240 240 print e.getExceptionStr()
241 241
242 242
243 243 Non-standard Python modules used:
244 244 None
245 245
246 246 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
247 247 throwing exceptions, since this is the class that will generally be called when there is a problem.
248 248
249 249 Change history:
250 250
251 251 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Dec. 1, 2015
252 252 """
253 253
254 #constants
254 # constants
255 255
256 256 def __init__(self):
257 257 """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
258 258
259 259 Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
260 260 this is the class that will generally be called when there is a problem.
261 261
262 262 Inputs: Existing SchainDB object, by default = None.
263 263
264 264 Returns: void
265 265
266 266 Affects: Initializes self.__binDir.
267 267
268 268 Exceptions: None.
269 269 """
270 270
271 271 # note that the main configuration file is unavailable
272 272 # the best that can be done is send an email to root using localhost mailserver
273 273 confObj = SchainConfigure()
274 274
275 275 self.__emailFromAddress = confObj.getSenderEmail()
276 276 self.__emailPass = confObj.getSenderPass()
277 277 self.__emailToAddress = confObj.getAdminEmail()
278 self.__emailServer = confObj.getEmailServer()
278 self.__emailServer = confObj.getEmailServer()
279 279
280 280 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
281 281
282 282 if not email_to:
283 283 return 0
284 284
285 285 if not self.__emailServer:
286 286 return 0
287 287
288 288 log.success('Sending email to {}...'.format(email_to), 'System')
289 289
290 290 msg = MIMEMultipart()
291 291 msg['Subject'] = subject
292 292 msg['From'] = "SChain API (v{}) <{}>".format(schainpy.__version__, email_from)
293 293 msg['Reply-to'] = email_from
294 294 msg['To'] = email_to
295 295
296 296 # That is what u see if dont have an email reader:
297 297 msg.preamble = 'SChainPy'
298 298
299 299 if html_format:
300 message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
300 message = "<h1> %s </h1>" % subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
301 301 message = "<html>\n" + message + '</html>'
302 302
303 303 # This is the textual part:
304 304 part = MIMEText(message, "html")
305 305 else:
306 306 message = subject + "\n" + subtitle + "\n" + message
307 307 part = MIMEText(message)
308 308
309 309 msg.attach(part)
310 310
311 311 if filename and os.path.isfile(filename):
312 312 # This is the binary part(The Attachment):
313 part = MIMEApplication(open(filename,"rb").read())
314 part.add_header('Content-Disposition',
313 part = MIMEApplication(open(filename, "rb").read())
314 part.add_header('Content-Disposition',
315 315 'attachment',
316 316 filename=os.path.basename(filename))
317 317 msg.attach(part)
318 318
319 319 # Create an instance in SMTP server
320 320 try:
321 321 smtp = smtplib.SMTP(self.__emailServer)
322 322 except:
323 323 log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
324 324 return 0
325 325
326 326 # Start the server:
327 327 # smtp.ehlo()
328 328 if self.__emailPass:
329 329 smtp.login(self.__emailFromAddress, self.__emailPass)
330 330
331 331 # Send the email
332 332 try:
333 333 smtp.sendmail(msg['From'], msg['To'], msg.as_string())
334 334 except:
335 335 log.error('Could not send the email to {}'.format(msg['To']), 'System')
336 336 smtp.quit()
337 337 return 0
338 338
339 339 smtp.quit()
340 340
341 341 log.success('Email sent ', 'System')
342 342
343 343 return 1
344 344
345 def sendAlert(self, message, subject = "", subtitle="", filename=""):
345 def sendAlert(self, message, subject="", subtitle="", filename=""):
346 346 """sendAlert sends an email with the given message and optional title.
347 347
348 348 Inputs: message (string), and optional title (string)
349 349
350 350 Returns: void
351 351
352 352 Affects: none
353 353
354 354 Exceptions: None.
355 355 """
356 356
357 357 if not self.__emailToAddress:
358 358 return 0
359 359
360 print("***** Sending alert to %s *****" %self.__emailToAddress)
360 print("***** Sending alert to %s *****" % self.__emailToAddress)
361 361 # set up message
362 362
363 sent=self.sendEmail(email_from=self.__emailFromAddress,
363 sent = self.sendEmail(email_from=self.__emailFromAddress,
364 364 email_to=self.__emailToAddress,
365 365 subject=subject,
366 366 message=message,
367 subtitle=subtitle,
367 subtitle=subtitle,
368 368 filename=filename)
369 369
370 370 if not sent:
371 371 return 0
372 372
373 373 return 1
374 374
375 def notify(self, email, message, subject = "", subtitle="", filename=""):
375 def notify(self, email, message, subject="", subtitle="", filename=""):
376 376 """notify sends an email with the given message and title to email.
377 377
378 378 Inputs: email (string), message (string), and subject (string)
379 379
380 380 Returns: void
381 381
382 382 Affects: none
383 383
384 384 Exceptions: None.
385 385 """
386 386
387 387 if email is None:
388 388 email = self.__emailToAddress
389 389
390 390 self.sendEmail(
391 391 email_from=self.__emailFromAddress,
392 392 email_to=email,
393 393 subject=subject,
394 394 message=message,
395 subtitle=subtitle,
395 subtitle=subtitle,
396 396 filename=filename
397 397 )
398 398
399 399
400 400 class SchainError(Exception):
401 401 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
402 402
403 403 Usage example:
404 404
405 405 import sys, traceback
406 406 import schainpy.admin
407 407
408 408 try:
409 409
410 410 test = open('ImportantFile.txt', 'r')
411 411
412 412 except:
413 413
414 414 raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
415 415 traceback.format_exception(sys.exc_info()[0],
416 416 sys.exc_info()[1],
417 417 sys.exc_info()[2]))
418 418 """
419 419
420 420
421 421 def __init__(self, strInterpretation, exceptionList=None):
422 422 """ __init__ gathers the interpretation string along with all information from sys.exc_info().
423 423
424 424 Inputs:
425 425 strIntepretation - A string representing the programmer's interpretation of
426 426 why the exception occurred
427 427
428 428 exceptionList - a list of strings completely describing the exception.
429 429 Generated by traceback.format_exception(sys.exc_info()[0],
430 430 sys.exc_info()[1],
431 431 sys.exc_info()[2])
432 432
433 433 Returns: Void.
434 434
435 435 Affects: Initializes class member variables _strInterp, _strExcList.
436 436
437 437 Exceptions: None.
438 438 """
439 439
440 440 if not exceptionList:
441 441 exceptionList = traceback.format_exception(sys.exc_info()[0],
442 442 sys.exc_info()[1],
443 443 sys.exc_info()[2])
444 444
445 445 self._strInterp = strInterpretation
446 446 self._strExcList = exceptionList
447 447
448 448
449 449 def getExceptionStr(self):
450 450 """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
451 451
452 452 Inputs: None
453 453
454 454 Returns: A formatted string ready for printing completely describing the exception.
455 455
456 456 Affects: None
457 457
458 458 Exceptions: None.
459 459 """
460 460 excStr = ''
461 461 excStr = excStr + self._strInterp + '\n\n'
462 462
463 463 if self._strExcList != None:
464 464 for item in self._strExcList:
465 465 excStr = excStr + str(item) + '\n'
466 466
467 467 return excStr
468 468
469 469 def __str__(self):
470 470
471 471 return(self.getExceptionStr())
472 472
473 473
474 474 def getExceptionHtml(self):
475 475 """ getExceptionHtml returns an Html formatted string completely describing the exception.
476 476
477 477 Inputs: None
478 478
479 479 Returns: A formatted string ready for printing completely describing the exception.
480 480
481 481 Affects: None
482 482
483 483 Exceptions: None.
484 484 """
485 485
486 486 excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
487 487 excStr = excStr + self._strInterp + '\n<BR>\n'
488 488
489 489 if self._strExcList != None:
490 490 for item in self._strExcList:
491 491 excStr = excStr + str(item) + '\n<BR>'
492 492
493 493 return excStr
494 494
495 495 class SchainWarning(Exception):
496 496 pass
497 497
498 498
499 499 if __name__ == '__main__':
500 500
501 501 test = SchainNotify()
502 502
503 503 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
504 504
505 print('Hopefully message sent - check.') No newline at end of file
505 print('Hopefully message sent - check.')
Show file before | Show file after | Hide comments
@@ -1,238 +1,238
1 1 import click
2 2 import subprocess
3 3 import os
4 4 import sys
5 5 import glob
6 6 import schainpy
7 7 from schainpy.controller import Project
8 8 from schainpy.model import Operation, ProcessingUnit
9 9 from schainpy.utils import log
10 10 from importlib import import_module
11 11 from pydoc import locate
12 12 from fuzzywuzzy import process
13 13 from schainpy.cli import templates
14 14 import inspect
15 15 try:
16 16 from queue import Queue
17 17 except:
18 18 from Queue import Queue
19 19
20 20
21 21 def getProcs():
22 22 modules = dir(schainpy.model)
23 23 procs = check_module(modules, 'processing')
24 24 try:
25 25 procs.remove('ProcessingUnit')
26 26 except Exception as e:
27 27 pass
28 28 return procs
29 29
30 30 def getOperations():
31 31 module = dir(schainpy.model)
32 32 noProcs = [x for x in module if not x.endswith('Proc')]
33 33 operations = check_module(noProcs, 'operation')
34 34 try:
35 35 operations.remove('Operation')
36 36 operations.remove('Figure')
37 37 operations.remove('Plot')
38 38 except Exception as e:
39 39 pass
40 40 return operations
41 41
42 42 def getArgs(op):
43 43 module = locate('schainpy.model.{}'.format(op))
44 44 try:
45 45 obj = module(1, 2, 3, Queue())
46 46 except:
47 47 obj = module()
48 48
49 49 if hasattr(obj, '__attrs__'):
50 50 args = obj.__attrs__
51 51 else:
52 52 if hasattr(obj, 'myrun'):
53 53 args = inspect.getfullargspec(obj.myrun).args
54 54 else:
55 55 args = inspect.getfullargspec(obj.run).args
56 56
57 57 try:
58 58 args.remove('self')
59 59 except Exception as e:
60 60 pass
61 61 try:
62 62 args.remove('dataOut')
63 63 except Exception as e:
64 64 pass
65 65 return args
66 66
67 67 def getDoc(obj):
68 68 module = locate('schainpy.model.{}'.format(obj))
69 69 try:
70 70 obj = module(1, 2, 3, Queue())
71 71 except:
72 72 obj = module()
73 73 return obj.__doc__
74 74
75 75 def getAll():
76 76 modules = getOperations()
77 77 modules.extend(getProcs())
78 78 return modules
79 79
80 80
81 81 def print_version(ctx, param, value):
82 82 if not value or ctx.resilient_parsing:
83 83 return
84 84 click.echo(schainpy.__version__)
85 85 ctx.exit()
86 86
87 87
88 88 PREFIX = 'experiment'
89 89
90 90 @click.command()
91 91 @click.option('--version', '-v', is_flag=True, callback=print_version, help='SChain version', type=str)
92 92 @click.argument('command', default='run', required=True)
93 93 @click.argument('nextcommand', default=None, required=False, type=str)
94 94 def main(command, nextcommand, version):
95 95 """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY V3.0\n
96 96 Available commands:\n
97 97 xml: runs a schain XML generated file\n
98 98 run: runs any python script'\n
99 99 generate: generates a template schain script\n
100 100 list: return a list of available procs and operations\n
101 101 search: return avilable operations, procs or arguments of the given
102 102 operation/proc\n"""
103 103 if command == 'xml':
104 104 runFromXML(nextcommand)
105 105 elif command == 'generate':
106 106 generate()
107 107 elif command == 'test':
108 108 test()
109 109 elif command == 'run':
110 110 runschain(nextcommand)
111 111 elif command == 'search':
112 112 search(nextcommand)
113 113 elif command == 'list':
114 114 cmdlist(nextcommand)
115 115 else:
116 116 log.error('Command {} is not defined'.format(command))
117 117
118 118
119 119 def check_module(possible, instance):
120 120 def check(x):
121 121 try:
122 122 instancia = locate('schainpy.model.{}'.format(x))
123 123 ret = instancia.proc_type == instance
124 124 return ret
125 125 except Exception as e:
126 126 return False
127 127 clean = clean_modules(possible)
128 128 return [x for x in clean if check(x)]
129 129
130 130
131 131 def clean_modules(module):
132 132 noEndsUnder = [x for x in module if not x.endswith('__')]
133 133 noStartUnder = [x for x in noEndsUnder if not x.startswith('__')]
134 134 noFullUpper = [x for x in noStartUnder if not x.isupper()]
135 135 return noFullUpper
136 136
137 137 def cmdlist(nextcommand):
138 138 if nextcommand is None:
139 139 log.error('Missing argument, available arguments: procs, operations', '')
140 140 elif nextcommand == 'procs':
141 141 procs = getProcs()
142 142 log.success(
143 143 'Current ProcessingUnits are:\n {}'.format('\n '.join(procs)), '')
144 144 elif nextcommand == 'operations':
145 145 operations = getOperations()
146 146 log.success('Current Operations are:\n {}'.format(
147 147 '\n '.join(operations)), '')
148 148 else:
149 149 log.error('Wrong argument', '')
150 150
151 151 def search(nextcommand):
152 152 if nextcommand is None:
153 153 log.error('There is no Operation/ProcessingUnit to search', '')
154 154 else:
155 155 try:
156 156 args = getArgs(nextcommand)
157 157 doc = getDoc(nextcommand)
158 158 log.success('{}\n{}\n\nparameters:\n {}'.format(
159 159 nextcommand, doc, ', '.join(args)), ''
160 160 )
161 161 except Exception as e:
162 162 log.error('Module `{}` does not exists'.format(nextcommand), '')
163 163 allModules = getAll()
164 similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1]>80]
164 similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1] > 80]
165 165 log.success('Possible modules are: {}'.format(', '.join(similar)), '')
166 166
167 167 def runschain(nextcommand):
168 168 if nextcommand is None:
169 169 currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
170 170 numberfiles = len(currentfiles)
171 171 if numberfiles > 1:
172 172 log.error('There is more than one file to run')
173 173 elif numberfiles == 1:
174 174 subprocess.call(['python ' + currentfiles[0]], shell=True)
175 175 else:
176 176 log.error('There is no file to run')
177 177 else:
178 178 try:
179 179 subprocess.call(['python ' + nextcommand], shell=True)
180 180 except Exception as e:
181 181 log.error("I cannot run the file. Does it exists?")
182 182
183 183
184 184 def basicInputs():
185 185 inputs = {}
186 186 inputs['name'] = click.prompt(
187 187 'Name of the project', default="project", type=str)
188 188 inputs['desc'] = click.prompt(
189 189 'Enter a description', default="A schain project", type=str)
190 190 inputs['multiprocess'] = click.prompt(
191 191 '''Select data type:
192 192
193 193 - Voltage (*.r): [1]
194 194 - Spectra (*.pdata): [2]
195 195 - Voltage and Spectra (*.r): [3]
196 196
197 197 -->''', type=int)
198 198 inputs['path'] = click.prompt('Data path', default=os.getcwd(
199 199 ), type=click.Path(exists=True, resolve_path=True))
200 200 inputs['startDate'] = click.prompt(
201 201 'Start date', default='1970/01/01', type=str)
202 202 inputs['endDate'] = click.prompt(
203 203 'End date', default='2018/12/31', type=str)
204 204 inputs['startHour'] = click.prompt(
205 205 'Start hour', default='00:00:00', type=str)
206 206 inputs['endHour'] = click.prompt('End hour', default='23:59:59', type=str)
207 207 inputs['figpath'] = inputs['path'] + '/figs'
208 208 return inputs
209 209
210 210
211 211 def generate():
212 212 inputs = basicInputs()
213 213
214 214 if inputs['multiprocess'] == 1:
215 215 current = templates.voltage.format(**inputs)
216 216 elif inputs['multiprocess'] == 2:
217 217 current = templates.spectra.format(**inputs)
218 218 elif inputs['multiprocess'] == 3:
219 219 current = templates.voltagespectra.format(**inputs)
220 220 scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
221 221 script = open(scriptname, 'w')
222 222 try:
223 223 script.write(current)
224 224 log.success('Script {} generated'.format(scriptname))
225 225 except Exception as e:
226 226 log.error('I cannot create the file. Do you have writing permissions?')
227 227
228 228
229 229 def test():
230 230 log.warning('testing')
231 231
232 232
233 233 def runFromXML(filename):
234 234 controller = Project()
235 235 if not controller.readXml(filename):
236 236 return
237 237 controller.start()
238 238 return
Show file before | Show file after | Hide comments
@@ -1,659 +1,659
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """API to create signal chain projects
6 6
7 7 The API is provide through class: Project
8 8 """
9 9
10 10 import re
11 11 import sys
12 12 import ast
13 13 import datetime
14 14 import traceback
15 15 import time
16 16 import multiprocessing
17 17 from multiprocessing import Process, Queue
18 18 from threading import Thread
19 19 from xml.etree.ElementTree import ElementTree, Element, SubElement
20 20
21 21 from schainpy.admin import Alarm, SchainWarning
22 22 from schainpy.model import *
23 23 from schainpy.utils import log
24 24
25 25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 26 multiprocessing.set_start_method('fork')
27 27
28 28 class ConfBase():
29 29
30 30 def __init__(self):
31 31
32 32 self.id = '0'
33 33 self.name = None
34 34 self.priority = None
35 35 self.parameters = {}
36 36 self.object = None
37 37 self.operations = []
38 38
39 39 def getId(self):
40 40
41 41 return self.id
42 42
43 43 def getNewId(self):
44 44
45 45 return int(self.id) * 10 + len(self.operations) + 1
46 46
47 47 def updateId(self, new_id):
48 48
49 49 self.id = str(new_id)
50 50
51 51 n = 1
52 52 for conf in self.operations:
53 53 conf_id = str(int(new_id) * 10 + n)
54 54 conf.updateId(conf_id)
55 55 n += 1
56 56
57 57 def getKwargs(self):
58 58
59 59 params = {}
60 60
61 61 for key, value in self.parameters.items():
62 62 if value not in (None, '', ' '):
63 63 params[key] = value
64 64
65 65 return params
66 66
67 67 def update(self, **kwargs):
68 68
69 69 for key, value in kwargs.items():
70 70 self.addParameter(name=key, value=value)
71 71
72 72 def addParameter(self, name, value, format=None):
73 73 '''
74 74 '''
75 75
76 76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
77 77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
78 78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
79 79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
80 80 else:
81 81 try:
82 82 self.parameters[name] = ast.literal_eval(value)
83 83 except:
84 84 if isinstance(value, str) and ',' in value:
85 85 self.parameters[name] = value.split(',')
86 86 else:
87 87 self.parameters[name] = value
88 88
89 89 def getParameters(self):
90 90
91 91 params = {}
92 92 for key, value in self.parameters.items():
93 93 s = type(value).__name__
94 94 if s == 'date':
95 95 params[key] = value.strftime('%Y/%m/%d')
96 96 elif s == 'time':
97 97 params[key] = value.strftime('%H:%M:%S')
98 98 else:
99 99 params[key] = str(value)
100 100
101 101 return params
102 102
103 103 def makeXml(self, element):
104 104
105 105 xml = SubElement(element, self.ELEMENTNAME)
106 106 for label in self.xml_labels:
107 107 xml.set(label, str(getattr(self, label)))
108 108
109 109 for key, value in self.getParameters().items():
110 110 xml_param = SubElement(xml, 'Parameter')
111 111 xml_param.set('name', key)
112 112 xml_param.set('value', value)
113 113
114 114 for conf in self.operations:
115 115 conf.makeXml(xml)
116 116
117 117 def __str__(self):
118 118
119 119 if self.ELEMENTNAME == 'Operation':
120 120 s = ' {}[id={}]\n'.format(self.name, self.id)
121 121 else:
122 122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
123 123
124 124 for key, value in self.parameters.items():
125 125 if self.ELEMENTNAME == 'Operation':
126 126 s += ' {}: {}\n'.format(key, value)
127 127 else:
128 128 s += ' {}: {}\n'.format(key, value)
129 129
130 130 for conf in self.operations:
131 131 s += str(conf)
132 132
133 133 return s
134 134
135 135 class OperationConf(ConfBase):
136 136
137 137 ELEMENTNAME = 'Operation'
138 138 xml_labels = ['id', 'name']
139 139
140 140 def setup(self, id, name, priority, project_id, err_queue):
141 141
142 142 self.id = str(id)
143 143 self.project_id = project_id
144 144 self.name = name
145 145 self.type = 'other'
146 146 self.err_queue = err_queue
147 147
148 148 def readXml(self, element, project_id, err_queue):
149 149
150 150 self.id = element.get('id')
151 151 self.name = element.get('name')
152 152 self.type = 'other'
153 153 self.project_id = str(project_id)
154 154 self.err_queue = err_queue
155 155
156 156 for elm in element.iter('Parameter'):
157 157 self.addParameter(elm.get('name'), elm.get('value'))
158 158
159 159 def createObject(self):
160 160
161 161 className = eval(self.name)
162 162
163 163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
164 164 kwargs = self.getKwargs()
165 165 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
166 166 opObj.start()
167 167 self.type = 'external'
168 168 else:
169 169 opObj = className()
170 170
171 171 self.object = opObj
172 172 return opObj
173 173
174 174 class ProcUnitConf(ConfBase):
175 175
176 176 ELEMENTNAME = 'ProcUnit'
177 177 xml_labels = ['id', 'inputId', 'name']
178 178
179 179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
180 180 '''
181 181 '''
182 182
183 183 if datatype == None and name == None:
184 184 raise ValueError('datatype or name should be defined')
185 185
186 186 if name == None:
187 187 if 'Proc' in datatype:
188 188 name = datatype
189 189 else:
190 190 name = '%sProc' % (datatype)
191 191
192 192 if datatype == None:
193 193 datatype = name.replace('Proc', '')
194 194
195 195 self.id = str(id)
196 196 self.project_id = project_id
197 197 self.name = name
198 198 self.datatype = datatype
199 199 self.inputId = inputId
200 200 self.err_queue = err_queue
201 201 self.operations = []
202 202 self.parameters = {}
203 203
204 204 def removeOperation(self, id):
205 205
206 i = [1 if x.id==id else 0 for x in self.operations]
206 i = [1 if x.id == id else 0 for x in self.operations]
207 207 self.operations.pop(i.index(1))
208 208
209 209 def getOperation(self, id):
210 210
211 211 for conf in self.operations:
212 212 if conf.id == id:
213 213 return conf
214 214
215 215 def addOperation(self, name, optype='self'):
216 216 '''
217 217 '''
218 218
219 219 id = self.getNewId()
220 220 conf = OperationConf()
221 221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
222 222 self.operations.append(conf)
223 223
224 224 return conf
225 225
226 226 def readXml(self, element, project_id, err_queue):
227 227
228 228 self.id = element.get('id')
229 229 self.name = element.get('name')
230 230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
231 231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
232 232 self.project_id = str(project_id)
233 233 self.err_queue = err_queue
234 234 self.operations = []
235 235 self.parameters = {}
236 236
237 237 for elm in element:
238 238 if elm.tag == 'Parameter':
239 239 self.addParameter(elm.get('name'), elm.get('value'))
240 240 elif elm.tag == 'Operation':
241 241 conf = OperationConf()
242 242 conf.readXml(elm, project_id, err_queue)
243 243 self.operations.append(conf)
244 244
245 245 def createObjects(self):
246 246 '''
247 247 Instancia de unidades de procesamiento.
248 248 '''
249 249
250 250 className = eval(self.name)
251 251 kwargs = self.getKwargs()
252 252 procUnitObj = className()
253 253 procUnitObj.name = self.name
254 254 log.success('creating process...', self.name)
255 255
256 256 for conf in self.operations:
257 257
258 258 opObj = conf.createObject()
259 259
260 260 log.success('adding operation: {}, type:{}'.format(
261 261 conf.name,
262 262 conf.type), self.name)
263 263
264 264 procUnitObj.addOperation(conf, opObj)
265 265
266 266 self.object = procUnitObj
267 267
268 268 def run(self):
269 269 '''
270 270 '''
271 271
272 272 return self.object.call(**self.getKwargs())
273 273
274 274
275 275 class ReadUnitConf(ProcUnitConf):
276 276
277 277 ELEMENTNAME = 'ReadUnit'
278 278
279 279 def __init__(self):
280 280
281 281 self.id = None
282 282 self.datatype = None
283 283 self.name = None
284 284 self.inputId = None
285 285 self.operations = []
286 286 self.parameters = {}
287 287
288 288 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
289 289 startTime='', endTime='', server=None, **kwargs):
290 290
291 291 if datatype == None and name == None:
292 292 raise ValueError('datatype or name should be defined')
293 293 if name == None:
294 294 if 'Reader' in datatype:
295 295 name = datatype
296 datatype = name.replace('Reader','')
296 datatype = name.replace('Reader', '')
297 297 else:
298 298 name = '{}Reader'.format(datatype)
299 299 if datatype == None:
300 300 if 'Reader' in name:
301 datatype = name.replace('Reader','')
301 datatype = name.replace('Reader', '')
302 302 else:
303 303 datatype = name
304 304 name = '{}Reader'.format(name)
305 305
306 306 self.id = id
307 307 self.project_id = project_id
308 308 self.name = name
309 309 self.datatype = datatype
310 310 self.err_queue = err_queue
311 311
312 312 self.addParameter(name='path', value=path)
313 313 self.addParameter(name='startDate', value=startDate)
314 314 self.addParameter(name='endDate', value=endDate)
315 315 self.addParameter(name='startTime', value=startTime)
316 316 self.addParameter(name='endTime', value=endTime)
317 317
318 318 for key, value in kwargs.items():
319 319 self.addParameter(name=key, value=value)
320 320
321 321
322 322 class Project(Process):
323 323 """API to create signal chain projects"""
324 324
325 325 ELEMENTNAME = 'Project'
326 326
327 327 def __init__(self, name=''):
328 328
329 329 Process.__init__(self)
330 330 self.id = '1'
331 331 if name:
332 332 self.name = '{} ({})'.format(Process.__name__, name)
333 333 self.filename = None
334 334 self.description = None
335 335 self.email = None
336 336 self.alarm = []
337 337 self.configurations = {}
338 338 # self.err_queue = Queue()
339 339 self.err_queue = None
340 340 self.started = False
341 341
342 342 def getNewId(self):
343 343
344 344 idList = list(self.configurations.keys())
345 345 id = int(self.id) * 10
346 346
347 347 while True:
348 348 id += 1
349 349
350 350 if str(id) in idList:
351 351 continue
352 352
353 353 break
354 354
355 355 return str(id)
356 356
357 357 def updateId(self, new_id):
358 358
359 359 self.id = str(new_id)
360 360
361 361 keyList = list(self.configurations.keys())
362 362 keyList.sort()
363 363
364 364 n = 1
365 365 new_confs = {}
366 366
367 367 for procKey in keyList:
368 368
369 369 conf = self.configurations[procKey]
370 370 idProcUnit = str(int(self.id) * 10 + n)
371 371 conf.updateId(idProcUnit)
372 372 new_confs[idProcUnit] = conf
373 373 n += 1
374 374
375 375 self.configurations = new_confs
376 376
377 377 def setup(self, id=1, name='', description='', email=None, alarm=[]):
378 378
379 379 self.id = str(id)
380 380 self.description = description
381 381 self.email = email
382 382 self.alarm = alarm
383 383 if name:
384 384 self.name = '{} ({})'.format(Process.__name__, name)
385 385
386 386 def update(self, **kwargs):
387 387
388 388 for key, value in kwargs.items():
389 389 setattr(self, key, value)
390 390
391 391 def clone(self):
392 392
393 393 p = Project()
394 394 p.id = self.id
395 395 p.name = self.name
396 396 p.description = self.description
397 397 p.configurations = self.configurations.copy()
398 398
399 399 return p
400 400
401 401 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
402 402
403 403 '''
404 404 '''
405 405
406 406 if id is None:
407 407 idReadUnit = self.getNewId()
408 408 else:
409 409 idReadUnit = str(id)
410 410
411 411 conf = ReadUnitConf()
412 412 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
413 413 self.configurations[conf.id] = conf
414 414
415 415 return conf
416 416
417 417 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
418 418
419 419 '''
420 420 '''
421 421
422 422 if id is None:
423 423 idProcUnit = self.getNewId()
424 424 else:
425 425 idProcUnit = id
426 426
427 427 conf = ProcUnitConf()
428 428 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
429 429 self.configurations[conf.id] = conf
430 430
431 431 return conf
432 432
433 433 def removeProcUnit(self, id):
434 434
435 435 if id in self.configurations:
436 436 self.configurations.pop(id)
437 437
438 438 def getReadUnit(self):
439 439
440 440 for obj in list(self.configurations.values()):
441 441 if obj.ELEMENTNAME == 'ReadUnit':
442 442 return obj
443 443
444 444 return None
445 445
446 446 def getProcUnit(self, id):
447 447
448 448 return self.configurations[id]
449 449
450 450 def getUnits(self):
451 451
452 452 keys = list(self.configurations)
453 453 keys.sort()
454 454
455 455 for key in keys:
456 456 yield self.configurations[key]
457 457
458 458 def updateUnit(self, id, **kwargs):
459 459
460 460 conf = self.configurations[id].update(**kwargs)
461 461
462 462 def makeXml(self):
463 463
464 464 xml = Element('Project')
465 465 xml.set('id', str(self.id))
466 466 xml.set('name', self.name)
467 467 xml.set('description', self.description)
468 468
469 469 for conf in self.configurations.values():
470 470 conf.makeXml(xml)
471 471
472 472 self.xml = xml
473 473
474 474 def writeXml(self, filename=None):
475 475
476 476 if filename == None:
477 477 if self.filename:
478 478 filename = self.filename
479 479 else:
480 480 filename = 'schain.xml'
481 481
482 482 if not filename:
483 483 print('filename has not been defined. Use setFilename(filename) for do it.')
484 484 return 0
485 485
486 486 abs_file = os.path.abspath(filename)
487 487
488 488 if not os.access(os.path.dirname(abs_file), os.W_OK):
489 489 print('No write permission on %s' % os.path.dirname(abs_file))
490 490 return 0
491 491
492 492 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
493 493 print('File %s already exists and it could not be overwriten' % abs_file)
494 494 return 0
495 495
496 496 self.makeXml()
497 497
498 498 ElementTree(self.xml).write(abs_file, method='xml')
499 499
500 500 self.filename = abs_file
501 501
502 502 return 1
503 503
504 504 def readXml(self, filename):
505 505
506 506 abs_file = os.path.abspath(filename)
507 507
508 508 self.configurations = {}
509 509
510 510 try:
511 511 self.xml = ElementTree().parse(abs_file)
512 512 except:
513 513 log.error('Error reading %s, verify file format' % filename)
514 514 return 0
515 515
516 516 self.id = self.xml.get('id')
517 517 self.name = self.xml.get('name')
518 518 self.description = self.xml.get('description')
519 519
520 520 for element in self.xml:
521 521 if element.tag == 'ReadUnit':
522 522 conf = ReadUnitConf()
523 523 conf.readXml(element, self.id, self.err_queue)
524 524 self.configurations[conf.id] = conf
525 525 elif element.tag == 'ProcUnit':
526 526 conf = ProcUnitConf()
527 527 input_proc = self.configurations[element.get('inputId')]
528 528 conf.readXml(element, self.id, self.err_queue)
529 529 self.configurations[conf.id] = conf
530 530
531 531 self.filename = abs_file
532 532
533 533 return 1
534 534
535 535 def __str__(self):
536 536
537 537 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
538 538 self.id,
539 539 self.name,
540 540 self.description,
541 541 )
542 542
543 543 for conf in self.configurations.values():
544 544 text += '{}'.format(conf)
545 545
546 546 return text
547 547
548 548 def createObjects(self):
549 549
550 550 keys = list(self.configurations.keys())
551 551 keys.sort()
552 552 for key in keys:
553 553 conf = self.configurations[key]
554 554 conf.createObjects()
555 555 if conf.inputId is not None:
556 556 conf.object.setInput(self.configurations[conf.inputId].object)
557 557
558 558 def monitor(self):
559 559
560 560 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
561 561 t.start()
562 562
563 563 def _monitor(self, queue, ctx):
564 564
565 565 import socket
566 566
567 567 procs = 0
568 568 err_msg = ''
569 569
570 570 while True:
571 571 msg = queue.get()
572 572 if '#_start_#' in msg:
573 573 procs += 1
574 574 elif '#_end_#' in msg:
575 procs -=1
575 procs -= 1
576 576 else:
577 577 err_msg = msg
578 578
579 579 if procs == 0 or 'Traceback' in err_msg:
580 580 break
581 581 time.sleep(0.1)
582 582
583 583 if '|' in err_msg:
584 584 name, err = err_msg.split('|')
585 585 if 'SchainWarning' in err:
586 586 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
587 587 elif 'SchainError' in err:
588 588 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
589 589 else:
590 590 log.error(err, name)
591 591 else:
592 592 name, err = self.name, err_msg
593 593
594 594 time.sleep(1)
595 595
596 596 ctx.term()
597 597
598 598 message = ''.join(err)
599 599
600 600 if err_msg:
601 601 subject = 'SChain v%s: Error running %s\n' % (
602 602 schainpy.__version__, self.name)
603 603
604 604 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
605 605 socket.gethostname())
606 606 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
607 607 subtitle += 'Configuration file: %s\n' % self.filename
608 608 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
609 609
610 610 readUnitConfObj = self.getReadUnit()
611 611 if readUnitConfObj:
612 612 subtitle += '\nInput parameters:\n'
613 613 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
614 614 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
615 615 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
616 616 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
617 617 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
618 618
619 619 a = Alarm(
620 modes=self.alarm,
620 modes=self.alarm,
621 621 email=self.email,
622 622 message=message,
623 623 subject=subject,
624 624 subtitle=subtitle,
625 625 filename=self.filename
626 626 )
627 627
628 628 a.start()
629 629
630 630 def setFilename(self, filename):
631 631
632 632 self.filename = filename
633 633
634 634 def runProcs(self):
635 635
636 636 err = False
637 637 n = len(self.configurations)
638 638
639 639 while not err:
640 640 for conf in self.getUnits():
641 641 ok = conf.run()
642 642 if ok == 'Error':
643 643 n -= 1
644 644 continue
645 645 elif not ok:
646 646 break
647 647 if n == 0:
648 648 err = True
649 649
650 650 def run(self):
651 651
652 652 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
653 653 self.started = True
654 654 self.start_time = time.time()
655 655 self.createObjects()
656 656 self.runProcs()
657 657 log.success('{} Done (Time: {:4.2f}s)'.format(
658 658 self.name,
659 time.time()-self.start_time), '')
659 time.time() - self.start_time), '')
Show file before | Show file after | Hide comments
@@ -1,404 +1,404
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 5 '''
6 6 import sys
7 7 import numpy
8 8 import copy
9 9 import datetime
10 10
11 11
12 12 SPEED_OF_LIGHT = 299792458
13 13 SPEED_OF_LIGHT = 3e8
14 14
15 FILE_STRUCTURE = numpy.dtype([ #HEADER 48bytes
16 ('FileMgcNumber','<u4'), #0x23020100
17 ('nFDTdataRecors','<u4'), #No Of FDT data records in this file (0 or more)
18 ('RadarUnitId','<u4'),
19 ('SiteName','<s32'), #Null terminated
15 FILE_STRUCTURE = numpy.dtype([ # HEADER 48bytes
16 ('FileMgcNumber', '<u4'), # 0x23020100
17 ('nFDTdataRecors', '<u4'), # No Of FDT data records in this file (0 or more)
18 ('RadarUnitId', '<u4'),
19 ('SiteName', '<s32'), # Null terminated
20 20 ])
21 21
22 RECORD_STRUCTURE = numpy.dtype([ #RECORD HEADER 180+20N bytes
23 ('RecMgcNumber','<u4'), #0x23030001
24 ('RecCounter','<u4'), #Record counter(0,1, ...)
25 ('Off2StartNxtRec','<u4'), #Offset to start of next record form start of this record
26 ('Off2StartData','<u4'), #Offset to start of data from start of this record
27 ('EpTimeStamp','<i4'), #Epoch time stamp of start of acquisition (seconds)
28 ('msCompTimeStamp','<u4'), #Millisecond component of time stamp (0,...,999)
29 ('ExpTagName','<s32'), #Experiment tag name (null terminated)
30 ('ExpComment','<s32'), #Experiment comment (null terminated)
31 ('SiteLatDegrees','<f4'), #Site latitude (from GPS) in degrees (positive implies North)
32 ('SiteLongDegrees','<f4'), #Site longitude (from GPS) in degrees (positive implies East)
33 ('RTCgpsStatus','<u4'), #RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
34 ('TransmitFrec','<u4'), #Transmit frequency (Hz)
35 ('ReceiveFrec','<u4'), #Receive frequency
36 ('FirstOsciFrec','<u4'), #First local oscillator frequency (Hz)
37 ('Polarisation','<u4'), #(0="O", 1="E", 2="linear 1", 3="linear2")
38 ('ReceiverFiltSett','<u4'), #Receiver filter settings (0,1,2,3)
39 ('nModesInUse','<u4'), #Number of modes in use (1 or 2)
40 ('DualModeIndex','<u4'), #Dual Mode index number for these data (0 or 1)
41 ('DualModeRange','<u4'), #Dual Mode range correction for these data (m)
42 ('nDigChannels','<u4'), #Number of digital channels acquired (2*N)
43 ('SampResolution','<u4'), #Sampling resolution (meters)
44 ('nRangeGatesSamp','<u4'), #Number of range gates sampled
45 ('StartRangeSamp','<u4'), #Start range of sampling (meters)
46 ('PRFhz','<u4'), #PRF (Hz)
47 ('Integrations','<u4'), #Integrations
48 ('nDataPointsTrsf','<u4'), #Number of data points transformed
49 ('nReceiveBeams','<u4'), #Number of receive beams stored in file (1 or N)
50 ('nSpectAverages','<u4'), #Number of spectral averages
51 ('FFTwindowingInd','<u4'), #FFT windowing index (0 = no window)
52 ('BeamAngleAzim','<f4'), #Beam steer angle (azimuth) in degrees (clockwise from true North)
53 ('BeamAngleZen','<f4'), #Beam steer angle (zenith) in degrees (0=> vertical)
54 ('AntennaCoord','<f24'), #Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
55 ('RecPhaseCalibr','<f12'), #Receiver phase calibration (degrees) - N values
56 ('RecAmpCalibr','<f12'), #Receiver amplitude calibration (ratio relative to receiver one) - N values
57 ('ReceiverGaindB','<u12'), #Receiver gains in dB - N values
22 RECORD_STRUCTURE = numpy.dtype([ # RECORD HEADER 180+20N bytes
23 ('RecMgcNumber', '<u4'), # 0x23030001
24 ('RecCounter', '<u4'), # Record counter(0,1, ...)
25 ('Off2StartNxtRec', '<u4'), # Offset to start of next record form start of this record
26 ('Off2StartData', '<u4'), # Offset to start of data from start of this record
27 ('EpTimeStamp', '<i4'), # Epoch time stamp of start of acquisition (seconds)
28 ('msCompTimeStamp', '<u4'), # Millisecond component of time stamp (0,...,999)
29 ('ExpTagName', '<s32'), # Experiment tag name (null terminated)
30 ('ExpComment', '<s32'), # Experiment comment (null terminated)
31 ('SiteLatDegrees', '<f4'), # Site latitude (from GPS) in degrees (positive implies North)
32 ('SiteLongDegrees', '<f4'), # Site longitude (from GPS) in degrees (positive implies East)
33 ('RTCgpsStatus', '<u4'), # RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
34 ('TransmitFrec', '<u4'), # Transmit frequency (Hz)
35 ('ReceiveFrec', '<u4'), # Receive frequency
36 ('FirstOsciFrec', '<u4'), # First local oscillator frequency (Hz)
37 ('Polarisation', '<u4'), # (0="O", 1="E", 2="linear 1", 3="linear2")
38 ('ReceiverFiltSett', '<u4'), # Receiver filter settings (0,1,2,3)
39 ('nModesInUse', '<u4'), # Number of modes in use (1 or 2)
40 ('DualModeIndex', '<u4'), # Dual Mode index number for these data (0 or 1)
41 ('DualModeRange', '<u4'), # Dual Mode range correction for these data (m)
42 ('nDigChannels', '<u4'), # Number of digital channels acquired (2*N)
43 ('SampResolution', '<u4'), # Sampling resolution (meters)
44 ('nRangeGatesSamp', '<u4'), # Number of range gates sampled
45 ('StartRangeSamp', '<u4'), # Start range of sampling (meters)
46 ('PRFhz', '<u4'), # PRF (Hz)
47 ('Integrations', '<u4'), # Integrations
48 ('nDataPointsTrsf', '<u4'), # Number of data points transformed
49 ('nReceiveBeams', '<u4'), # Number of receive beams stored in file (1 or N)
50 ('nSpectAverages', '<u4'), # Number of spectral averages
51 ('FFTwindowingInd', '<u4'), # FFT windowing index (0 = no window)
52 ('BeamAngleAzim', '<f4'), # Beam steer angle (azimuth) in degrees (clockwise from true North)
53 ('BeamAngleZen', '<f4'), # Beam steer angle (zenith) in degrees (0=> vertical)
54 ('AntennaCoord', '<f24'), # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
55 ('RecPhaseCalibr', '<f12'), # Receiver phase calibration (degrees) - N values
56 ('RecAmpCalibr', '<f12'), # Receiver amplitude calibration (ratio relative to receiver one) - N values
57 ('ReceiverGaindB', '<u12'), # Receiver gains in dB - N values
58 58 ])
59 59
60 60
61 61 class Header(object):
62 62
63 63 def __init__(self):
64 64 raise NotImplementedError
65 65
66 66
67 67 def read(self):
68 68
69 69 raise NotImplementedError
70 70
71 71 def write(self):
72 72
73 73 raise NotImplementedError
74 74
75 75 def printInfo(self):
76 76
77 77 message = "#"*50 + "\n"
78 78 message += self.__class__.__name__.upper() + "\n"
79 79 message += "#"*50 + "\n"
80 80
81 81 keyList = list(self.__dict__.keys())
82 82 keyList.sort()
83 83
84 84 for key in keyList:
85 message += "%s = %s" %(key, self.__dict__[key]) + "\n"
85 message += "%s = %s" % (key, self.__dict__[key]) + "\n"
86 86
87 87 if "size" not in keyList:
88 88 attr = getattr(self, "size")
89 89
90 90 if attr:
91 message += "%s = %s" %("size", attr) + "\n"
91 message += "%s = %s" % ("size", attr) + "\n"
92 92
93 93 print(message)
94 94
95 95 class FileHeader(Header):
96 96
97 FileMgcNumber= None
98 nFDTdataRecors=None #No Of FDT data records in this file (0 or more)
99 RadarUnitId= None
100 SiteName= None
97 FileMgcNumber = None
98 nFDTdataRecors = None # No Of FDT data records in this file (0 or more)
99 RadarUnitId = None
100 SiteName = None
101 101
102 #__LOCALTIME = None
102 # __LOCALTIME = None
103 103
104 104 def __init__(self, useLocalTime=True):
105 105
106 self.FileMgcNumber= 0 #0x23020100
107 self.nFDTdataRecors=0 #No Of FDT data records in this file (0 or more)
108 self.RadarUnitId= 0
109 self.SiteName= ""
106 self.FileMgcNumber = 0 # 0x23020100
107 self.nFDTdataRecors = 0 # No Of FDT data records in this file (0 or more)
108 self.RadarUnitId = 0
109 self.SiteName = ""
110 110 self.size = 48
111 111
112 #self.useLocalTime = useLocalTime
112 # self.useLocalTime = useLocalTime
113 113
114 114 def read(self, fp):
115 115
116 116 try:
117 header = numpy.fromfile(fp, FILE_STRUCTURE,1)
117 header = numpy.fromfile(fp, FILE_STRUCTURE, 1)
118 118 ''' numpy.fromfile(file, dtype, count, sep='')
119 119 file : file or str
120 120 Open file object or filename.
121 121
122 122 dtype : data-type
123 123 Data type of the returned array. For binary files, it is used to determine
124 124 the size and byte-order of the items in the file.
125 125
126 126 count : int
127 127 Number of items to read. -1 means all items (i.e., the complete file).
128 128
129 129 sep : str
130 130 Separator between items if file is a text file. Empty (“”) separator means
131 131 the file should be treated as binary. Spaces (” ”) in the separator match zero
132 132 or more whitespace characters. A separator consisting only of spaces must match
133 133 at least one whitespace.
134 134
135 135 '''
136 136
137 137 except Exception as e:
138 138 print("FileHeader: ")
139 139 print(eBasicHeader)
140 140 return 0
141 141
142 self.FileMgcNumber= byte(header['FileMgcNumber'][0])
143 self.nFDTdataRecors=int(header['nFDTdataRecors'][0]) #No Of FDT data records in this file (0 or more)
144 self.RadarUnitId= int(header['RadarUnitId'][0])
145 self.SiteName= char(header['SiteName'][0])
142 self.FileMgcNumber = byte(header['FileMgcNumber'][0])
143 self.nFDTdataRecors = int(header['nFDTdataRecors'][0]) # No Of FDT data records in this file (0 or more)
144 self.RadarUnitId = int(header['RadarUnitId'][0])
145 self.SiteName = char(header['SiteName'][0])
146 146
147 147
148 if self.size <48:
148 if self.size < 48:
149 149 return 0
150 150
151 151 return 1
152 152
153 153 def write(self, fp):
154 154
155 155 headerTuple = (self.FileMgcNumber,
156 156 self.nFDTdataRecors,
157 157 self.RadarUnitId,
158 158 self.SiteName,
159 159 self.size)
160 160
161 161
162 162 header = numpy.array(headerTuple, FILE_STRUCTURE)
163 163 # numpy.array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0)
164 164 header.tofile(fp)
165 165 ''' ndarray.tofile(fid, sep, format) Write array to a file as text or binary (default).
166 166
167 167 fid : file or str
168 168 An open file object, or a string containing a filename.
169 169
170 170 sep : str
171 171 Separator between array items for text output. If “” (empty), a binary file is written,
172 172 equivalent to file.write(a.tobytes()).
173 173
174 174 format : str
175 175 Format string for text file output. Each entry in the array is formatted to text by
176 176 first converting it to the closest Python type, and then using “format” % item.
177 177
178 178 '''
179 179
180 180 return 1
181 181
182 182
183 183 class RecordHeader(Header):
184 184
185 RecMgcNumber=None #0x23030001
186 RecCounter= None
187 Off2StartNxtRec= None
188 EpTimeStamp= None
189 msCompTimeStamp= None
190 ExpTagName= None
191 ExpComment=None
192 SiteLatDegrees=None
193 SiteLongDegrees= None
194 RTCgpsStatus= None
195 TransmitFrec= None
196 ReceiveFrec= None
197 FirstOsciFrec= None
198 Polarisation= None
199 ReceiverFiltSett= None
200 nModesInUse= None
201 DualModeIndex= None
202 DualModeRange= None
203 nDigChannels= None
204 SampResolution= None
205 nRangeGatesSamp= None
206 StartRangeSamp= None
207 PRFhz= None
208 Integrations= None
209 nDataPointsTrsf= None
210 nReceiveBeams= None
211 nSpectAverages= None
212 FFTwindowingInd= None
213 BeamAngleAzim= None
214 BeamAngleZen= None
215 AntennaCoord= None
216 RecPhaseCalibr= None
217 RecAmpCalibr= None
218 ReceiverGaindB= None
185 RecMgcNumber = None # 0x23030001
186 RecCounter = None
187 Off2StartNxtRec = None
188 EpTimeStamp = None
189 msCompTimeStamp = None
190 ExpTagName = None
191 ExpComment = None
192 SiteLatDegrees = None
193 SiteLongDegrees = None
194 RTCgpsStatus = None
195 TransmitFrec = None
196 ReceiveFrec = None
197 FirstOsciFrec = None
198 Polarisation = None
199 ReceiverFiltSett = None
200 nModesInUse = None
201 DualModeIndex = None
202 DualModeRange = None
203 nDigChannels = None
204 SampResolution = None
205 nRangeGatesSamp = None
206 StartRangeSamp = None
207 PRFhz = None
208 Integrations = None
209 nDataPointsTrsf = None
210 nReceiveBeams = None
211 nSpectAverages = None
212 FFTwindowingInd = None
213 BeamAngleAzim = None
214 BeamAngleZen = None
215 AntennaCoord = None
216 RecPhaseCalibr = None
217 RecAmpCalibr = None
218 ReceiverGaindB = None
219 219
220 220 '''size = None
221 221 nSamples = None
222 222 nProfiles = None
223 223 nChannels = None
224 224 adcResolution = None
225 225 pciDioBusWidth = None'''
226 226
227 def __init__(self, RecMgcNumber=None, RecCounter= 0, Off2StartNxtRec= 0,
228 EpTimeStamp= 0, msCompTimeStamp= 0, ExpTagName= None,
229 ExpComment=None, SiteLatDegrees=0, SiteLongDegrees= 0,
230 RTCgpsStatus= 0, TransmitFrec= 0, ReceiveFrec= 0,
231 FirstOsciFrec= 0, Polarisation= 0, ReceiverFiltSett= 0,
232 nModesInUse= 0, DualModeIndex= 0, DualModeRange= 0,
233 nDigChannels= 0, SampResolution= 0, nRangeGatesSamp= 0,
234 StartRangeSamp= 0, PRFhz= 0, Integrations= 0,
235 nDataPointsTrsf= 0, nReceiveBeams= 0, nSpectAverages= 0,
236 FFTwindowingInd= 0, BeamAngleAzim= 0, BeamAngleZen= 0,
237 AntennaCoord= 0, RecPhaseCalibr= 0, RecAmpCalibr= 0,
238 ReceiverGaindB= 0):
227 def __init__(self, RecMgcNumber=None, RecCounter=0, Off2StartNxtRec=0,
228 EpTimeStamp=0, msCompTimeStamp=0, ExpTagName=None,
229 ExpComment=None, SiteLatDegrees=0, SiteLongDegrees=0,
230 RTCgpsStatus=0, TransmitFrec=0, ReceiveFrec=0,
231 FirstOsciFrec=0, Polarisation=0, ReceiverFiltSett=0,
232 nModesInUse=0, DualModeIndex=0, DualModeRange=0,
233 nDigChannels=0, SampResolution=0, nRangeGatesSamp=0,
234 StartRangeSamp=0, PRFhz=0, Integrations=0,
235 nDataPointsTrsf=0, nReceiveBeams=0, nSpectAverages=0,
236 FFTwindowingInd=0, BeamAngleAzim=0, BeamAngleZen=0,
237 AntennaCoord=0, RecPhaseCalibr=0, RecAmpCalibr=0,
238 ReceiverGaindB=0):
239 239
240 self.RecMgcNumber = RecMgcNumber #0x23030001
240 self.RecMgcNumber = RecMgcNumber # 0x23030001
241 241 self.RecCounter = RecCounter
242 242 self.Off2StartNxtRec = Off2StartNxtRec
243 243 self.EpTimeStamp = EpTimeStamp
244 244 self.msCompTimeStamp = msCompTimeStamp
245 245 self.ExpTagName = ExpTagName
246 246 self.ExpComment = ExpComment
247 247 self.SiteLatDegrees = SiteLatDegrees
248 248 self.SiteLongDegrees = SiteLongDegrees
249 249 self.RTCgpsStatus = RTCgpsStatus
250 250 self.TransmitFrec = TransmitFrec
251 251 self.ReceiveFrec = ReceiveFrec
252 252 self.FirstOsciFrec = FirstOsciFrec
253 253 self.Polarisation = Polarisation
254 254 self.ReceiverFiltSett = ReceiverFiltSett
255 255 self.nModesInUse = nModesInUse
256 256 self.DualModeIndex = DualModeIndex
257 257 self.DualModeRange = DualModeRange
258 258 self.nDigChannels = nDigChannels
259 259 self.SampResolution = SampResolution
260 260 self.nRangeGatesSamp = nRangeGatesSamp
261 261 self.StartRangeSamp = StartRangeSamp
262 262 self.PRFhz = PRFhz
263 263 self.Integrations = Integrations
264 264 self.nDataPointsTrsf = nDataPointsTrsf
265 265 self.nReceiveBeams = nReceiveBeams
266 266 self.nSpectAverages = nSpectAverages
267 267 self.FFTwindowingInd = FFTwindowingInd
268 268 self.BeamAngleAzim = BeamAngleAzim
269 269 self.BeamAngleZen = BeamAngleZen
270 270 self.AntennaCoord = AntennaCoord
271 271 self.RecPhaseCalibr = RecPhaseCalibr
272 272 self.RecAmpCalibr = RecAmpCalibr
273 273 self.ReceiverGaindB = ReceiverGaindB
274 274
275 275
276 276 def read(self, fp):
277 277
278 startFp = fp.tell() #The method tell() returns the current position of the file read/write pointer within the file.
278 startFp = fp.tell() # The method tell() returns the current position of the file read/write pointer within the file.
279 279
280 280 try:
281 header = numpy.fromfile(fp,RECORD_STRUCTURE,1)
281 header = numpy.fromfile(fp, RECORD_STRUCTURE, 1)
282 282 except Exception as e:
283 283 print("System Header: " + e)
284 284 return 0
285 285
286 self.RecMgcNumber = header['RecMgcNumber'][0] #0x23030001
286 self.RecMgcNumber = header['RecMgcNumber'][0] # 0x23030001
287 287 self.RecCounter = header['RecCounter'][0]
288 288 self.Off2StartNxtRec = header['Off2StartNxtRec'][0]
289 289 self.EpTimeStamp = header['EpTimeStamp'][0]
290 290 self.msCompTimeStamp = header['msCompTimeStamp'][0]
291 291 self.ExpTagName = header['ExpTagName'][0]
292 292 self.ExpComment = header['ExpComment'][0]
293 293 self.SiteLatDegrees = header['SiteLatDegrees'][0]
294 294 self.SiteLongDegrees = header['SiteLongDegrees'][0]
295 295 self.RTCgpsStatus = header['RTCgpsStatus'][0]
296 296 self.TransmitFrec = header['TransmitFrec'][0]
297 297 self.ReceiveFrec = header['ReceiveFrec'][0]
298 298 self.FirstOsciFrec = header['FirstOsciFrec'][0]
299 299 self.Polarisation = header['Polarisation'][0]
300 300 self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
301 301 self.nModesInUse = header['nModesInUse'][0]
302 302 self.DualModeIndex = header['DualModeIndex'][0]
303 303 self.DualModeRange = header['DualModeRange'][0]
304 304 self.nDigChannels = header['nDigChannels'][0]
305 305 self.SampResolution = header['SampResolution'][0]
306 306 self.nRangeGatesSamp = header['nRangeGatesSamp'][0]
307 307 self.StartRangeSamp = header['StartRangeSamp'][0]
308 308 self.PRFhz = header['PRFhz'][0]
309 309 self.Integrations = header['Integrations'][0]
310 310 self.nDataPointsTrsf = header['nDataPointsTrsf'][0]
311 311 self.nReceiveBeams = header['nReceiveBeams'][0]
312 312 self.nSpectAverages = header['nSpectAverages'][0]
313 313 self.FFTwindowingInd = header['FFTwindowingInd'][0]
314 314 self.BeamAngleAzim = header['BeamAngleAzim'][0]
315 315 self.BeamAngleZen = header['BeamAngleZen'][0]
316 316 self.AntennaCoord = header['AntennaCoord'][0]
317 317 self.RecPhaseCalibr = header['RecPhaseCalibr'][0]
318 318 self.RecAmpCalibr = header['RecAmpCalibr'][0]
319 319 self.ReceiverGaindB = header['ReceiverGaindB'][0]
320 320
321 Self.size = 180+20*3
321 Self.size = 180 + 20 * 3
322 322
323 323 endFp = self.size + startFp
324 324
325 325 if fp.tell() > endFp:
326 sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name)
326 sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
327 327 return 0
328 328
329 329 if fp.tell() < endFp:
330 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
330 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
331 331 return 0
332 332
333 333 return 1
334 334
335 335 def write(self, fp):
336 336
337 337 headerTuple = (self.RecMgcNumber,
338 self.RecCounter,
339 self.Off2StartNxtRec,
340 self.EpTimeStamp,
341 self.msCompTimeStamp,
342 self.ExpTagName,
343 self.ExpComment,
344 self.SiteLatDegrees,
345 self.SiteLongDegrees,
346 self.RTCgpsStatus,
347 self.TransmitFrec,
348 self.ReceiveFrec,
349 self.FirstOsciFrec,
350 self.Polarisation,
351 self.ReceiverFiltSett,
352 self.nModesInUse,
353 self.DualModeIndex,
354 self.DualModeRange,
338 self.RecCounter,
339 self.Off2StartNxtRec,
340 self.EpTimeStamp,
341 self.msCompTimeStamp,
342 self.ExpTagName,
343 self.ExpComment,
344 self.SiteLatDegrees,
345 self.SiteLongDegrees,
346 self.RTCgpsStatus,
347 self.TransmitFrec,
348 self.ReceiveFrec,
349 self.FirstOsciFrec,
350 self.Polarisation,
351 self.ReceiverFiltSett,
352 self.nModesInUse,
353 self.DualModeIndex,
354 self.DualModeRange,
355 355 self.nDigChannels,
356 self.SampResolution,
357 self.nRangeGatesSamp,
358 self.StartRangeSamp,
359 self.PRFhz,
360 self.Integrations,
361 self.nDataPointsTrsf,
362 self.nReceiveBeams,
363 self.nSpectAverages,
364 self.FFTwindowingInd,
365 self.BeamAngleAzim,
366 self.BeamAngleZen,
367 self.AntennaCoord,
368 self.RecPhaseCalibr,
369 self.RecAmpCalibr,
356 self.SampResolution,
357 self.nRangeGatesSamp,
358 self.StartRangeSamp,
359 self.PRFhz,
360 self.Integrations,
361 self.nDataPointsTrsf,
362 self.nReceiveBeams,
363 self.nSpectAverages,
364 self.FFTwindowingInd,
365 self.BeamAngleAzim,
366 self.BeamAngleZen,
367 self.AntennaCoord,
368 self.RecPhaseCalibr,
369 self.RecAmpCalibr,
370 370 self.ReceiverGaindB)
371 371
372 372 # self.size,self.nSamples,
373 373 # self.nProfiles,
374 374 # self.nChannels,
375 375 # self.adcResolution,
376 376 # self.pciDioBusWidth
377 377
378 header = numpy.array(headerTuple,RECORD_STRUCTURE)
378 header = numpy.array(headerTuple, RECORD_STRUCTURE)
379 379 header.tofile(fp)
380 380
381 381 return 1
382 382
383 383
384 384 def get_dtype_index(numpy_dtype):
385 385
386 386 index = None
387 387
388 388 for i in range(len(NUMPY_DTYPE_LIST)):
389 389 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
390 390 index = i
391 391 break
392 392
393 393 return index
394 394
395 395 def get_numpy_dtype(index):
396 396
397 #dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
397 # dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
398 398
399 399 return NUMPY_DTYPE_LIST[index]
400 400
401 401
402 402 def get_dtype_width(index):
403 403
404 return DTYPE_WIDTH[index] No newline at end of file
404 return DTYPE_WIDTH[index]
Show file before | Show file after | Hide comments
@@ -1,3 +1,3
1 1 from .jrodata import *
2 2 from .jroheaderIO import *
3 from .jroamisr import * No newline at end of file
3 from .jroamisr import *
Show file before | Show file after | Hide comments
@@ -1,87 +1,87
1 1 import numpy
2 2 import copy
3 3
4 4 class Beam:
5 5 def __init__(self):
6 6 self.codeList = []
7 7 self.azimuthList = []
8 8 self.zenithList = []
9 9
10 10
11 11 class AMISR:
12 12 def __init__(self):
13 13 self.flagNoData = True
14 14 self.data = None
15 15 self.utctime = None
16 16 self.type = "AMISR"
17 17
18 #propiedades para compatibilidad con Voltages
19 self.timeZone = 0#timezone like jroheader, difference in minutes between UTC and localtime
20 self.dstFlag = 0#self.dataIn.dstFlag
21 self.errorCount = 0#self.dataIn.errorCount
22 self.useLocalTime = True#self.dataIn.useLocalTime
18 # propiedades para compatibilidad con Voltages
19 self.timeZone = 0 # timezone like jroheader, difference in minutes between UTC and localtime
20 self.dstFlag = 0 # self.dataIn.dstFlag
21 self.errorCount = 0 # self.dataIn.errorCount
22 self.useLocalTime = True # self.dataIn.useLocalTime
23 23
24 self.radarControllerHeaderObj = None#self.dataIn.radarControllerHeaderObj.copy()
25 self.systemHeaderObj = None#self.dataIn.systemHeaderObj.copy()
26 self.channelList = [0]#self.dataIn.channelList esto solo aplica para el caso de AMISR
27 self.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
24 self.radarControllerHeaderObj = None # self.dataIn.radarControllerHeaderObj.copy()
25 self.systemHeaderObj = None # self.dataIn.systemHeaderObj.copy()
26 self.channelList = [0] # self.dataIn.channelList esto solo aplica para el caso de AMISR
27 self.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
28 28
29 self.flagDiscontinuousBlock = None#self.dataIn.flagDiscontinuousBlock
30 #self.utctime = #self.firstdatatime
31 self.flagDecodeData = None#self.dataIn.flagDecodeData #asumo q la data esta decodificada
32 self.flagDeflipData = None#self.dataIn.flagDeflipData #asumo q la data esta sin flip
29 self.flagDiscontinuousBlock = None # self.dataIn.flagDiscontinuousBlock
30 # self.utctime = #self.firstdatatime
31 self.flagDecodeData = None # self.dataIn.flagDecodeData #asumo q la data esta decodificada
32 self.flagDeflipData = None # self.dataIn.flagDeflipData #asumo q la data esta sin flip
33 33
34 self.nCohInt = 1#self.dataIn.nCohInt
34 self.nCohInt = 1 # self.dataIn.nCohInt
35 35 self.nIncohInt = 1
36 self.ippSeconds = None#self.dataIn.ippSeconds, segun el filename/Setup/Tufile
37 self.windowOfFilter = None#self.dataIn.windowOfFilter
36 self.ippSeconds = None # self.dataIn.ippSeconds, segun el filename/Setup/Tufile
37 self.windowOfFilter = None # self.dataIn.windowOfFilter
38 38
39 self.timeInterval = None#self.dataIn.timeInterval*self.dataOut.nFFTPoints*self.dataOut.nIncohInt
40 self.frequency = None#self.dataIn.frequency
41 self.realtime = 0#self.dataIn.realtime
39 self.timeInterval = None # self.dataIn.timeInterval*self.dataOut.nFFTPoints*self.dataOut.nIncohInt
40 self.frequency = None # self.dataIn.frequency
41 self.realtime = 0 # self.dataIn.realtime
42 42
43 #actualizar en la lectura de datos
44 self.heightList = None#self.dataIn.heightList
45 self.nProfiles = None#Number of samples or nFFTPoints
43 # actualizar en la lectura de datos
44 self.heightList = None # self.dataIn.heightList
45 self.nProfiles = None # Number of samples or nFFTPoints
46 46 self.nRecords = None
47 47 self.nBeams = None
48 self.nBaud = None#self.dataIn.nBaud
49 self.nCode = None#self.dataIn.nCode
50 self.code = None#self.dataIn.code
48 self.nBaud = None # self.dataIn.nBaud
49 self.nCode = None # self.dataIn.nCode
50 self.code = None # self.dataIn.code
51 51
52 #consideracion para los Beams
52 # consideracion para los Beams
53 53 self.beamCodeDict = None
54 54 self.beamRangeDict = None
55 55 self.beamcode = None
56 56 self.azimuth = None
57 57 self.zenith = None
58 58 self.gain = None
59 59
60 60 self.npulseByFrame = None
61 61
62 62 self.profileIndex = None
63 63
64 64 self.beam = Beam()
65 65
66 66 def copy(self, inputObj=None):
67 67
68 68 if inputObj is None:
69 69 return copy.deepcopy(self)
70 70
71 71 for key in list(inputObj.__dict__.keys()):
72 72 self.__dict__[key] = inputObj.__dict__[key]
73 73
74 74 @property
75 75 def nHeights(self):
76 76
77 77 return len(self.heightList)
78 78
79 79
80 80 def isEmpty(self):
81 81
82 82 return self.flagNoData
83 83
84 84 @property
85 85 def timeInterval(self):
86 86
87 87 return self.ippSeconds * self.nCohInt
Show file before | Show file after | Hide comments
@@ -1,1066 +1,1066
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Definition of diferent Data objects for different types of data
6 6
7 7 Here you will find the diferent data objects for the different types
8 8 of data, this data objects must be used as dataIn or dataOut objects in
9 9 processing units and operations. Currently the supported data objects are:
10 10 Voltage, Spectra, SpectraHeis, Fits, Correlation and Parameters
11 11 """
12 12
13 13 import copy
14 14 import numpy
15 15 import datetime
16 16 import json
17 17
18 18 import schainpy.admin
19 19 from schainpy.utils import log
20 20 from .jroheaderIO import SystemHeader, RadarControllerHeader
21 21 from schainpy.model.data import _noise
22 22
23 23
24 24 def getNumpyDtype(dataTypeCode):
25 25
26 26 if dataTypeCode == 0:
27 27 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
28 28 elif dataTypeCode == 1:
29 29 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
30 30 elif dataTypeCode == 2:
31 31 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
32 32 elif dataTypeCode == 3:
33 33 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
34 34 elif dataTypeCode == 4:
35 35 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
36 36 elif dataTypeCode == 5:
37 37 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
38 38 else:
39 39 raise ValueError('dataTypeCode was not defined')
40 40
41 41 return numpyDtype
42 42
43 43
44 44 def getDataTypeCode(numpyDtype):
45 45
46 46 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
47 47 datatype = 0
48 48 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
49 49 datatype = 1
50 50 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
51 51 datatype = 2
52 52 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
53 53 datatype = 3
54 54 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
55 55 datatype = 4
56 56 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
57 57 datatype = 5
58 58 else:
59 59 datatype = None
60 60
61 61 return datatype
62 62
63 63
64 64 def hildebrand_sekhon(data, navg):
65 65 """
66 66 This method is for the objective determination of the noise level in Doppler spectra. This
67 67 implementation technique is based on the fact that the standard deviation of the spectral
68 68 densities is equal to the mean spectral density for white Gaussian noise
69 69
70 70 Inputs:
71 71 Data : heights
72 72 navg : numbers of averages
73 73
74 74 Return:
75 75 mean : noise's level
76 76 """
77 77
78 78 sortdata = numpy.sort(data, axis=None)
79 79 '''
80 80 lenOfData = len(sortdata)
81 81 nums_min = lenOfData*0.2
82 82
83 83 if nums_min <= 5:
84 84
85 85 nums_min = 5
86 86
87 87 sump = 0.
88 88 sumq = 0.
89 89
90 90 j = 0
91 91 cont = 1
92 92
93 93 while((cont == 1)and(j < lenOfData)):
94 94
95 95 sump += sortdata[j]
96 96 sumq += sortdata[j]**2
97 97
98 98 if j > nums_min:
99 99 rtest = float(j)/(j-1) + 1.0/navg
100 100 if ((sumq*j) > (rtest*sump**2)):
101 101 j = j - 1
102 102 sump = sump - sortdata[j]
103 103 sumq = sumq - sortdata[j]**2
104 104 cont = 0
105 105
106 106 j += 1
107 107
108 108 lnoise = sump / j
109 109 '''
110 110 return _noise.hildebrand_sekhon(sortdata, navg)
111 111
112 112
113 113 class Beam:
114 114
115 115 def __init__(self):
116 116 self.codeList = []
117 117 self.azimuthList = []
118 118 self.zenithList = []
119 119
120 120
121 121 class GenericData(object):
122 122
123 123 flagNoData = True
124 124
125 125 def copy(self, inputObj=None):
126 126
127 127 if inputObj == None:
128 128 return copy.deepcopy(self)
129 129
130 130 for key in list(inputObj.__dict__.keys()):
131 131
132 132 attribute = inputObj.__dict__[key]
133 133
134 134 # If this attribute is a tuple or list
135 135 if type(inputObj.__dict__[key]) in (tuple, list):
136 136 self.__dict__[key] = attribute[:]
137 137 continue
138 138
139 139 # If this attribute is another object or instance
140 140 if hasattr(attribute, '__dict__'):
141 141 self.__dict__[key] = attribute.copy()
142 142 continue
143 143
144 144 self.__dict__[key] = inputObj.__dict__[key]
145 145
146 146 def deepcopy(self):
147 147
148 148 return copy.deepcopy(self)
149 149
150 150 def isEmpty(self):
151 151
152 152 return self.flagNoData
153 153
154 154 def isReady(self):
155 155
156 156 return not self.flagNoData
157 157
158 158
159 159 class JROData(GenericData):
160 160
161 161 systemHeaderObj = SystemHeader()
162 162 radarControllerHeaderObj = RadarControllerHeader()
163 163 type = None
164 164 datatype = None # dtype but in string
165 165 nProfiles = None
166 166 heightList = None
167 167 channelList = None
168 168 flagDiscontinuousBlock = False
169 169 useLocalTime = False
170 170 utctime = None
171 171 timeZone = None
172 172 dstFlag = None
173 173 errorCount = None
174 174 blocksize = None
175 175 flagDecodeData = False # asumo q la data no esta decodificada
176 176 flagDeflipData = False # asumo q la data no esta sin flip
177 177 flagShiftFFT = False
178 178 nCohInt = None
179 179 windowOfFilter = 1
180 180 C = 3e8
181 181 frequency = 49.92e6
182 182 realtime = False
183 183 beacon_heiIndexList = None
184 184 last_block = None
185 185 blocknow = None
186 186 azimuth = None
187 187 zenith = None
188 188 beam = Beam()
189 189 profileIndex = None
190 190 error = None
191 191 data = None
192 192 nmodes = None
193 193 metadata_list = ['heightList', 'timeZone', 'type']
194 194
195 195 def __str__(self):
196 196
197 197 return '{} - {}'.format(self.type, self.datatime())
198 198
199 199 def getNoise(self):
200 200
201 201 raise NotImplementedError
202 202
203 203 @property
204 204 def nChannels(self):
205 205
206 206 return len(self.channelList)
207 207
208 208 @property
209 209 def channelIndexList(self):
210 210
211 211 return list(range(self.nChannels))
212 212
213 213 @property
214 214 def nHeights(self):
215 215
216 216 return len(self.heightList)
217 217
218 218 def getDeltaH(self):
219 219
220 220 return self.heightList[1] - self.heightList[0]
221 221
222 222 @property
223 223 def ltctime(self):
224 224
225 225 if self.useLocalTime:
226 226 return self.utctime - self.timeZone * 60
227 227
228 228 return self.utctime
229 229
230 230 @property
231 231 def datatime(self):
232 232
233 233 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
234 234 return datatimeValue
235 235
236 236 def getTimeRange(self):
237 237
238 238 datatime = []
239 239
240 240 datatime.append(self.ltctime)
241 241 datatime.append(self.ltctime + self.timeInterval + 1)
242 242
243 243 datatime = numpy.array(datatime)
244 244
245 245 return datatime
246 246
247 247 def getFmaxTimeResponse(self):
248 248
249 period = (10**-6) * self.getDeltaH() / (0.15)
249 period = (10 ** -6) * self.getDeltaH() / (0.15)
250 250
251 251 PRF = 1. / (period * self.nCohInt)
252 252
253 253 fmax = PRF
254 254
255 255 return fmax
256 256
257 257 def getFmax(self):
258 258 PRF = 1. / (self.ippSeconds * self.nCohInt)
259 259
260 260 fmax = PRF
261 261 return fmax
262 262
263 263 def getVmax(self):
264 264
265 265 _lambda = self.C / self.frequency
266 266
267 267 vmax = self.getFmax() * _lambda / 2
268 268
269 269 return vmax
270 270
271 271 @property
272 272 def ippSeconds(self):
273 273 '''
274 274 '''
275 275 return self.radarControllerHeaderObj.ippSeconds
276 276
277 277 @ippSeconds.setter
278 278 def ippSeconds(self, ippSeconds):
279 279 '''
280 280 '''
281 281 self.radarControllerHeaderObj.ippSeconds = ippSeconds
282 282
283 283 @property
284 284 def code(self):
285 285 '''
286 286 '''
287 287 return self.radarControllerHeaderObj.code
288 288
289 289 @code.setter
290 290 def code(self, code):
291 291 '''
292 292 '''
293 293 self.radarControllerHeaderObj.code = code
294 294
295 295 @property
296 296 def nCode(self):
297 297 '''
298 298 '''
299 299 return self.radarControllerHeaderObj.nCode
300 300
301 301 @nCode.setter
302 302 def nCode(self, ncode):
303 303 '''
304 304 '''
305 305 self.radarControllerHeaderObj.nCode = ncode
306 306
307 307 @property
308 308 def nBaud(self):
309 309 '''
310 310 '''
311 311 return self.radarControllerHeaderObj.nBaud
312 312
313 313 @nBaud.setter
314 314 def nBaud(self, nbaud):
315 315 '''
316 316 '''
317 317 self.radarControllerHeaderObj.nBaud = nbaud
318 318
319 319 @property
320 320 def ipp(self):
321 321 '''
322 322 '''
323 323 return self.radarControllerHeaderObj.ipp
324 324
325 325 @ipp.setter
326 326 def ipp(self, ipp):
327 327 '''
328 328 '''
329 329 self.radarControllerHeaderObj.ipp = ipp
330 330
331 331 @property
332 332 def metadata(self):
333 333 '''
334 334 '''
335 335
336 336 return {attr: getattr(self, attr) for attr in self.metadata_list}
337 337
338 338
339 339 class Voltage(JROData):
340 340
341 dataPP_POW = None
342 dataPP_DOP = None
341 dataPP_POW = None
342 dataPP_DOP = None
343 343 dataPP_WIDTH = None
344 dataPP_SNR = None
344 dataPP_SNR = None
345 345
346 346 def __init__(self):
347 347 '''
348 348 Constructor
349 349 '''
350 350
351 351 self.useLocalTime = True
352 352 self.radarControllerHeaderObj = RadarControllerHeader()
353 353 self.systemHeaderObj = SystemHeader()
354 354 self.type = "Voltage"
355 355 self.data = None
356 356 self.nProfiles = None
357 357 self.heightList = None
358 358 self.channelList = None
359 359 self.flagNoData = True
360 360 self.flagDiscontinuousBlock = False
361 361 self.utctime = None
362 362 self.timeZone = 0
363 363 self.dstFlag = None
364 364 self.errorCount = None
365 365 self.nCohInt = None
366 366 self.blocksize = None
367 367 self.flagCohInt = False
368 368 self.flagDecodeData = False # asumo q la data no esta decodificada
369 369 self.flagDeflipData = False # asumo q la data no esta sin flip
370 370 self.flagShiftFFT = False
371 371 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
372 372 self.profileIndex = 0
373 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
373 self.metadata_list = ['type', 'heightList', 'timeZone', 'nProfiles', 'channelList', 'nCohInt',
374 374 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp']
375 375
376 376 def getNoisebyHildebrand(self, channel=None):
377 377 """
378 378 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
379 379
380 380 Return:
381 381 noiselevel
382 382 """
383 383
384 384 if channel != None:
385 385 data = self.data[channel]
386 386 nChannels = 1
387 387 else:
388 388 data = self.data
389 389 nChannels = self.nChannels
390 390
391 391 noise = numpy.zeros(nChannels)
392 392 power = data * numpy.conjugate(data)
393 393
394 394 for thisChannel in range(nChannels):
395 395 if nChannels == 1:
396 396 daux = power[:].real
397 397 else:
398 398 daux = power[thisChannel, :].real
399 399 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
400 400
401 401 return noise
402 402
403 403 def getNoise(self, type=1, channel=None):
404 404
405 405 if type == 1:
406 406 noise = self.getNoisebyHildebrand(channel)
407 407
408 408 return noise
409 409
410 410 def getPower(self, channel=None):
411 411
412 412 if channel != None:
413 413 data = self.data[channel]
414 414 else:
415 415 data = self.data
416 416
417 417 power = data * numpy.conjugate(data)
418 418 powerdB = 10 * numpy.log10(power.real)
419 419 powerdB = numpy.squeeze(powerdB)
420 420
421 421 return powerdB
422 422
423 423 @property
424 424 def timeInterval(self):
425 425
426 426 return self.ippSeconds * self.nCohInt
427 427
428 428 noise = property(getNoise, "I'm the 'nHeights' property.")
429 429
430 430
431 431 class Spectra(JROData):
432 432
433 433 def __init__(self):
434 434 '''
435 435 Constructor
436 436 '''
437 437
438 438 self.useLocalTime = True
439 439 self.radarControllerHeaderObj = RadarControllerHeader()
440 440 self.systemHeaderObj = SystemHeader()
441 441 self.type = "Spectra"
442 442 self.timeZone = 0
443 443 self.nProfiles = None
444 444 self.heightList = None
445 445 self.channelList = None
446 446 self.pairsList = None
447 447 self.flagNoData = True
448 448 self.flagDiscontinuousBlock = False
449 449 self.utctime = None
450 450 self.nCohInt = None
451 451 self.nIncohInt = None
452 452 self.blocksize = None
453 453 self.nFFTPoints = None
454 454 self.wavelength = None
455 455 self.flagDecodeData = False # asumo q la data no esta decodificada
456 456 self.flagDeflipData = False # asumo q la data no esta sin flip
457 457 self.flagShiftFFT = False
458 458 self.ippFactor = 1
459 459 self.beacon_heiIndexList = []
460 460 self.noise_estimation = None
461 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
462 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
461 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
462 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp', 'nIncohInt', 'nFFTPoints', 'nProfiles']
463 463
464 464 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
465 465 """
466 466 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
467 467
468 468 Return:
469 469 noiselevel
470 470 """
471 471
472 472 noise = numpy.zeros(self.nChannels)
473 473
474 474 for channel in range(self.nChannels):
475 475 daux = self.data_spc[channel,
476 476 xmin_index:xmax_index, ymin_index:ymax_index]
477 477 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
478 478
479 479 return noise
480 480
481 481 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
482 482
483 483 if self.noise_estimation is not None:
484 484 # this was estimated by getNoise Operation defined in jroproc_spectra.py
485 485 return self.noise_estimation
486 486 else:
487 487 noise = self.getNoisebyHildebrand(
488 488 xmin_index, xmax_index, ymin_index, ymax_index)
489 489 return noise
490 490
491 491 def getFreqRangeTimeResponse(self, extrapoints=0):
492 492
493 493 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
494 494 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
495 495
496 496 return freqrange
497 497
498 498 def getAcfRange(self, extrapoints=0):
499 499
500 500 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
501 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
501 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
502 502
503 503 return freqrange
504 504
505 505 def getFreqRange(self, extrapoints=0):
506 506
507 507 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
508 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) -self.nFFTPoints / 2.) - deltafreq / 2
508 freqrange = deltafreq * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.) - deltafreq / 2
509 509
510 510 return freqrange
511 511
512 512 def getVelRange(self, extrapoints=0):
513 513
514 514 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
515 515 velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
516 516
517 517 if self.nmodes:
518 return velrange/self.nmodes
518 return velrange / self.nmodes
519 519 else:
520 520 return velrange
521 521
522 522 @property
523 523 def nPairs(self):
524 524
525 525 return len(self.pairsList)
526 526
527 527 @property
528 528 def pairsIndexList(self):
529 529
530 530 return list(range(self.nPairs))
531 531
532 532 @property
533 533 def normFactor(self):
534 534
535 535 pwcode = 1
536 536
537 537 if self.flagDecodeData:
538 pwcode = numpy.sum(self.code[0]**2)
539 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
538 pwcode = numpy.sum(self.code[0] ** 2)
539 # normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
540 540 normFactor = self.nProfiles * self.nIncohInt * self.nCohInt * pwcode * self.windowOfFilter
541 541
542 542 return normFactor
543 543
544 544 @property
545 545 def flag_cspc(self):
546 546
547 547 if self.data_cspc is None:
548 548 return True
549 549
550 550 return False
551 551
552 552 @property
553 553 def flag_dc(self):
554 554
555 555 if self.data_dc is None:
556 556 return True
557 557
558 558 return False
559 559
560 560 @property
561 561 def timeInterval(self):
562 562
563 563 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles * self.ippFactor
564 564 if self.nmodes:
565 return self.nmodes*timeInterval
565 return self.nmodes * timeInterval
566 566 else:
567 567 return timeInterval
568 568
569 569 def getPower(self):
570 570
571 571 factor = self.normFactor
572 572 z = self.data_spc / factor
573 573 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
574 574 avg = numpy.average(z, axis=1)
575 575
576 576 return 10 * numpy.log10(avg)
577 577
578 578 def getCoherence(self, pairsList=None, phase=False):
579 579
580 580 z = []
581 581 if pairsList is None:
582 582 pairsIndexList = self.pairsIndexList
583 583 else:
584 584 pairsIndexList = []
585 585 for pair in pairsList:
586 586 if pair not in self.pairsList:
587 587 raise ValueError("Pair %s is not in dataOut.pairsList" % (
588 588 pair))
589 589 pairsIndexList.append(self.pairsList.index(pair))
590 590 for i in range(len(pairsIndexList)):
591 591 pair = self.pairsList[pairsIndexList[i]]
592 592 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
593 593 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
594 594 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
595 595 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
596 596 if phase:
597 597 data = numpy.arctan2(avgcoherenceComplex.imag,
598 598 avgcoherenceComplex.real) * 180 / numpy.pi
599 599 else:
600 600 data = numpy.abs(avgcoherenceComplex)
601 601
602 602 z.append(data)
603 603
604 604 return numpy.array(z)
605 605
606 606 def setValue(self, value):
607 607
608 608 print("This property should not be initialized")
609 609
610 610 return
611 611
612 612 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
613 613
614 614
615 615 class SpectraHeis(Spectra):
616 616
617 617 def __init__(self):
618 618
619 619 self.radarControllerHeaderObj = RadarControllerHeader()
620 620 self.systemHeaderObj = SystemHeader()
621 621 self.type = "SpectraHeis"
622 622 self.nProfiles = None
623 623 self.heightList = None
624 624 self.channelList = None
625 625 self.flagNoData = True
626 626 self.flagDiscontinuousBlock = False
627 627 self.utctime = None
628 628 self.blocksize = None
629 629 self.profileIndex = 0
630 630 self.nCohInt = 1
631 631 self.nIncohInt = 1
632 632
633 633 @property
634 634 def normFactor(self):
635 635 pwcode = 1
636 636 if self.flagDecodeData:
637 pwcode = numpy.sum(self.code[0]**2)
637 pwcode = numpy.sum(self.code[0] ** 2)
638 638
639 639 normFactor = self.nIncohInt * self.nCohInt * pwcode
640 640
641 641 return normFactor
642 642
643 643 @property
644 644 def timeInterval(self):
645 645
646 646 return self.ippSeconds * self.nCohInt * self.nIncohInt
647 647
648 648
649 649 class Fits(JROData):
650 650
651 651 def __init__(self):
652 652
653 653 self.type = "Fits"
654 654 self.nProfiles = None
655 655 self.heightList = None
656 656 self.channelList = None
657 657 self.flagNoData = True
658 658 self.utctime = None
659 659 self.nCohInt = 1
660 660 self.nIncohInt = 1
661 661 self.useLocalTime = True
662 662 self.profileIndex = 0
663 663 self.timeZone = 0
664 664
665 665 def getTimeRange(self):
666 666
667 667 datatime = []
668 668
669 669 datatime.append(self.ltctime)
670 670 datatime.append(self.ltctime + self.timeInterval)
671 671
672 672 datatime = numpy.array(datatime)
673 673
674 674 return datatime
675 675
676 676 def getChannelIndexList(self):
677 677
678 678 return list(range(self.nChannels))
679 679
680 680 def getNoise(self, type=1):
681 681
682 682
683 683 if type == 1:
684 684 noise = self.getNoisebyHildebrand()
685 685
686 686 if type == 2:
687 687 noise = self.getNoisebySort()
688 688
689 689 if type == 3:
690 690 noise = self.getNoisebyWindow()
691 691
692 692 return noise
693 693
694 694 @property
695 695 def timeInterval(self):
696 696
697 697 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
698 698
699 699 return timeInterval
700 700
701 701 @property
702 702 def ippSeconds(self):
703 703 '''
704 704 '''
705 705 return self.ipp_sec
706 706
707 707 noise = property(getNoise, "I'm the 'nHeights' property.")
708 708
709 709
710 710 class Correlation(JROData):
711 711
712 712 def __init__(self):
713 713 '''
714 714 Constructor
715 715 '''
716 716 self.radarControllerHeaderObj = RadarControllerHeader()
717 717 self.systemHeaderObj = SystemHeader()
718 718 self.type = "Correlation"
719 719 self.data = None
720 720 self.dtype = None
721 721 self.nProfiles = None
722 722 self.heightList = None
723 723 self.channelList = None
724 724 self.flagNoData = True
725 725 self.flagDiscontinuousBlock = False
726 726 self.utctime = None
727 727 self.timeZone = 0
728 728 self.dstFlag = None
729 729 self.errorCount = None
730 730 self.blocksize = None
731 731 self.flagDecodeData = False # asumo q la data no esta decodificada
732 732 self.flagDeflipData = False # asumo q la data no esta sin flip
733 733 self.pairsList = None
734 734 self.nPoints = None
735 735
736 736 def getPairsList(self):
737 737
738 738 return self.pairsList
739 739
740 740 def getNoise(self, mode=2):
741 741
742 742 indR = numpy.where(self.lagR == 0)[0][0]
743 743 indT = numpy.where(self.lagT == 0)[0][0]
744 744
745 745 jspectra0 = self.data_corr[:, :, indR, :]
746 746 jspectra = copy.copy(jspectra0)
747 747
748 748 num_chan = jspectra.shape[0]
749 749 num_hei = jspectra.shape[2]
750 750
751 751 freq_dc = jspectra.shape[1] / 2
752 752 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
753 753
754 754 if ind_vel[0] < 0:
755 755 ind_vel[list(range(0, 1))] = ind_vel[list(
756 756 range(0, 1))] + self.num_prof
757 757
758 758 if mode == 1:
759 759 jspectra[:, freq_dc, :] = (
760 760 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
761 761
762 762 if mode == 2:
763 763
764 764 vel = numpy.array([-2, -1, 1, 2])
765 765 xx = numpy.zeros([4, 4])
766 766
767 767 for fil in range(4):
768 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
768 xx[fil, :] = vel[fil] ** numpy.asarray(list(range(4)))
769 769
770 770 xx_inv = numpy.linalg.inv(xx)
771 771 xx_aux = xx_inv[0, :]
772 772
773 773 for ich in range(num_chan):
774 774 yy = jspectra[ich, ind_vel, :]
775 775 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
776 776
777 777 junkid = jspectra[ich, freq_dc, :] <= 0
778 778 cjunkid = sum(junkid)
779 779
780 780 if cjunkid.any():
781 781 jspectra[ich, freq_dc, junkid.nonzero()] = (
782 782 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
783 783
784 784 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
785 785
786 786 return noise
787 787
788 788 @property
789 789 def timeInterval(self):
790 790
791 791 return self.ippSeconds * self.nCohInt * self.nProfiles
792 792
793 793 def splitFunctions(self):
794 794
795 795 pairsList = self.pairsList
796 796 ccf_pairs = []
797 797 acf_pairs = []
798 798 ccf_ind = []
799 799 acf_ind = []
800 800 for l in range(len(pairsList)):
801 801 chan0 = pairsList[l][0]
802 802 chan1 = pairsList[l][1]
803 803
804 804 # Obteniendo pares de Autocorrelacion
805 805 if chan0 == chan1:
806 806 acf_pairs.append(chan0)
807 807 acf_ind.append(l)
808 808 else:
809 809 ccf_pairs.append(pairsList[l])
810 810 ccf_ind.append(l)
811 811
812 812 data_acf = self.data_cf[acf_ind]
813 813 data_ccf = self.data_cf[ccf_ind]
814 814
815 815 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
816 816
817 817 @property
818 818 def normFactor(self):
819 819 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
820 820 acf_pairs = numpy.array(acf_pairs)
821 821 normFactor = numpy.zeros((self.nPairs, self.nHeights))
822 822
823 823 for p in range(self.nPairs):
824 824 pair = self.pairsList[p]
825 825
826 826 ch0 = pair[0]
827 827 ch1 = pair[1]
828 828
829 829 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
830 830 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
831 831 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
832 832
833 833 return normFactor
834 834
835 835
836 836 class Parameters(Spectra):
837 837
838 838 groupList = None # List of Pairs, Groups, etc
839 839 data_param = None # Parameters obtained
840 840 data_pre = None # Data Pre Parametrization
841 841 data_SNR = None # Signal to Noise Ratio
842 842 abscissaList = None # Abscissa, can be velocities, lags or time
843 843 utctimeInit = None # Initial UTC time
844 844 paramInterval = None # Time interval to calculate Parameters in seconds
845 845 useLocalTime = True
846 846 # Fitting
847 847 data_error = None # Error of the estimation
848 848 constants = None
849 849 library = None
850 850 # Output signal
851 851 outputInterval = None # Time interval to calculate output signal in seconds
852 852 data_output = None # Out signal
853 853 nAvg = None
854 854 noise_estimation = None
855 855 GauSPC = None # Fit gaussian SPC
856 856
857 857 def __init__(self):
858 858 '''
859 859 Constructor
860 860 '''
861 861 self.radarControllerHeaderObj = RadarControllerHeader()
862 862 self.systemHeaderObj = SystemHeader()
863 863 self.type = "Parameters"
864 864 self.timeZone = 0
865 865
866 866 def getTimeRange1(self, interval):
867 867
868 868 datatime = []
869 869
870 870 if self.useLocalTime:
871 871 time1 = self.utctimeInit - self.timeZone * 60
872 872 else:
873 873 time1 = self.utctimeInit
874 874
875 875 datatime.append(time1)
876 876 datatime.append(time1 + interval)
877 877 datatime = numpy.array(datatime)
878 878
879 879 return datatime
880 880
881 881 @property
882 882 def timeInterval(self):
883 883
884 884 if hasattr(self, 'timeInterval1'):
885 885 return self.timeInterval1
886 886 else:
887 887 return self.paramInterval
888 888
889 889 def setValue(self, value):
890 890
891 891 print("This property should not be initialized")
892 892
893 893 return
894 894
895 895 def getNoise(self):
896 896
897 897 return self.spc_noise
898 898
899 899 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
900 900
901 901
902 902 class PlotterData(object):
903 903 '''
904 904 Object to hold data to be plotted
905 905 '''
906 906
907 907 MAXNUMX = 200
908 908 MAXNUMY = 200
909 909
910 910 def __init__(self, code, exp_code, localtime=True):
911 911
912 912 self.key = code
913 913 self.exp_code = exp_code
914 914 self.ready = False
915 915 self.flagNoData = False
916 916 self.localtime = localtime
917 917 self.data = {}
918 918 self.meta = {}
919 919 self.__heights = []
920 920
921 921 def __str__(self):
922 922 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
923 923 return 'Data[{}][{}]'.format(';'.join(dum), len(self.times))
924 924
925 925 def __len__(self):
926 926 return len(self.data)
927 927
928 928 def __getitem__(self, key):
929 929 if isinstance(key, int):
930 930 return self.data[self.times[key]]
931 931 elif isinstance(key, str):
932 932 ret = numpy.array([self.data[x][key] for x in self.times])
933 933 if ret.ndim > 1:
934 934 ret = numpy.swapaxes(ret, 0, 1)
935 935 return ret
936 936
937 937 def __contains__(self, key):
938 938 return key in self.data[self.min_time]
939 939
940 940 def setup(self):
941 941 '''
942 942 Configure object
943 943 '''
944 944 self.type = ''
945 945 self.ready = False
946 946 del self.data
947 947 self.data = {}
948 948 self.__heights = []
949 949 self.__all_heights = set()
950 950
951 951 def shape(self, key):
952 952 '''
953 953 Get the shape of the one-element data for the given key
954 954 '''
955 955
956 956 if len(self.data[self.min_time][key]):
957 957 return self.data[self.min_time][key].shape
958 958 return (0,)
959 959
960 960 def update(self, data, tm, meta={}):
961 961 '''
962 962 Update data object with new dataOut
963 963 '''
964 964
965 965 self.data[tm] = data
966 966
967 967 for key, value in meta.items():
968 968 setattr(self, key, value)
969 969
970 970 def normalize_heights(self):
971 971 '''
972 972 Ensure same-dimension of the data for different heighList
973 973 '''
974 974
975 975 H = numpy.array(list(self.__all_heights))
976 976 H.sort()
977 977 for key in self.data:
978 978 shape = self.shape(key)[:-1] + H.shape
979 979 for tm, obj in list(self.data[key].items()):
980 980 h = self.__heights[self.times.tolist().index(tm)]
981 981 if H.size == h.size:
982 982 continue
983 983 index = numpy.where(numpy.in1d(H, h))[0]
984 984 dummy = numpy.zeros(shape) + numpy.nan
985 985 if len(shape) == 2:
986 986 dummy[:, index] = obj
987 987 else:
988 988 dummy[index] = obj
989 989 self.data[key][tm] = dummy
990 990
991 991 self.__heights = [H for tm in self.times]
992 992
993 993 def jsonify(self, tm, plot_name, plot_type, decimate=False):
994 994 '''
995 995 Convert data to json
996 996 '''
997 997
998 998 meta = {}
999 999 meta['xrange'] = []
1000 dy = int(len(self.yrange)/self.MAXNUMY) + 1
1000 dy = int(len(self.yrange) / self.MAXNUMY) + 1
1001 1001 tmp = self.data[tm][self.key]
1002 1002 shape = tmp.shape
1003 1003 if len(shape) == 2:
1004 1004 data = self.roundFloats(self.data[tm][self.key][::, ::dy].tolist())
1005 1005 elif len(shape) == 3:
1006 dx = int(self.data[tm][self.key].shape[1]/self.MAXNUMX) + 1
1006 dx = int(self.data[tm][self.key].shape[1] / self.MAXNUMX) + 1
1007 1007 data = self.roundFloats(
1008 1008 self.data[tm][self.key][::, ::dx, ::dy].tolist())
1009 1009 meta['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1010 1010 else:
1011 1011 data = self.roundFloats(self.data[tm][self.key].tolist())
1012 1012
1013 1013 ret = {
1014 1014 'plot': plot_name,
1015 1015 'code': self.exp_code,
1016 1016 'time': float(tm),
1017 1017 'data': data,
1018 1018 }
1019 1019 meta['type'] = plot_type
1020 1020 meta['interval'] = float(self.interval)
1021 1021 meta['localtime'] = self.localtime
1022 1022 meta['yrange'] = self.roundFloats(self.yrange[::dy].tolist())
1023 1023 meta.update(self.meta)
1024 1024 ret['metadata'] = meta
1025 1025 return json.dumps(ret)
1026 1026
1027 1027 @property
1028 1028 def times(self):
1029 1029 '''
1030 1030 Return the list of times of the current data
1031 1031 '''
1032 1032
1033 1033 ret = [t for t in self.data]
1034 1034 ret.sort()
1035 1035 return numpy.array(ret)
1036 1036
1037 1037 @property
1038 1038 def min_time(self):
1039 1039 '''
1040 1040 Return the minimun time value
1041 1041 '''
1042 1042
1043 1043 return self.times[0]
1044 1044
1045 1045 @property
1046 1046 def max_time(self):
1047 1047 '''
1048 1048 Return the maximun time value
1049 1049 '''
1050 1050
1051 1051 return self.times[-1]
1052 1052
1053 1053 # @property
1054 1054 # def heights(self):
1055 1055 # '''
1056 1056 # Return the list of heights of the current data
1057 1057 # '''
1058 1058
1059 1059 # return numpy.array(self.__heights[-1])
1060 1060
1061 1061 @staticmethod
1062 1062 def roundFloats(obj):
1063 1063 if isinstance(obj, list):
1064 1064 return list(map(PlotterData.roundFloats, obj))
1065 1065 elif isinstance(obj, float):
1066 1066 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,906 +1,906
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 5 '''
6 6 import sys
7 7 import numpy
8 8 import copy
9 9 import datetime
10 10 import inspect
11 11 from schainpy.utils import log
12 12
13 13 SPEED_OF_LIGHT = 299792458
14 14 SPEED_OF_LIGHT = 3e8
15 15
16 16 BASIC_STRUCTURE = numpy.dtype([
17 17 ('nSize', '<u4'),
18 18 ('nVersion', '<u2'),
19 19 ('nDataBlockId', '<u4'),
20 20 ('nUtime', '<u4'),
21 21 ('nMilsec', '<u2'),
22 22 ('nTimezone', '<i2'),
23 23 ('nDstflag', '<i2'),
24 24 ('nErrorCount', '<u4')
25 25 ])
26 26
27 27 SYSTEM_STRUCTURE = numpy.dtype([
28 28 ('nSize', '<u4'),
29 29 ('nNumSamples', '<u4'),
30 30 ('nNumProfiles', '<u4'),
31 31 ('nNumChannels', '<u4'),
32 32 ('nADCResolution', '<u4'),
33 33 ('nPCDIOBusWidth', '<u4'),
34 34 ])
35 35
36 36 RADAR_STRUCTURE = numpy.dtype([
37 37 ('nSize', '<u4'),
38 38 ('nExpType', '<u4'),
39 39 ('nNTx', '<u4'),
40 40 ('fIpp', '<f4'),
41 41 ('fTxA', '<f4'),
42 42 ('fTxB', '<f4'),
43 43 ('nNumWindows', '<u4'),
44 44 ('nNumTaus', '<u4'),
45 45 ('nCodeType', '<u4'),
46 46 ('nLine6Function', '<u4'),
47 47 ('nLine5Function', '<u4'),
48 48 ('fClock', '<f4'),
49 49 ('nPrePulseBefore', '<u4'),
50 50 ('nPrePulseAfter', '<u4'),
51 51 ('sRangeIPP', '<a20'),
52 52 ('sRangeTxA', '<a20'),
53 53 ('sRangeTxB', '<a20'),
54 54 ])
55 55
56 56 SAMPLING_STRUCTURE = numpy.dtype(
57 57 [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')])
58 58
59 59
60 60 PROCESSING_STRUCTURE = numpy.dtype([
61 61 ('nSize', '<u4'),
62 62 ('nDataType', '<u4'),
63 63 ('nSizeOfDataBlock', '<u4'),
64 64 ('nProfilesperBlock', '<u4'),
65 65 ('nDataBlocksperFile', '<u4'),
66 66 ('nNumWindows', '<u4'),
67 67 ('nProcessFlags', '<u4'),
68 68 ('nCoherentIntegrations', '<u4'),
69 69 ('nIncoherentIntegrations', '<u4'),
70 70 ('nTotalSpectra', '<u4')
71 71 ])
72 72
73 73
74 74 class Header(object):
75 75
76 76 def __init__(self):
77 77 raise NotImplementedError
78 78
79 79 def copy(self):
80 80 return copy.deepcopy(self)
81 81
82 82 def read(self):
83 83
84 84 raise NotImplementedError
85 85
86 86 def write(self):
87 87
88 88 raise NotImplementedError
89 89
90 90 def getAllowedArgs(self):
91 91 args = inspect.getargspec(self.__init__).args
92 92 try:
93 93 args.remove('self')
94 94 except:
95 95 pass
96 96 return args
97 97
98 98 def getAsDict(self):
99 99 args = self.getAllowedArgs()
100 100 asDict = {}
101 101 for x in args:
102 102 asDict[x] = self[x]
103 103 return asDict
104 104
105 105 def __getitem__(self, name):
106 106 return getattr(self, name)
107 107
108 108 def printInfo(self):
109 109
110 110 message = "#" * 50 + "\n"
111 111 message += self.__class__.__name__.upper() + "\n"
112 112 message += "#" * 50 + "\n"
113 113
114 114 keyList = list(self.__dict__.keys())
115 115 keyList.sort()
116 116
117 117 for key in keyList:
118 118 message += "%s = %s" % (key, self.__dict__[key]) + "\n"
119 119
120 120 if "size" not in keyList:
121 121 attr = getattr(self, "size")
122 122
123 123 if attr:
124 124 message += "%s = %s" % ("size", attr) + "\n"
125 125
126 126 print(message)
127 127
128 128
129 129 class BasicHeader(Header):
130 130
131 131 size = None
132 132 version = None
133 133 dataBlock = None
134 134 utc = None
135 135 ltc = None
136 136 miliSecond = None
137 137 timeZone = None
138 138 dstFlag = None
139 139 errorCount = None
140 140 datatime = None
141 141 structure = BASIC_STRUCTURE
142 142 __LOCALTIME = None
143 143
144 144 def __init__(self, useLocalTime=True):
145 145
146 146 self.size = 24
147 147 self.version = 0
148 148 self.dataBlock = 0
149 149 self.utc = 0
150 150 self.miliSecond = 0
151 151 self.timeZone = 0
152 152 self.dstFlag = 0
153 153 self.errorCount = 0
154 154
155 155 self.useLocalTime = useLocalTime
156 156
157 157 def read(self, fp):
158 158
159 159 self.length = 0
160 160 try:
161 161 if hasattr(fp, 'read'):
162 162 header = numpy.fromfile(fp, BASIC_STRUCTURE, 1)
163 163 else:
164 164 header = numpy.fromstring(fp, BASIC_STRUCTURE, 1)
165 165 except Exception as e:
166 166 print("BasicHeader: ")
167 167 print(e)
168 168 return 0
169 169
170 170 self.size = int(header['nSize'][0])
171 171 self.version = int(header['nVersion'][0])
172 172 self.dataBlock = int(header['nDataBlockId'][0])
173 173 self.utc = int(header['nUtime'][0])
174 174 self.miliSecond = int(header['nMilsec'][0])
175 175 self.timeZone = int(header['nTimezone'][0])
176 176 self.dstFlag = int(header['nDstflag'][0])
177 177 self.errorCount = int(header['nErrorCount'][0])
178 178
179 179 if self.size < 24:
180 180 return 0
181 181
182 182 self.length = header.nbytes
183 183 return 1
184 184
185 185 def write(self, fp):
186 186
187 187 headerTuple = (self.size, self.version, self.dataBlock, self.utc,
188 188 self.miliSecond, self.timeZone, self.dstFlag, self.errorCount)
189 189 header = numpy.array(headerTuple, BASIC_STRUCTURE)
190 190 header.tofile(fp)
191 191
192 192 return 1
193 193
194 194 def get_ltc(self):
195 195
196 196 return self.utc - self.timeZone * 60
197 197
198 198 def set_ltc(self, value):
199 199
200 200 self.utc = value + self.timeZone * 60
201 201
202 202 def get_datatime(self):
203 203
204 204 return datetime.datetime.utcfromtimestamp(self.ltc)
205 205
206 206 ltc = property(get_ltc, set_ltc)
207 207 datatime = property(get_datatime)
208 208
209 209
210 210 class SystemHeader(Header):
211 211
212 212 size = None
213 213 nSamples = None
214 214 nProfiles = None
215 215 nChannels = None
216 216 adcResolution = None
217 217 pciDioBusWidth = None
218 218 structure = SYSTEM_STRUCTURE
219 219
220 220 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
221 221
222 222 self.size = 24
223 223 self.nSamples = nSamples
224 224 self.nProfiles = nProfiles
225 225 self.nChannels = nChannels
226 226 self.adcResolution = adcResolution
227 227 self.pciDioBusWidth = pciDioBusWidth
228 228
229 229 def read(self, fp):
230 230 self.length = 0
231 231 try:
232 232 startFp = fp.tell()
233 233 except Exception as e:
234 234 startFp = None
235 235 pass
236 236
237 237 try:
238 238 if hasattr(fp, 'read'):
239 239 header = numpy.fromfile(fp, SYSTEM_STRUCTURE, 1)
240 240 else:
241 241 header = numpy.fromstring(fp, SYSTEM_STRUCTURE, 1)
242 242 except Exception as e:
243 243 print("System Header: " + str(e))
244 244 return 0
245 245
246 246 self.size = header['nSize'][0]
247 247 self.nSamples = header['nNumSamples'][0]
248 248 self.nProfiles = header['nNumProfiles'][0]
249 249 self.nChannels = header['nNumChannels'][0]
250 250 self.adcResolution = header['nADCResolution'][0]
251 251 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
252 252
253 253 if startFp is not None:
254 254 endFp = self.size + startFp
255 255
256 256 if fp.tell() > endFp:
257 257 sys.stderr.write(
258 258 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp.name)
259 259 return 0
260 260
261 261 if fp.tell() < endFp:
262 262 sys.stderr.write(
263 263 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp.name)
264 264 return 0
265 265
266 266 self.length = header.nbytes
267 267 return 1
268 268
269 269 def write(self, fp):
270 270
271 271 headerTuple = (self.size, self.nSamples, self.nProfiles,
272 272 self.nChannels, self.adcResolution, self.pciDioBusWidth)
273 273 header = numpy.array(headerTuple, SYSTEM_STRUCTURE)
274 274 header.tofile(fp)
275 275
276 276 return 1
277 277
278 278
279 279 class RadarControllerHeader(Header):
280 280
281 281 expType = None
282 282 nTx = None
283 283 ipp = None
284 284 txA = None
285 285 txB = None
286 286 nWindows = None
287 287 numTaus = None
288 288 codeType = None
289 289 line6Function = None
290 290 line5Function = None
291 291 fClock = None
292 292 prePulseBefore = None
293 293 prePulseAfter = None
294 294 rangeIpp = None
295 295 rangeTxA = None
296 296 rangeTxB = None
297 297 structure = RADAR_STRUCTURE
298 298 __size = None
299 299
300 300 def __init__(self, expType=2, nTx=1,
301 301 ipp=None, txA=0, txB=0,
302 302 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
303 303 numTaus=0, line6Function=0, line5Function=0, fClock=None,
304 304 prePulseBefore=0, prePulseAfter=0,
305 305 codeType=0, nCode=0, nBaud=0, code=[],
306 306 flip1=0, flip2=0):
307 307
308 308 # self.size = 116
309 309 self.expType = expType
310 310 self.nTx = nTx
311 311 self.ipp = ipp
312 312 self.txA = txA
313 313 self.txB = txB
314 314 self.rangeIpp = ipp
315 315 self.rangeTxA = txA
316 316 self.rangeTxB = txB
317 317
318 318 self.nWindows = nWindows
319 319 self.numTaus = numTaus
320 320 self.codeType = codeType
321 321 self.line6Function = line6Function
322 322 self.line5Function = line5Function
323 323 self.fClock = fClock
324 324 self.prePulseBefore = prePulseBefore
325 325 self.prePulseAfter = prePulseAfter
326 326
327 327 self.nHeights = nHeights
328 328 self.firstHeight = firstHeight
329 329 self.deltaHeight = deltaHeight
330 330 self.samplesWin = nHeights
331 331
332 332 self.nCode = nCode
333 333 self.nBaud = nBaud
334 334 self.code = code
335 335 self.flip1 = flip1
336 336 self.flip2 = flip2
337 337
338 338 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
339 339 # self.dynamic = numpy.array([],numpy.dtype('byte'))
340 340
341 341 if self.fClock is None and self.deltaHeight is not None:
342 342 self.fClock = 0.15 / (deltaHeight * 1e-6) # 0.15Km / (height * 1u)
343 343
344 344 def read(self, fp):
345 345 self.length = 0
346 346 try:
347 347 startFp = fp.tell()
348 348 except Exception as e:
349 349 startFp = None
350 350 pass
351 351
352 352 try:
353 353 if hasattr(fp, 'read'):
354 354 header = numpy.fromfile(fp, RADAR_STRUCTURE, 1)
355 355 else:
356 356 header = numpy.fromstring(fp, RADAR_STRUCTURE, 1)
357 357 self.length += header.nbytes
358 358 except Exception as e:
359 359 print("RadarControllerHeader: " + str(e))
360 360 return 0
361 361
362 362 size = int(header['nSize'][0])
363 363 self.expType = int(header['nExpType'][0])
364 364 self.nTx = int(header['nNTx'][0])
365 365 self.ipp = float(header['fIpp'][0])
366 366 self.txA = float(header['fTxA'][0])
367 367 self.txB = float(header['fTxB'][0])
368 368 self.nWindows = int(header['nNumWindows'][0])
369 369 self.numTaus = int(header['nNumTaus'][0])
370 370 self.codeType = int(header['nCodeType'][0])
371 371 self.line6Function = int(header['nLine6Function'][0])
372 372 self.line5Function = int(header['nLine5Function'][0])
373 373 self.fClock = float(header['fClock'][0])
374 374 self.prePulseBefore = int(header['nPrePulseBefore'][0])
375 375 self.prePulseAfter = int(header['nPrePulseAfter'][0])
376 376 self.rangeIpp = header['sRangeIPP'][0]
377 377 self.rangeTxA = header['sRangeTxA'][0]
378 378 self.rangeTxB = header['sRangeTxB'][0]
379 379
380 380 try:
381 381 if hasattr(fp, 'read'):
382 382 samplingWindow = numpy.fromfile(
383 383 fp, SAMPLING_STRUCTURE, self.nWindows)
384 384 else:
385 385 samplingWindow = numpy.fromstring(
386 386 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
387 387 self.length += samplingWindow.nbytes
388 388 except Exception as e:
389 389 print("RadarControllerHeader: " + str(e))
390 390 return 0
391 391 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
392 392 self.firstHeight = samplingWindow['h0']
393 393 self.deltaHeight = samplingWindow['dh']
394 394 self.samplesWin = samplingWindow['nsa']
395 395
396 396 try:
397 397 if hasattr(fp, 'read'):
398 398 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
399 399 else:
400 400 self.Taus = numpy.fromstring(
401 401 fp[self.length:], '<f4', self.numTaus)
402 402 self.length += self.Taus.nbytes
403 403 except Exception as e:
404 404 print("RadarControllerHeader: " + str(e))
405 405 return 0
406 406
407 407 self.code_size = 0
408 408 if self.codeType != 0:
409 409
410 410 try:
411 411 if hasattr(fp, 'read'):
412 412 self.nCode = numpy.fromfile(fp, '<u4', 1)[0]
413 413 self.length += self.nCode.nbytes
414 414 self.nBaud = numpy.fromfile(fp, '<u4', 1)[0]
415 415 self.length += self.nBaud.nbytes
416 416 else:
417 417 self.nCode = numpy.fromstring(
418 418 fp[self.length:], '<u4', 1)[0]
419 419 self.length += self.nCode.nbytes
420 420 self.nBaud = numpy.fromstring(
421 421 fp[self.length:], '<u4', 1)[0]
422 422 self.length += self.nBaud.nbytes
423 423 except Exception as e:
424 424 print("RadarControllerHeader: " + str(e))
425 425 return 0
426 426 code = numpy.empty([self.nCode, self.nBaud], dtype='i1')
427 427
428 428 for ic in range(self.nCode):
429 429 try:
430 430 if hasattr(fp, 'read'):
431 431 temp = numpy.fromfile(fp, 'u4', int(
432 432 numpy.ceil(self.nBaud / 32.)))
433 433 else:
434 434 temp = numpy.fromstring(
435 435 fp, 'u4', int(numpy.ceil(self.nBaud / 32.)))
436 436 self.length += temp.nbytes
437 437 except Exception as e:
438 438 print("RadarControllerHeader: " + str(e))
439 439 return 0
440 440
441 441 for ib in range(self.nBaud - 1, -1, -1):
442 442 code[ic, ib] = temp[int(ib / 32)] % 2
443 443 temp[int(ib / 32)] = temp[int(ib / 32)] / 2
444 444
445 445 self.code = 2.0 * code - 1.0
446 446 self.code_size = int(numpy.ceil(self.nBaud / 32.)) * self.nCode * 4
447 447
448 448 # if self.line5Function == RCfunction.FLIP:
449 449 # self.flip1 = numpy.fromfile(fp,'<u4',1)
450 450 #
451 451 # if self.line6Function == RCfunction.FLIP:
452 452 # self.flip2 = numpy.fromfile(fp,'<u4',1)
453 453 if startFp is not None:
454 454 endFp = size + startFp
455 455
456 456 if fp.tell() != endFp:
457 457 # fp.seek(endFp)
458 458 print("%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" % (fp.name, fp.tell() - startFp, size))
459 459 # return 0
460 460
461 461 if fp.tell() > endFp:
462 462 sys.stderr.write(
463 463 "Warning %s: Size value read from Radar Controller header is lower than it has to be\n" % fp.name)
464 464 # return 0
465 465
466 466 if fp.tell() < endFp:
467 467 sys.stderr.write(
468 468 "Warning %s: Size value read from Radar Controller header is greater than it has to be\n" % fp.name)
469 469
470 470 return 1
471 471
472 472 def write(self, fp):
473 473
474 474 headerTuple = (self.size,
475 475 self.expType,
476 476 self.nTx,
477 477 self.ipp,
478 478 self.txA,
479 479 self.txB,
480 480 self.nWindows,
481 481 self.numTaus,
482 482 self.codeType,
483 483 self.line6Function,
484 484 self.line5Function,
485 485 self.fClock,
486 486 self.prePulseBefore,
487 487 self.prePulseAfter,
488 488 self.rangeIpp,
489 489 self.rangeTxA,
490 490 self.rangeTxB)
491 491
492 492 header = numpy.array(headerTuple, RADAR_STRUCTURE)
493 493 header.tofile(fp)
494 494
495 495 sampleWindowTuple = (
496 496 self.firstHeight, self.deltaHeight, self.samplesWin)
497 497 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
498 498 samplingWindow.tofile(fp)
499 499
500 500 if self.numTaus > 0:
501 501 self.Taus.tofile(fp)
502 502
503 503 if self.codeType != 0:
504 504 nCode = numpy.array(self.nCode, '<u4')
505 505 nCode.tofile(fp)
506 506 nBaud = numpy.array(self.nBaud, '<u4')
507 507 nBaud.tofile(fp)
508 508 code1 = (self.code + 1.0) / 2.
509 509
510 510 for ic in range(self.nCode):
511 511 tempx = numpy.zeros(int(numpy.ceil(self.nBaud / 32.)))
512 512 start = 0
513 513 end = 32
514 514 for i in range(len(tempx)):
515 515 code_selected = code1[ic, start:end]
516 516 for j in range(len(code_selected) - 1, -1, -1):
517 517 if code_selected[j] == 1:
518 518 tempx[i] = tempx[i] + \
519 2**(len(code_selected) - 1 - j)
519 2 ** (len(code_selected) - 1 - j)
520 520 start = start + 32
521 521 end = end + 32
522 522
523 523 tempx = tempx.astype('u4')
524 524 tempx.tofile(fp)
525 525
526 526 # if self.line5Function == RCfunction.FLIP:
527 527 # self.flip1.tofile(fp)
528 528 #
529 529 # if self.line6Function == RCfunction.FLIP:
530 530 # self.flip2.tofile(fp)
531 531
532 532 return 1
533 533
534 534 def get_ippSeconds(self):
535 535 '''
536 536 '''
537 537 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
538 538
539 539 return ippSeconds
540 540
541 541 def set_ippSeconds(self, ippSeconds):
542 542 '''
543 543 '''
544 544
545 545 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0 * 1000)
546 546
547 547 return
548 548
549 549 def get_size(self):
550 550
551 551 self.__size = 116 + 12 * self.nWindows + 4 * self.numTaus
552 552
553 553 if self.codeType != 0:
554 554 self.__size += 4 + 4 + 4 * self.nCode * \
555 555 numpy.ceil(self.nBaud / 32.)
556 556
557 557 return self.__size
558 558
559 559 def set_size(self, value):
560 560
561 561 raise IOError("size is a property and it cannot be set, just read")
562 562
563 563 return
564 564
565 565 ippSeconds = property(get_ippSeconds, set_ippSeconds)
566 566 size = property(get_size, set_size)
567 567
568 568
569 569 class ProcessingHeader(Header):
570 570
571 571 # size = None
572 572 dtype = None
573 573 blockSize = None
574 574 profilesPerBlock = None
575 575 dataBlocksPerFile = None
576 576 nWindows = None
577 577 processFlags = None
578 578 nCohInt = None
579 579 nIncohInt = None
580 580 totalSpectra = None
581 581 structure = PROCESSING_STRUCTURE
582 582 flag_dc = None
583 583 flag_cspc = None
584 584
585 585 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
586 586 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
587 587 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
588 588 ):
589 589
590 590 # self.size = 0
591 591 self.dtype = dtype
592 592 self.blockSize = blockSize
593 593 self.profilesPerBlock = 0
594 594 self.dataBlocksPerFile = 0
595 595 self.nWindows = 0
596 596 self.processFlags = 0
597 597 self.nCohInt = 0
598 598 self.nIncohInt = 0
599 599 self.totalSpectra = 0
600 600
601 601 self.nHeights = 0
602 602 self.firstHeight = 0
603 603 self.deltaHeight = 0
604 604 self.samplesWin = 0
605 605 self.spectraComb = 0
606 606 self.nCode = None
607 607 self.code = None
608 608 self.nBaud = None
609 609
610 610 self.shif_fft = False
611 611 self.flag_dc = False
612 612 self.flag_cspc = False
613 613 self.flag_decode = False
614 614 self.flag_deflip = False
615 615 self.length = 0
616 616
617 617 def read(self, fp):
618 618 self.length = 0
619 619 try:
620 620 startFp = fp.tell()
621 621 except Exception as e:
622 622 startFp = None
623 623 pass
624 624
625 625 try:
626 626 if hasattr(fp, 'read'):
627 627 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
628 628 else:
629 629 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
630 630 self.length += header.nbytes
631 631 except Exception as e:
632 632 print("ProcessingHeader: " + str(e))
633 633 return 0
634 634
635 635 size = int(header['nSize'][0])
636 636 self.dtype = int(header['nDataType'][0])
637 637 self.blockSize = int(header['nSizeOfDataBlock'][0])
638 638 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
639 639 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
640 640 self.nWindows = int(header['nNumWindows'][0])
641 641 self.processFlags = header['nProcessFlags']
642 642 self.nCohInt = int(header['nCoherentIntegrations'][0])
643 643 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
644 644 self.totalSpectra = int(header['nTotalSpectra'][0])
645 645
646 646 try:
647 647 if hasattr(fp, 'read'):
648 648 samplingWindow = numpy.fromfile(
649 649 fp, SAMPLING_STRUCTURE, self.nWindows)
650 650 else:
651 651 samplingWindow = numpy.fromstring(
652 652 fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
653 653 self.length += samplingWindow.nbytes
654 654 except Exception as e:
655 655 print("ProcessingHeader: " + str(e))
656 656 return 0
657 657
658 658 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
659 659 self.firstHeight = float(samplingWindow['h0'][0])
660 660 self.deltaHeight = float(samplingWindow['dh'][0])
661 661 self.samplesWin = samplingWindow['nsa'][0]
662 662
663 663 try:
664 664 if hasattr(fp, 'read'):
665 665 self.spectraComb = numpy.fromfile(
666 666 fp, 'u1', 2 * self.totalSpectra)
667 667 else:
668 668 self.spectraComb = numpy.fromstring(
669 669 fp[self.length:], 'u1', 2 * self.totalSpectra)
670 670 self.length += self.spectraComb.nbytes
671 671 except Exception as e:
672 672 print("ProcessingHeader: " + str(e))
673 673 return 0
674 674
675 675 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
676 676 self.nCode = int(numpy.fromfile(fp, '<u4', 1))
677 677 self.nBaud = int(numpy.fromfile(fp, '<u4', 1))
678 678 self.code = numpy.fromfile(
679 679 fp, '<f4', self.nCode * self.nBaud).reshape(self.nCode, self.nBaud)
680 680
681 681 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
682 682 exp_name_len = int(numpy.fromfile(fp, '<u4', 1))
683 683 exp_name = numpy.fromfile(fp, 'u1', exp_name_len + 1)
684 684
685 685 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
686 686 self.shif_fft = True
687 687 else:
688 688 self.shif_fft = False
689 689
690 690 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
691 691 self.flag_dc = True
692 692 else:
693 693 self.flag_dc = False
694 694
695 695 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
696 696 self.flag_decode = True
697 697 else:
698 698 self.flag_decode = False
699 699
700 700 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
701 701 self.flag_deflip = True
702 702 else:
703 703 self.flag_deflip = False
704 704
705 705 nChannels = 0
706 706 nPairs = 0
707 707 pairList = []
708 708
709 709 for i in range(0, self.totalSpectra * 2, 2):
710 710 if self.spectraComb[i] == self.spectraComb[i + 1]:
711 711 nChannels = nChannels + 1 # par de canales iguales
712 712 else:
713 713 nPairs = nPairs + 1 # par de canales diferentes
714 714 pairList.append((self.spectraComb[i], self.spectraComb[i + 1]))
715 715
716 716 self.flag_cspc = False
717 717 if nPairs > 0:
718 718 self.flag_cspc = True
719 719
720 720 if startFp is not None:
721 721 endFp = size + startFp
722 722 if fp.tell() > endFp:
723 723 sys.stderr.write(
724 724 "Warning: Processing header size is lower than it has to be")
725 725 return 0
726 726
727 727 if fp.tell() < endFp:
728 728 sys.stderr.write(
729 729 "Warning: Processing header size is greater than it is considered")
730 730
731 731 return 1
732 732
733 733 def write(self, fp):
734 734 # Clear DEFINE_PROCESS_CODE
735 735 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
736 736
737 737 headerTuple = (self.size,
738 738 self.dtype,
739 739 self.blockSize,
740 740 self.profilesPerBlock,
741 741 self.dataBlocksPerFile,
742 742 self.nWindows,
743 743 self.processFlags,
744 744 self.nCohInt,
745 745 self.nIncohInt,
746 746 self.totalSpectra)
747 747
748 748 header = numpy.array(headerTuple, PROCESSING_STRUCTURE)
749 749 header.tofile(fp)
750 750
751 751 if self.nWindows != 0:
752 752 sampleWindowTuple = (
753 753 self.firstHeight, self.deltaHeight, self.samplesWin)
754 754 samplingWindow = numpy.array(sampleWindowTuple, SAMPLING_STRUCTURE)
755 755 samplingWindow.tofile(fp)
756 756
757 757 if self.totalSpectra != 0:
758 758 # spectraComb = numpy.array([],numpy.dtype('u1'))
759 759 spectraComb = self.spectraComb
760 760 spectraComb.tofile(fp)
761 761
762 762 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
763 763 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
764 764 # nCode.tofile(fp)
765 765 #
766 766 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
767 767 # nBaud.tofile(fp)
768 768 #
769 769 # code = self.code.reshape(self.nCode*self.nBaud)
770 770 # code = code.astype(numpy.dtype('<f4'))
771 771 # code.tofile(fp)
772 772
773 773 return 1
774 774
775 775 def get_size(self):
776 776
777 777 self.__size = 40 + 12 * self.nWindows + 2 * self.totalSpectra
778 778
779 779 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
780 780 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
781 781 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
782 782
783 783 return self.__size
784 784
785 785 def set_size(self, value):
786 786
787 787 raise IOError("size is a property and it cannot be set, just read")
788 788
789 789 return
790 790
791 791 size = property(get_size, set_size)
792 792
793 793
794 794 class RCfunction:
795 795 NONE = 0
796 796 FLIP = 1
797 797 CODE = 2
798 798 SAMPLING = 3
799 799 LIN6DIV256 = 4
800 800 SYNCHRO = 5
801 801
802 802
803 803 class nCodeType:
804 804 NONE = 0
805 805 USERDEFINE = 1
806 806 BARKER2 = 2
807 807 BARKER3 = 3
808 808 BARKER4 = 4
809 809 BARKER5 = 5
810 810 BARKER7 = 6
811 811 BARKER11 = 7
812 812 BARKER13 = 8
813 813 AC128 = 9
814 814 COMPLEMENTARYCODE2 = 10
815 815 COMPLEMENTARYCODE4 = 11
816 816 COMPLEMENTARYCODE8 = 12
817 817 COMPLEMENTARYCODE16 = 13
818 818 COMPLEMENTARYCODE32 = 14
819 819 COMPLEMENTARYCODE64 = 15
820 820 COMPLEMENTARYCODE128 = 16
821 821 CODE_BINARY28 = 17
822 822
823 823
824 824 class PROCFLAG:
825 825
826 826 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
827 827 DECODE_DATA = numpy.uint32(0x00000002)
828 828 SPECTRA_CALC = numpy.uint32(0x00000004)
829 829 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
830 830 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
831 831 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
832 832
833 833 DATATYPE_CHAR = numpy.uint32(0x00000040)
834 834 DATATYPE_SHORT = numpy.uint32(0x00000080)
835 835 DATATYPE_LONG = numpy.uint32(0x00000100)
836 836 DATATYPE_INT64 = numpy.uint32(0x00000200)
837 837 DATATYPE_FLOAT = numpy.uint32(0x00000400)
838 838 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
839 839
840 840 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
841 841 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
842 842 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
843 843
844 844 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
845 845 DEFLIP_DATA = numpy.uint32(0x00010000)
846 846 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
847 847
848 848 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
849 849 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
850 850 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
851 851 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
852 852 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
853 853
854 854 EXP_NAME_ESP = numpy.uint32(0x00200000)
855 855 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
856 856
857 857 OPERATION_MASK = numpy.uint32(0x0000003F)
858 858 DATATYPE_MASK = numpy.uint32(0x00000FC0)
859 859 DATAARRANGE_MASK = numpy.uint32(0x00007000)
860 860 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
861 861
862 862
863 863 dtype0 = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
864 864 dtype1 = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
865 865 dtype2 = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
866 866 dtype3 = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
867 867 dtype4 = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
868 868 dtype5 = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
869 869
870 870 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
871 871
872 872 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
873 873 PROCFLAG.DATATYPE_SHORT,
874 874 PROCFLAG.DATATYPE_LONG,
875 875 PROCFLAG.DATATYPE_INT64,
876 876 PROCFLAG.DATATYPE_FLOAT,
877 877 PROCFLAG.DATATYPE_DOUBLE]
878 878
879 879 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
880 880
881 881
882 882 def get_dtype_index(numpy_dtype):
883 883
884 884 index = None
885 885
886 886 for i in range(len(NUMPY_DTYPE_LIST)):
887 887 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
888 888 index = i
889 889 break
890 890
891 891 return index
892 892
893 893
894 894 def get_numpy_dtype(index):
895 895
896 896 return NUMPY_DTYPE_LIST[index]
897 897
898 898
899 899 def get_procflag_dtype(index):
900 900
901 901 return PROCFLAG_DTYPE_LIST[index]
902 902
903 903
904 904 def get_dtype_width(index):
905 905
906 return DTYPE_WIDTH[index] No newline at end of file
906 return DTYPE_WIDTH[index]
Show file before | Show file after | Hide comments
@@ -1,691 +1,691
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Base class to create plot operations
6 6
7 7 """
8 8
9 9 import os
10 10 import sys
11 11 import zmq
12 12 import time
13 13 import numpy
14 14 import datetime
15 15 from collections import deque
16 16 from functools import wraps
17 17 from threading import Thread
18 18 import matplotlib
19 19
20 20 if 'BACKEND' in os.environ:
21 21 matplotlib.use(os.environ['BACKEND'])
22 22 elif 'linux' in sys.platform:
23 23 matplotlib.use("TkAgg")
24 24 elif 'darwin' in sys.platform:
25 25 matplotlib.use('MacOSX')
26 26 else:
27 27 from schainpy.utils import log
28 28 log.warning('Using default Backend="Agg"', 'INFO')
29 29 matplotlib.use('Agg')
30 30
31 31 import matplotlib.pyplot as plt
32 32 from matplotlib.patches import Polygon
33 33 from mpl_toolkits.axes_grid1 import make_axes_locatable
34 34 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
35 35
36 36 from schainpy.model.data.jrodata import PlotterData
37 37 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
38 38 from schainpy.utils import log
39 39
40 40 jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90]
41 41 blu_values = matplotlib.pyplot.get_cmap(
42 42 'seismic_r', 20)(numpy.arange(20))[10:15]
43 43 ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
44 44 'jro', numpy.vstack((blu_values, jet_values)))
45 45 matplotlib.pyplot.register_cmap(cmap=ncmap)
46 46
47 47 CMAPS = [plt.get_cmap(s) for s in ('jro', 'jet', 'viridis',
48 48 'plasma', 'inferno', 'Greys', 'seismic', 'bwr', 'coolwarm')]
49 49
50 50 EARTH_RADIUS = 6.3710e3
51 51
52 52 def ll2xy(lat1, lon1, lat2, lon2):
53 53
54 54 p = 0.017453292519943295
55 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
55 a = 0.5 - numpy.cos((lat2 - lat1) * p) / 2 + numpy.cos(lat1 * p) * \
56 56 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
57 57 r = 12742 * numpy.arcsin(numpy.sqrt(a))
58 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
59 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
60 theta = -theta + numpy.pi/2
61 return r*numpy.cos(theta), r*numpy.sin(theta)
58 theta = numpy.arctan2(numpy.sin((lon2 - lon1) * p) * numpy.cos(lat2 * p), numpy.cos(lat1 * p)
59 * numpy.sin(lat2 * p) - numpy.sin(lat1 * p) * numpy.cos(lat2 * p) * numpy.cos((lon2 - lon1) * p))
60 theta = -theta + numpy.pi / 2
61 return r * numpy.cos(theta), r * numpy.sin(theta)
62 62
63 63
64 64 def km2deg(km):
65 65 '''
66 66 Convert distance in km to degrees
67 67 '''
68 68
69 return numpy.rad2deg(km/EARTH_RADIUS)
69 return numpy.rad2deg(km / EARTH_RADIUS)
70 70
71 71
72 72 def figpause(interval):
73 73 backend = plt.rcParams['backend']
74 74 if backend in matplotlib.rcsetup.interactive_bk:
75 75 figManager = matplotlib._pylab_helpers.Gcf.get_active()
76 76 if figManager is not None:
77 77 canvas = figManager.canvas
78 78 if canvas.figure.stale:
79 79 canvas.draw()
80 80 try:
81 81 canvas.start_event_loop(interval)
82 82 except:
83 83 pass
84 84 return
85 85
86 86 def popup(message):
87 87 '''
88 88 '''
89 89
90 90 fig = plt.figure(figsize=(12, 8), facecolor='r')
91 91 text = '\n'.join([s.strip() for s in message.split(':')])
92 92 fig.text(0.01, 0.5, text, ha='left', va='center',
93 93 size='20', weight='heavy', color='w')
94 94 fig.show()
95 95 figpause(1000)
96 96
97 97
98 98 class Throttle(object):
99 99 '''
100 100 Decorator that prevents a function from being called more than once every
101 101 time period.
102 102 To create a function that cannot be called more than once a minute, but
103 103 will sleep until it can be called:
104 104 @Throttle(minutes=1)
105 105 def foo():
106 106 pass
107 107
108 108 for i in range(10):
109 109 foo()
110 110 print "This function has run %s times." % i
111 111 '''
112 112
113 113 def __init__(self, seconds=0, minutes=0, hours=0):
114 114 self.throttle_period = datetime.timedelta(
115 115 seconds=seconds, minutes=minutes, hours=hours
116 116 )
117 117
118 118 self.time_of_last_call = datetime.datetime.min
119 119
120 120 def __call__(self, fn):
121 121 @wraps(fn)
122 122 def wrapper(*args, **kwargs):
123 123 coerce = kwargs.pop('coerce', None)
124 124 if coerce:
125 125 self.time_of_last_call = datetime.datetime.now()
126 126 return fn(*args, **kwargs)
127 127 else:
128 128 now = datetime.datetime.now()
129 129 time_since_last_call = now - self.time_of_last_call
130 130 time_left = self.throttle_period - time_since_last_call
131 131
132 132 if time_left > datetime.timedelta(seconds=0):
133 133 return
134 134
135 135 self.time_of_last_call = datetime.datetime.now()
136 136 return fn(*args, **kwargs)
137 137
138 138 return wrapper
139 139
140 140 def apply_throttle(value):
141 141
142 142 @Throttle(seconds=value)
143 143 def fnThrottled(fn):
144 144 fn()
145 145
146 146 return fnThrottled
147 147
148 148
149 149 @MPDecorator
150 150 class Plot(Operation):
151 151 """Base class for Schain plotting operations
152 152
153 153 This class should never be use directtly you must subclass a new operation,
154 154 children classes must be defined as follow:
155 155
156 156 ExamplePlot(Plot):
157 157
158 158 CODE = 'code'
159 159 colormap = 'jet'
160 160 plot_type = 'pcolor' # options are ('pcolor', 'pcolorbuffer', 'scatter', 'scatterbuffer')
161 161
162 162 def setup(self):
163 163 pass
164 164
165 165 def plot(self):
166 166 pass
167 167
168 168 """
169 169
170 170 CODE = 'Figure'
171 171 colormap = 'jet'
172 172 bgcolor = 'white'
173 173 buffering = True
174 174 __missing = 1E30
175 175
176 176 __attrs__ = ['show', 'save', 'ymin', 'ymax', 'zmin', 'zmax', 'title',
177 177 'showprofile']
178 178
179 179 def __init__(self):
180 180
181 181 Operation.__init__(self)
182 182 self.isConfig = False
183 183 self.isPlotConfig = False
184 184 self.save_time = 0
185 185 self.sender_time = 0
186 186 self.data = None
187 187 self.firsttime = True
188 188 self.sender_queue = deque(maxlen=10)
189 189 self.plots_adjust = {'left': 0.125, 'right': 0.9, 'bottom': 0.15, 'top': 0.9, 'wspace': 0.2, 'hspace': 0.2}
190 190
191 191 def __fmtTime(self, x, pos):
192 192 '''
193 193 '''
194 194
195 195 return '{}'.format(self.getDateTime(x).strftime('%H:%M'))
196 196
197 197 def __setup(self, **kwargs):
198 198 '''
199 199 Initialize variables
200 200 '''
201 201
202 202 self.figures = []
203 203 self.axes = []
204 204 self.cb_axes = []
205 205 self.localtime = kwargs.pop('localtime', True)
206 206 self.show = kwargs.get('show', True)
207 207 self.save = kwargs.get('save', False)
208 208 self.save_period = kwargs.get('save_period', 0)
209 209 self.colormap = kwargs.get('colormap', self.colormap)
210 210 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
211 211 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
212 212 self.colormaps = kwargs.get('colormaps', None)
213 213 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
214 214 self.showprofile = kwargs.get('showprofile', False)
215 215 self.title = kwargs.get('wintitle', self.CODE.upper())
216 216 self.cb_label = kwargs.get('cb_label', None)
217 217 self.cb_labels = kwargs.get('cb_labels', None)
218 218 self.labels = kwargs.get('labels', None)
219 219 self.xaxis = kwargs.get('xaxis', 'frequency')
220 220 self.zmin = kwargs.get('zmin', None)
221 221 self.zmax = kwargs.get('zmax', None)
222 222 self.zlimits = kwargs.get('zlimits', None)
223 223 self.xmin = kwargs.get('xmin', None)
224 224 self.xmax = kwargs.get('xmax', None)
225 225 self.xrange = kwargs.get('xrange', 12)
226 226 self.xscale = kwargs.get('xscale', None)
227 227 self.ymin = kwargs.get('ymin', None)
228 228 self.ymax = kwargs.get('ymax', None)
229 229 self.yscale = kwargs.get('yscale', None)
230 230 self.xlabel = kwargs.get('xlabel', None)
231 231 self.attr_time = kwargs.get('attr_time', 'utctime')
232 232 self.attr_data = kwargs.get('attr_data', 'data_param')
233 233 self.decimation = kwargs.get('decimation', None)
234 234 self.oneFigure = kwargs.get('oneFigure', True)
235 235 self.width = kwargs.get('width', None)
236 236 self.height = kwargs.get('height', None)
237 237 self.colorbar = kwargs.get('colorbar', True)
238 238 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
239 239 self.channels = kwargs.get('channels', None)
240 240 self.titles = kwargs.get('titles', [])
241 241 self.polar = False
242 242 self.type = kwargs.get('type', 'iq')
243 243 self.grid = kwargs.get('grid', False)
244 244 self.pause = kwargs.get('pause', False)
245 245 self.save_code = kwargs.get('save_code', self.CODE)
246 246 self.throttle = kwargs.get('throttle', 0)
247 247 self.exp_code = kwargs.get('exp_code', None)
248 248 self.server = kwargs.get('server', False)
249 249 self.sender_period = kwargs.get('sender_period', 60)
250 250 self.tag = kwargs.get('tag', '')
251 251 self.height_index = kwargs.get('height_index', None)
252 252 self.__throttle_plot = apply_throttle(self.throttle)
253 253 code = self.attr_data if self.attr_data else self.CODE
254 254 self.data = PlotterData(self.CODE, self.exp_code, self.localtime)
255 255
256 256 if self.server:
257 257 if not self.server.startswith('tcp://'):
258 258 self.server = 'tcp://{}'.format(self.server)
259 259 log.success(
260 260 'Sending to server: {}'.format(self.server),
261 261 self.name
262 262 )
263 263
264 264 if isinstance(self.attr_data, str):
265 265 self.attr_data = [self.attr_data]
266 266
267 267 def __setup_plot(self):
268 268 '''
269 269 Common setup for all figures, here figures and axes are created
270 270 '''
271 271
272 272 self.setup()
273 273
274 274 self.time_label = 'LT' if self.localtime else 'UTC'
275 275
276 276 if self.width is None:
277 277 self.width = 8
278 278
279 279 self.figures = []
280 280 self.axes = []
281 281 self.cb_axes = []
282 282 self.pf_axes = []
283 283 self.cmaps = []
284 284
285 285 size = '15%' if self.ncols == 1 else '30%'
286 286 pad = '4%' if self.ncols == 1 else '8%'
287 287
288 288 if self.oneFigure:
289 289 if self.height is None:
290 290 self.height = 1.4 * self.nrows + 1
291 291 fig = plt.figure(figsize=(self.width, self.height),
292 292 edgecolor='k',
293 293 facecolor='w')
294 294 self.figures.append(fig)
295 295 for n in range(self.nplots):
296 296 ax = fig.add_subplot(self.nrows, self.ncols,
297 297 n + 1, polar=self.polar)
298 298 ax.tick_params(labelsize=8)
299 299 ax.firsttime = True
300 300 ax.index = 0
301 301 ax.press = None
302 302 self.axes.append(ax)
303 303 if self.showprofile:
304 304 cax = self.__add_axes(ax, size=size, pad=pad)
305 305 cax.tick_params(labelsize=8)
306 306 self.pf_axes.append(cax)
307 307 else:
308 308 if self.height is None:
309 309 self.height = 3
310 310 for n in range(self.nplots):
311 311 fig = plt.figure(figsize=(self.width, self.height),
312 312 edgecolor='k',
313 313 facecolor='w')
314 314 ax = fig.add_subplot(1, 1, 1, polar=self.polar)
315 315 ax.tick_params(labelsize=8)
316 316 ax.firsttime = True
317 317 ax.index = 0
318 318 ax.press = None
319 319 self.figures.append(fig)
320 320 self.axes.append(ax)
321 321 if self.showprofile:
322 322 cax = self.__add_axes(ax, size=size, pad=pad)
323 323 cax.tick_params(labelsize=8)
324 324 self.pf_axes.append(cax)
325 325
326 326 for n in range(self.nrows):
327 327 print(self.nrows)
328 328 if self.colormaps is not None:
329 329 cmap = plt.get_cmap(self.colormaps[n])
330 330 else:
331 331 cmap = plt.get_cmap(self.colormap)
332 332 cmap.set_bad(self.bgcolor, 1.)
333 333 self.cmaps.append(cmap)
334 334
335 335 def __add_axes(self, ax, size='30%', pad='8%'):
336 336 '''
337 337 Add new axes to the given figure
338 338 '''
339 339 divider = make_axes_locatable(ax)
340 340 nax = divider.new_horizontal(size=size, pad=pad)
341 341 ax.figure.add_axes(nax)
342 342 return nax
343 343
344 344 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
345 345 '''
346 346 Create a masked array for missing data
347 347 '''
348 348 if x_buffer.shape[0] < 2:
349 349 return x_buffer, y_buffer, z_buffer
350 350
351 351 deltas = x_buffer[1:] - x_buffer[0:-1]
352 352 x_median = numpy.median(deltas)
353 353
354 354 index = numpy.where(deltas > 5 * x_median)
355 355
356 356 if len(index[0]) != 0:
357 357 z_buffer[::, index[0], ::] = self.__missing
358 358 z_buffer = numpy.ma.masked_inside(z_buffer,
359 359 0.99 * self.__missing,
360 360 1.01 * self.__missing)
361 361
362 362 return x_buffer, y_buffer, z_buffer
363 363
364 364 def decimate(self):
365 365
366 366 # dx = int(len(self.x)/self.__MAXNUMX) + 1
367 367 dy = int(len(self.y) / self.decimation) + 1
368 368
369 369 # x = self.x[::dx]
370 370 x = self.x
371 371 y = self.y[::dy]
372 372 z = self.z[::, ::, ::dy]
373 373
374 374 return x, y, z
375 375
376 376 def format(self):
377 377 '''
378 378 Set min and max values, labels, ticks and titles
379 379 '''
380 380
381 381 for n, ax in enumerate(self.axes):
382 382 if ax.firsttime:
383 383 if self.xaxis != 'time':
384 384 xmin = self.xmin
385 385 xmax = self.xmax
386 386 else:
387 387 xmin = self.tmin
388 xmax = self.tmin + self.xrange*60*60
388 xmax = self.tmin + self.xrange * 60 * 60
389 389 ax.xaxis.set_major_formatter(FuncFormatter(self.__fmtTime))
390 390 ax.xaxis.set_major_locator(LinearLocator(9))
391 391 ymin = self.ymin if self.ymin is not None else numpy.nanmin(self.y[numpy.isfinite(self.y)])
392 392 ymax = self.ymax if self.ymax is not None else numpy.nanmax(self.y[numpy.isfinite(self.y)])
393 393 ax.set_facecolor(self.bgcolor)
394 394 if self.xscale:
395 395 ax.xaxis.set_major_formatter(FuncFormatter(
396 lambda x, pos: '{0:g}'.format(x*self.xscale)))
396 lambda x, pos: '{0:g}'.format(x * self.xscale)))
397 397 if self.yscale:
398 398 ax.yaxis.set_major_formatter(FuncFormatter(
399 lambda x, pos: '{0:g}'.format(x*self.yscale)))
399 lambda x, pos: '{0:g}'.format(x * self.yscale)))
400 400 if self.xlabel is not None:
401 401 ax.set_xlabel(self.xlabel)
402 402 if self.ylabel is not None:
403 403 ax.set_ylabel(self.ylabel)
404 404 if self.showprofile:
405 405 self.pf_axes[n].set_ylim(ymin, ymax)
406 406 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
407 407 self.pf_axes[n].set_xlabel('dB')
408 408 self.pf_axes[n].grid(b=True, axis='x')
409 409 [tick.set_visible(False)
410 410 for tick in self.pf_axes[n].get_yticklabels()]
411 411 if self.colorbar:
412 412 ax.cbar = plt.colorbar(
413 413 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
414 414 ax.cbar.ax.tick_params(labelsize=8)
415 415 ax.cbar.ax.press = None
416 416 if self.cb_label:
417 417 ax.cbar.set_label(self.cb_label, size=8)
418 418 elif self.cb_labels:
419 419 ax.cbar.set_label(self.cb_labels[n], size=8)
420 420 else:
421 421 ax.cbar = None
422 422 ax.set_xlim(xmin, xmax)
423 423 ax.set_ylim(ymin, ymax)
424 424 ax.firsttime = False
425 425 if self.grid:
426 426 ax.grid(True)
427 427 if not self.polar:
428 428 ax.set_title('{} {} {}'.format(
429 429 self.titles[n],
430 430 self.getDateTime(self.data.max_time).strftime(
431 431 '%Y-%m-%d %H:%M:%S'),
432 432 self.time_label),
433 433 size=8)
434 434 else:
435 435 ax.set_title('{}'.format(self.titles[n]), size=8)
436 436 ax.set_ylim(0, 90)
437 437 ax.set_yticks(numpy.arange(0, 90, 20))
438 438 ax.yaxis.labelpad = 40
439 439
440 440 if self.firsttime:
441 441 for n, fig in enumerate(self.figures):
442 442 fig.subplots_adjust(**self.plots_adjust)
443 443 self.firsttime = False
444 444
445 445 def clear_figures(self):
446 446 '''
447 447 Reset axes for redraw plots
448 448 '''
449 449
450 for ax in self.axes+self.pf_axes+self.cb_axes:
450 for ax in self.axes + self.pf_axes + self.cb_axes:
451 451 ax.clear()
452 452 ax.firsttime = True
453 453 if hasattr(ax, 'cbar') and ax.cbar:
454 454 ax.cbar.remove()
455 455
456 456 def __plot(self):
457 457 '''
458 458 Main function to plot, format and save figures
459 459 '''
460 460
461 461 self.plot()
462 462 self.format()
463 463
464 464 for n, fig in enumerate(self.figures):
465 465 if self.nrows == 0 or self.nplots == 0:
466 466 log.warning('No data', self.name)
467 467 fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center')
468 468 fig.canvas.manager.set_window_title(self.CODE)
469 469 continue
470 470
471 471 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
472 472 self.getDateTime(self.data.max_time).strftime('%Y/%m/%d')))
473 473 fig.canvas.draw()
474 474 if self.show:
475 475 fig.show()
476 476 figpause(0.01)
477 477
478 478 if self.save:
479 479 self.save_figure(n)
480 480
481 481 if self.server:
482 482 self.send_to_server()
483 483
484 484 def __update(self, dataOut, timestamp):
485 485 '''
486 486 '''
487 487
488 488 metadata = {
489 489 'yrange': dataOut.heightList,
490 490 'interval': dataOut.timeInterval,
491 491 'channels': dataOut.channelList
492 492 }
493 493
494 494 data, meta = self.update(dataOut)
495 495 metadata.update(meta)
496 496 self.data.update(data, timestamp, metadata)
497 497
498 498 def save_figure(self, n):
499 499 '''
500 500 '''
501 501
502 502 if (self.data.max_time - self.save_time) <= self.save_period:
503 503 return
504 504
505 505 self.save_time = self.data.max_time
506 506
507 507 fig = self.figures[n]
508 508
509 509 if self.throttle == 0:
510 510 figname = os.path.join(
511 511 self.save,
512 512 self.save_code,
513 '{}_{}.png'.format(
513 '{}_{}.png'.format(
514 514 self.save_code,
515 515 self.getDateTime(self.data.max_time).strftime(
516 516 '%Y%m%d_%H%M%S'
517 517 ),
518 518 )
519 519 )
520 520 log.log('Saving figure: {}'.format(figname), self.name)
521 521 if not os.path.isdir(os.path.dirname(figname)):
522 522 os.makedirs(os.path.dirname(figname))
523 523 fig.savefig(figname)
524 524
525 525 figname = os.path.join(
526 526 self.save,
527 527 '{}_{}.png'.format(
528 528 self.save_code,
529 529 self.getDateTime(self.data.min_time).strftime(
530 530 '%Y%m%d'
531 531 ),
532 532 )
533 533 )
534 534 fig.savefig(figname)
535 535
536 536 def send_to_server(self):
537 537 '''
538 538 '''
539 539
540 540 if self.exp_code == None:
541 541 log.warning('Missing `exp_code` skipping sending to server...')
542 542
543 543 last_time = self.data.max_time
544 544 interval = last_time - self.sender_time
545 545 if interval < self.sender_period:
546 546 return
547 547
548 548 self.sender_time = last_time
549 549
550 550 attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
551 551 for attr in attrs:
552 552 value = getattr(self, attr)
553 553 if value:
554 554 if isinstance(value, (numpy.float32, numpy.float64)):
555 555 value = round(float(value), 2)
556 556 self.data.meta[attr] = value
557 557 if self.colormap == 'jet':
558 558 self.data.meta['colormap'] = 'Jet'
559 559 elif 'RdBu' in self.colormap:
560 560 self.data.meta['colormap'] = 'RdBu'
561 561 else:
562 562 self.data.meta['colormap'] = 'Viridis'
563 563 self.data.meta['interval'] = int(interval)
564 564
565 565 self.sender_queue.append(last_time)
566 566
567 567 while True:
568 568 try:
569 569 tm = self.sender_queue.popleft()
570 570 except IndexError:
571 571 break
572 572 msg = self.data.jsonify(tm, self.save_code, self.plot_type)
573 573 self.socket.send_string(msg)
574 574 socks = dict(self.poll.poll(2000))
575 575 if socks.get(self.socket) == zmq.POLLIN:
576 576 reply = self.socket.recv_string()
577 577 if reply == 'ok':
578 578 log.log("Response from server ok", self.name)
579 579 time.sleep(0.1)
580 580 continue
581 581 else:
582 582 log.warning(
583 583 "Malformed reply from server: {}".format(reply), self.name)
584 584 else:
585 585 log.warning(
586 586 "No response from server, retrying...", self.name)
587 587 self.sender_queue.appendleft(tm)
588 588 self.socket.setsockopt(zmq.LINGER, 0)
589 589 self.socket.close()
590 590 self.poll.unregister(self.socket)
591 591 self.socket = self.context.socket(zmq.REQ)
592 592 self.socket.connect(self.server)
593 593 self.poll.register(self.socket, zmq.POLLIN)
594 594 break
595 595
596 596 def setup(self):
597 597 '''
598 598 This method should be implemented in the child class, the following
599 599 attributes should be set:
600 600
601 601 self.nrows: number of rows
602 602 self.ncols: number of cols
603 603 self.nplots: number of plots (channels or pairs)
604 604 self.ylabel: label for Y axes
605 605 self.titles: list of axes title
606 606
607 607 '''
608 608 raise NotImplementedError
609 609
610 610 def plot(self):
611 611 '''
612 612 Must be defined in the child class, the actual plotting method
613 613 '''
614 614 raise NotImplementedError
615 615
616 616 def update(self, dataOut):
617 617 '''
618 618 Must be defined in the child class, update self.data with new data
619 619 '''
620 620
621 621 data = {
622 622 self.CODE: getattr(dataOut, 'data_{}'.format(self.CODE))
623 623 }
624 624 meta = {}
625 625
626 626 return data, meta
627 627
628 628 def run(self, dataOut, **kwargs):
629 629 '''
630 630 Main plotting routine
631 631 '''
632 632
633 633 if self.isConfig is False:
634 634 self.__setup(**kwargs)
635 635
636 636 if self.localtime:
637 637 self.getDateTime = datetime.datetime.fromtimestamp
638 638 else:
639 639 self.getDateTime = datetime.datetime.utcfromtimestamp
640 640
641 641 self.data.setup()
642 642 self.isConfig = True
643 643 if self.server:
644 644 self.context = zmq.Context()
645 645 self.socket = self.context.socket(zmq.REQ)
646 646 self.socket.connect(self.server)
647 647 self.poll = zmq.Poller()
648 648 self.poll.register(self.socket, zmq.POLLIN)
649 649
650 650 tm = getattr(dataOut, self.attr_time)
651 651
652 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange*60*60:
652 if self.data and 'time' in self.xaxis and (tm - self.tmin) >= self.xrange * 60 * 60:
653 653 self.save_time = tm
654 654 self.__plot()
655 self.tmin += self.xrange*60*60
655 self.tmin += self.xrange * 60 * 60
656 656 self.data.setup()
657 657 self.clear_figures()
658 658
659 659 self.__update(dataOut, tm)
660 660
661 661 if self.isPlotConfig is False:
662 662 self.__setup_plot()
663 663 self.isPlotConfig = True
664 664 if self.xaxis == 'time':
665 665 dt = self.getDateTime(tm)
666 666 if self.xmin is None:
667 667 self.tmin = tm
668 668 self.xmin = dt.hour
669 minutes = (self.xmin-int(self.xmin)) * 60
669 minutes = (self.xmin - int(self.xmin)) * 60
670 670 seconds = (minutes - int(minutes)) * 60
671 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
671 self.tmin = (dt.replace(hour=int(self.xmin), minute=int(minutes), second=int(seconds)) -
672 672 datetime.datetime(1970, 1, 1)).total_seconds()
673 673 if self.localtime:
674 674 self.tmin += time.timezone
675 675
676 676 if self.xmin is not None and self.xmax is not None:
677 677 self.xrange = self.xmax - self.xmin
678 678
679 679 if self.throttle == 0:
680 680 self.__plot()
681 681 else:
682 self.__throttle_plot(self.__plot)#, coerce=coerce)
682 self.__throttle_plot(self.__plot) # , coerce=coerce)
683 683
684 684 def close(self):
685 685
686 686 if self.data and not self.data.flagNoData:
687 687 self.save_time = self.data.max_time
688 688 self.__plot()
689 689 if self.data and not self.data.flagNoData and self.pause:
690 690 figpause(10)
691 691
Show file before | Show file after | Hide comments
@@ -1,187 +1,187
1 1 import os
2 2 import datetime
3 3 import numpy
4 4 import copy
5 5 from schainpy.model.graphics.jroplot_base import Plot
6 6
7 7
8 8 class CorrelationPlot(Plot):
9 9 isConfig = None
10 10 __nsubplots = None
11 11
12 12 WIDTHPROF = None
13 13 HEIGHTPROF = None
14 14 PREFIX = 'corr'
15 15
16 16 def __init__(self, **kwargs):
17 17 Figure.__init__(self, **kwargs)
18 18 self.isConfig = False
19 19 self.__nsubplots = 1
20 20
21 21 self.WIDTH = 280
22 22 self.HEIGHT = 250
23 23 self.WIDTHPROF = 120
24 24 self.HEIGHTPROF = 0
25 25 self.counter_imagwr = 0
26 26
27 27 self.PLOT_CODE = 1
28 28 self.FTP_WEI = None
29 29 self.EXP_CODE = None
30 30 self.SUB_EXP_CODE = None
31 31 self.PLOT_POS = None
32 32
33 33 def getSubplots(self):
34 34
35 ncol = int(numpy.sqrt(self.nplots)+0.9)
36 nrow = int(self.nplots*1./ncol + 0.9)
35 ncol = int(numpy.sqrt(self.nplots) + 0.9)
36 nrow = int(self.nplots * 1. / ncol + 0.9)
37 37
38 38 return nrow, ncol
39 39
40 40 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
41 41
42 42 showprofile = False
43 43 self.__showprofile = showprofile
44 44 self.nplots = nplots
45 45
46 46 ncolspan = 1
47 47 colspan = 1
48 48 if showprofile:
49 49 ncolspan = 3
50 50 colspan = 2
51 51 self.__nsubplots = 2
52 52
53 self.createFigure(id = id,
54 wintitle = wintitle,
55 widthplot = self.WIDTH + self.WIDTHPROF,
56 heightplot = self.HEIGHT + self.HEIGHTPROF,
53 self.createFigure(id=id,
54 wintitle=wintitle,
55 widthplot=self.WIDTH + self.WIDTHPROF,
56 heightplot=self.HEIGHT + self.HEIGHTPROF,
57 57 show=show)
58 58
59 59 nrow, ncol = self.getSubplots()
60 60
61 61 counter = 0
62 62 for y in range(nrow):
63 63 for x in range(ncol):
64 64
65 65 if counter >= self.nplots:
66 66 break
67 67
68 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
68 self.addAxes(nrow, ncol * ncolspan, y, x * ncolspan, colspan, 1)
69 69
70 70 if showprofile:
71 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
71 self.addAxes(nrow, ncol * ncolspan, y, x * ncolspan + colspan, 1, 1)
72 72
73 73 counter += 1
74 74
75 75 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
76 76 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
77 77 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
78 78 server=None, folder=None, username=None, password=None,
79 79 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
80 80
81 81 """
82 82
83 83 Input:
84 84 dataOut :
85 85 id :
86 86 wintitle :
87 87 channelList :
88 88 showProfile :
89 89 xmin : None,
90 90 xmax : None,
91 91 ymin : None,
92 92 ymax : None,
93 93 zmin : None,
94 94 zmax : None
95 95 """
96 96
97 97 if dataOut.flagNoData:
98 98 return None
99 99
100 100 if realtime:
101 if not(isRealtime(utcdatatime = dataOut.utctime)):
101 if not(isRealtime(utcdatatime=dataOut.utctime)):
102 102 print('Skipping this plot function')
103 103 return
104 104
105 105 if channelList == None:
106 106 channelIndexList = dataOut.channelIndexList
107 107 else:
108 108 channelIndexList = []
109 109 for channel in channelList:
110 110 if channel not in dataOut.channelList:
111 111 raise ValueError("Channel %d is not in dataOut.channelList")
112 112 channelIndexList.append(dataOut.channelList.index(channel))
113 113
114 114 factor = dataOut.normFactor
115 115 lenfactor = factor.shape[1]
116 116 x = dataOut.getLagTRange(1)
117 117 y = dataOut.heightList
118 118
119 z = copy.copy(dataOut.data_corr[:,:,0,:])
119 z = copy.copy(dataOut.data_corr[:, :, 0, :])
120 120 for i in range(dataOut.data_corr.shape[0]):
121 z[i,:,:] = z[i,:,:]/factor[i,:]
121 z[i, :, :] = z[i, :, :] / factor[i, :]
122 122 zdB = numpy.abs(z)
123 123
124 124 avg = numpy.average(z, axis=1)
125 125 # avg = numpy.nanmean(z, axis=1)
126 126 # noise = dataOut.noise/factor
127 127
128 #thisDatetime = dataOut.datatime
128 # thisDatetime = dataOut.datatime
129 129 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
130 130 title = wintitle + " Correlation"
131 131 xlabel = "Lag T (s)"
132 132 ylabel = "Range (Km)"
133 133
134 134 if not self.isConfig:
135 135
136 136 nplots = dataOut.data_corr.shape[0]
137 137
138 138 self.setup(id=id,
139 139 nplots=nplots,
140 140 wintitle=wintitle,
141 141 showprofile=showprofile,
142 142 show=show)
143 143
144 144 if xmin == None: xmin = numpy.nanmin(x)
145 145 if xmax == None: xmax = numpy.nanmax(x)
146 146 if ymin == None: ymin = numpy.nanmin(y)
147 147 if ymax == None: ymax = numpy.nanmax(y)
148 148 if zmin == None: zmin = 0
149 149 if zmax == None: zmax = 1
150 150
151 151 self.FTP_WEI = ftp_wei
152 152 self.EXP_CODE = exp_code
153 153 self.SUB_EXP_CODE = sub_exp_code
154 154 self.PLOT_POS = plot_pos
155 155
156 156 self.isConfig = True
157 157
158 158 self.setWinTitle(title)
159 159
160 160 for i in range(self.nplots):
161 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
162 title = "Channel %d and %d: : %s" %(dataOut.pairsList[i][0],dataOut.pairsList[i][1] , str_datetime)
163 axes = self.axesList[i*self.__nsubplots]
164 axes.pcolor(x, y, zdB[i,:,:],
161 str_datetime = '%s %s' % (thisDatetime.strftime("%Y/%m/%d"), thisDatetime.strftime("%H:%M:%S"))
162 title = "Channel %d and %d: : %s" % (dataOut.pairsList[i][0], dataOut.pairsList[i][1] , str_datetime)
163 axes = self.axesList[i * self.__nsubplots]
164 axes.pcolor(x, y, zdB[i, :, :],
165 165 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
166 166 xlabel=xlabel, ylabel=ylabel, title=title,
167 167 ticksize=9, cblabel='')
168 168
169 169 # if self.__showprofile:
170 170 # axes = self.axesList[i*self.__nsubplots +1]
171 171 # axes.pline(avgdB[i], y,
172 172 # xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
173 173 # xlabel='dB', ylabel='', title='',
174 174 # ytick_visible=False,
175 175 # grid='x')
176 176 #
177 177 # noiseline = numpy.repeat(noisedB[i], len(y))
178 178 # axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
179 179
180 180 self.draw()
181 181
182 182 self.save(figpath=figpath,
183 183 figfile=figfile,
184 184 save=save,
185 185 ftp=ftp,
186 186 wr_period=wr_period,
187 thisDatetime=thisDatetime) No newline at end of file
187 thisDatetime=thisDatetime)
Show file before | Show file after | Hide comments
@@ -1,101 +1,101
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Classes to plo Specra Heis data
6 6
7 7 """
8 8
9 9 import numpy
10 10
11 11 from schainpy.model.graphics.jroplot_base import Plot, plt
12 12
13 13
14 14 class SpectraHeisPlot(Plot):
15 15
16 16 CODE = 'spc_heis'
17 17
18 18 def setup(self):
19 19
20 20 self.nplots = len(self.data.channels)
21 21 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
22 22 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
23 23 self.height = 2.6 * self.nrows
24 24 self.width = 3.5 * self.ncols
25 25 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.95, 'bottom': 0.08})
26 26 self.ylabel = 'Intensity [dB]'
27 27 self.xlabel = 'Frequency [KHz]'
28 28 self.colorbar = False
29 29
30 30 def update(self, dataOut):
31 31
32 32 data = {}
33 33 meta = {}
34 spc = 10*numpy.log10(dataOut.data_spc / dataOut.normFactor)
34 spc = 10 * numpy.log10(dataOut.data_spc / dataOut.normFactor)
35 35 data['spc_heis'] = spc
36 36
37 37 return data, meta
38 38
39 39 def plot(self):
40 40
41 41 c = 3E8
42 42 deltaHeight = self.data.yrange[1] - self.data.yrange[0]
43 x = numpy.arange(-1*len(self.data.yrange)/2., len(self.data.yrange)/2.)*(c/(2*deltaHeight*len(self.data.yrange)*1000))
43 x = numpy.arange(-1 * len(self.data.yrange) / 2., len(self.data.yrange) / 2.) * (c / (2 * deltaHeight * len(self.data.yrange) * 1000))
44 44 self.y = self.data[-1]['spc_heis']
45 45 self.titles = []
46 46
47 47 for n, ax in enumerate(self.axes):
48 ychannel = self.y[n,:]
48 ychannel = self.y[n, :]
49 49 if ax.firsttime:
50 50 self.xmin = min(x) if self.xmin is None else self.xmin
51 51 self.xmax = max(x) if self.xmax is None else self.xmax
52 52 ax.plt = ax.plot(x, ychannel, lw=1, color='b')[0]
53 53 else:
54 54 ax.plt.set_data(x, ychannel)
55 55
56 56 self.titles.append("Channel {}: {:4.2f}dB".format(n, numpy.max(ychannel)))
57 57
58 58
59 59 class RTIHeisPlot(Plot):
60 60
61 61 CODE = 'rti_heis'
62 62
63 63 def setup(self):
64 64
65 65 self.xaxis = 'time'
66 66 self.ncols = 1
67 67 self.nrows = 1
68 68 self.nplots = 1
69 69 self.ylabel = 'Intensity [dB]'
70 70 self.xlabel = 'Time'
71 71 self.titles = ['RTI']
72 72 self.colorbar = False
73 73 self.height = 4
74 74 self.plots_adjust.update({'right': 0.85 })
75 75
76 76 def update(self, dataOut):
77 77
78 78 data = {}
79 79 meta = {}
80 80 spc = dataOut.data_spc / dataOut.normFactor
81 spc = 10*numpy.log10(numpy.average(spc, axis=1))
81 spc = 10 * numpy.log10(numpy.average(spc, axis=1))
82 82 data['rti_heis'] = spc
83 83
84 84 return data, meta
85 85
86 86 def plot(self):
87 87
88 88 x = self.data.times
89 89 Y = self.data['rti_heis']
90 90
91 91 if self.axes[0].firsttime:
92 92 self.ymin = numpy.nanmin(Y) - 5 if self.ymin == None else self.ymin
93 93 self.ymax = numpy.nanmax(Y) + 5 if self.ymax == None else self.ymax
94 94 for ch in self.data.channels:
95 95 y = Y[ch]
96 96 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
97 97 plt.legend(bbox_to_anchor=(1.18, 1.0))
98 98 else:
99 99 for ch in self.data.channels:
100 100 y = Y[ch]
101 101 self.axes[0].lines[ch].set_data(x, y)
Show file before | Show file after | Hide comments
@@ -1,357 +1,357
1 1 import os
2 2 import datetime
3 3 import numpy
4 4
5 5 from schainpy.model.graphics.jroplot_base import Plot, plt
6 6 from schainpy.model.graphics.jroplot_spectra import SpectraPlot, RTIPlot, CoherencePlot
7 7 from schainpy.utils import log
8 8
9 9 EARTH_RADIUS = 6.3710e3
10 10
11 11
12 12 def ll2xy(lat1, lon1, lat2, lon2):
13 13
14 14 p = 0.017453292519943295
15 a = 0.5 - numpy.cos((lat2 - lat1) * p)/2 + numpy.cos(lat1 * p) * \
15 a = 0.5 - numpy.cos((lat2 - lat1) * p) / 2 + numpy.cos(lat1 * p) * \
16 16 numpy.cos(lat2 * p) * (1 - numpy.cos((lon2 - lon1) * p)) / 2
17 17 r = 12742 * numpy.arcsin(numpy.sqrt(a))
18 theta = numpy.arctan2(numpy.sin((lon2-lon1)*p)*numpy.cos(lat2*p), numpy.cos(lat1*p)
19 * numpy.sin(lat2*p)-numpy.sin(lat1*p)*numpy.cos(lat2*p)*numpy.cos((lon2-lon1)*p))
20 theta = -theta + numpy.pi/2
21 return r*numpy.cos(theta), r*numpy.sin(theta)
18 theta = numpy.arctan2(numpy.sin((lon2 - lon1) * p) * numpy.cos(lat2 * p), numpy.cos(lat1 * p)
19 * numpy.sin(lat2 * p) - numpy.sin(lat1 * p) * numpy.cos(lat2 * p) * numpy.cos((lon2 - lon1) * p))
20 theta = -theta + numpy.pi / 2
21 return r * numpy.cos(theta), r * numpy.sin(theta)
22 22
23 23
24 24 def km2deg(km):
25 25 '''
26 26 Convert distance in km to degrees
27 27 '''
28 28
29 return numpy.rad2deg(km/EARTH_RADIUS)
29 return numpy.rad2deg(km / EARTH_RADIUS)
30 30
31 31
32 32
33 33 class SpectralMomentsPlot(SpectraPlot):
34 34 '''
35 35 Plot for Spectral Moments
36 36 '''
37 37 CODE = 'spc_moments'
38 38 colormap = 'jet'
39 39 plot_type = 'pcolor'
40 40
41 41
42 42 class SnrPlot(RTIPlot):
43 43 '''
44 44 Plot for SNR Data
45 45 '''
46 46
47 47 CODE = 'snr'
48 48 colormap = 'jet'
49 49
50 50 def update(self, dataOut):
51 51
52 52 data = {
53 'snr': 10*numpy.log10(dataOut.data_snr)
53 'snr': 10 * numpy.log10(dataOut.data_snr)
54 54 }
55 55
56 56 return data, {}
57 57
58 58 class DopplerPlot(RTIPlot):
59 59 '''
60 60 Plot for DOPPLER Data (1st moment)
61 61 '''
62 62
63 63 CODE = 'dop'
64 64 colormap = 'jet'
65 65
66 66 def update(self, dataOut):
67 67
68 68 data = {
69 'dop': 10*numpy.log10(dataOut.data_dop)
69 'dop': 10 * numpy.log10(dataOut.data_dop)
70 70 }
71 71
72 72 return data, {}
73 73
74 74 class PowerPlot(RTIPlot):
75 75 '''
76 76 Plot for Power Data (0 moment)
77 77 '''
78 78
79 79 CODE = 'pow'
80 80 colormap = 'jet'
81 81
82 82 def update(self, dataOut):
83 83
84 84 data = {
85 'pow': 10*numpy.log10(dataOut.data_pow)
85 'pow': 10 * numpy.log10(dataOut.data_pow)
86 86 }
87 87
88 88 return data, {}
89 89
90 90 class SpectralWidthPlot(RTIPlot):
91 91 '''
92 92 Plot for Spectral Width Data (2nd moment)
93 93 '''
94 94
95 95 CODE = 'width'
96 96 colormap = 'jet'
97 97
98 98 def update(self, dataOut):
99 99
100 100 data = {
101 101 'width': dataOut.data_width
102 102 }
103 103
104 104 return data, {}
105 105
106 106 class SkyMapPlot(Plot):
107 107 '''
108 108 Plot for meteors detection data
109 109 '''
110 110
111 111 CODE = 'param'
112 112
113 113 def setup(self):
114 114
115 115 self.ncols = 1
116 116 self.nrows = 1
117 117 self.width = 7.2
118 118 self.height = 7.2
119 119 self.nplots = 1
120 120 self.xlabel = 'Zonal Zenith Angle (deg)'
121 121 self.ylabel = 'Meridional Zenith Angle (deg)'
122 122 self.polar = True
123 123 self.ymin = -180
124 124 self.ymax = 180
125 125 self.colorbar = False
126 126
127 127 def plot(self):
128 128
129 129 arrayParameters = numpy.concatenate(self.data['param'])
130 130 error = arrayParameters[:, -1]
131 131 indValid = numpy.where(error == 0)[0]
132 132 finalMeteor = arrayParameters[indValid, :]
133 133 finalAzimuth = finalMeteor[:, 3]
134 134 finalZenith = finalMeteor[:, 4]
135 135
136 136 x = finalAzimuth * numpy.pi / 180
137 137 y = finalZenith
138 138
139 139 ax = self.axes[0]
140 140
141 141 if ax.firsttime:
142 142 ax.plot = ax.plot(x, y, 'bo', markersize=5)[0]
143 143 else:
144 144 ax.plot.set_data(x, y)
145 145
146 146 dt1 = self.getDateTime(self.data.min_time).strftime('%y/%m/%d %H:%M:%S')
147 147 dt2 = self.getDateTime(self.data.max_time).strftime('%y/%m/%d %H:%M:%S')
148 148 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
149 149 dt2,
150 150 len(x))
151 151 self.titles[0] = title
152 152
153 153
154 154 class GenericRTIPlot(Plot):
155 155 '''
156 156 Plot for data_xxxx object
157 157 '''
158 158
159 159 CODE = 'param'
160 160 colormap = 'viridis'
161 161 plot_type = 'pcolorbuffer'
162 162
163 163 def setup(self):
164 164 self.xaxis = 'time'
165 165 self.ncols = 1
166 166 self.nrows = self.data.shape('param')[0]
167 167 self.nplots = self.nrows
168 168 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95, 'top': 0.95})
169 169
170 170 if not self.xlabel:
171 171 self.xlabel = 'Time'
172 172
173 173 self.ylabel = 'Height [km]'
174 174 if not self.titles:
175 175 self.titles = ['Param {}'.format(x) for x in range(self.nrows)]
176 176
177 177 def update(self, dataOut):
178 178
179 179 data = {
180 180 'param' : numpy.concatenate([getattr(dataOut, attr) for attr in self.attr_data], axis=0)
181 181 }
182 182
183 183 meta = {}
184 184
185 185 return data, meta
186 186
187 187 def plot(self):
188 188 # self.data.normalize_heights()
189 189 self.x = self.data.times
190 190 self.y = self.data.yrange
191 191 self.z = self.data['param']
192 192
193 193 self.z = numpy.ma.masked_invalid(self.z)
194 194
195 195 if self.decimation is None:
196 196 x, y, z = self.fill_gaps(self.x, self.y, self.z)
197 197 else:
198 198 x, y, z = self.fill_gaps(*self.decimate())
199 199
200 200 for n, ax in enumerate(self.axes):
201 201
202 202 self.zmax = self.zmax if self.zmax is not None else numpy.max(
203 203 self.z[n])
204 204 self.zmin = self.zmin if self.zmin is not None else numpy.min(
205 205 self.z[n])
206 206
207 207 if ax.firsttime:
208 208 if self.zlimits is not None:
209 209 self.zmin, self.zmax = self.zlimits[n]
210 210
211 211 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
212 212 vmin=self.zmin,
213 213 vmax=self.zmax,
214 214 cmap=self.cmaps[n]
215 215 )
216 216 else:
217 217 if self.zlimits is not None:
218 218 self.zmin, self.zmax = self.zlimits[n]
219 219 ax.collections.remove(ax.collections[0])
220 220 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
221 221 vmin=self.zmin,
222 222 vmax=self.zmax,
223 223 cmap=self.cmaps[n]
224 224 )
225 225
226 226
227 227 class PolarMapPlot(Plot):
228 228 '''
229 229 Plot for weather radar
230 230 '''
231 231
232 232 CODE = 'param'
233 233 colormap = 'seismic'
234 234
235 235 def setup(self):
236 236 self.ncols = 1
237 237 self.nrows = 1
238 238 self.width = 9
239 239 self.height = 8
240 240 self.mode = self.data.meta['mode']
241 241 if self.channels is not None:
242 242 self.nplots = len(self.channels)
243 243 self.nrows = len(self.channels)
244 244 else:
245 245 self.nplots = self.data.shape(self.CODE)[0]
246 246 self.nrows = self.nplots
247 247 self.channels = list(range(self.nplots))
248 248 if self.mode == 'E':
249 249 self.xlabel = 'Longitude'
250 250 self.ylabel = 'Latitude'
251 251 else:
252 252 self.xlabel = 'Range (km)'
253 253 self.ylabel = 'Height (km)'
254 254 self.bgcolor = 'white'
255 255 self.cb_labels = self.data.meta['units']
256 256 self.lat = self.data.meta['latitude']
257 257 self.lon = self.data.meta['longitude']
258 258 self.xmin, self.xmax = float(
259 259 km2deg(self.xmin) + self.lon), float(km2deg(self.xmax) + self.lon)
260 260 self.ymin, self.ymax = float(
261 261 km2deg(self.ymin) + self.lat), float(km2deg(self.ymax) + self.lat)
262 262 # self.polar = True
263 263
264 264 def plot(self):
265 265
266 266 for n, ax in enumerate(self.axes):
267 267 data = self.data['param'][self.channels[n]]
268 268
269 269 zeniths = numpy.linspace(
270 270 0, self.data.meta['max_range'], data.shape[1])
271 271 if self.mode == 'E':
272 azimuths = -numpy.radians(self.data.yrange)+numpy.pi/2
272 azimuths = -numpy.radians(self.data.yrange) + numpy.pi / 2
273 273 r, theta = numpy.meshgrid(zeniths, azimuths)
274 x, y = r*numpy.cos(theta)*numpy.cos(numpy.radians(self.data.meta['elevation'])), r*numpy.sin(
275 theta)*numpy.cos(numpy.radians(self.data.meta['elevation']))
274 x, y = r * numpy.cos(theta) * numpy.cos(numpy.radians(self.data.meta['elevation'])), r * numpy.sin(
275 theta) * numpy.cos(numpy.radians(self.data.meta['elevation']))
276 276 x = km2deg(x) + self.lon
277 277 y = km2deg(y) + self.lat
278 278 else:
279 279 azimuths = numpy.radians(self.data.yrange)
280 280 r, theta = numpy.meshgrid(zeniths, azimuths)
281 x, y = r*numpy.cos(theta), r*numpy.sin(theta)
281 x, y = r * numpy.cos(theta), r * numpy.sin(theta)
282 282 self.y = zeniths
283 283
284 284 if ax.firsttime:
285 285 if self.zlimits is not None:
286 286 self.zmin, self.zmax = self.zlimits[n]
287 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
287 ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
288 288 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
289 289 vmin=self.zmin,
290 290 vmax=self.zmax,
291 291 cmap=self.cmaps[n])
292 292 else:
293 293 if self.zlimits is not None:
294 294 self.zmin, self.zmax = self.zlimits[n]
295 295 ax.collections.remove(ax.collections[0])
296 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
296 ax.plt = ax.pcolormesh(# r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
297 297 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
298 298 vmin=self.zmin,
299 299 vmax=self.zmax,
300 300 cmap=self.cmaps[n])
301 301
302 302 if self.mode == 'A':
303 303 continue
304 304
305 305 # plot district names
306 306 f = open('/data/workspace/schain_scripts/distrito.csv')
307 307 for line in f:
308 308 label, lon, lat = [s.strip() for s in line.split(',') if s]
309 309 lat = float(lat)
310 310 lon = float(lon)
311 311 # ax.plot(lon, lat, '.b', ms=2)
312 312 ax.text(lon, lat, label.decode('utf8'), ha='center',
313 313 va='bottom', size='8', color='black')
314 314
315 315 # plot limites
316 316 limites = []
317 317 tmp = []
318 318 for line in open('/data/workspace/schain_scripts/lima.csv'):
319 319 if '#' in line:
320 320 if tmp:
321 321 limites.append(tmp)
322 322 tmp = []
323 323 continue
324 324 values = line.strip().split(',')
325 325 tmp.append((float(values[0]), float(values[1])))
326 326 for points in limites:
327 327 ax.add_patch(
328 328 Polygon(points, ec='k', fc='none', ls='--', lw=0.5))
329 329
330 330 # plot Cuencas
331 331 for cuenca in ('rimac', 'lurin', 'mala', 'chillon', 'chilca', 'chancay-huaral'):
332 332 f = open('/data/workspace/schain_scripts/{}.csv'.format(cuenca))
333 333 values = [line.strip().split(',') for line in f]
334 334 points = [(float(s[0]), float(s[1])) for s in values]
335 335 ax.add_patch(Polygon(points, ec='b', fc='none'))
336 336
337 337 # plot grid
338 338 for r in (15, 30, 45, 60):
339 339 ax.add_artist(plt.Circle((self.lon, self.lat),
340 340 km2deg(r), color='0.6', fill=False, lw=0.2))
341 341 ax.text(
342 self.lon + (km2deg(r))*numpy.cos(60*numpy.pi/180),
343 self.lat + (km2deg(r))*numpy.sin(60*numpy.pi/180),
342 self.lon + (km2deg(r)) * numpy.cos(60 * numpy.pi / 180),
343 self.lat + (km2deg(r)) * numpy.sin(60 * numpy.pi / 180),
344 344 '{}km'.format(r),
345 345 ha='center', va='bottom', size='8', color='0.6', weight='heavy')
346 346
347 347 if self.mode == 'E':
348 348 title = 'El={}$^\circ$'.format(self.data.meta['elevation'])
349 349 label = 'E{:02d}'.format(int(self.data.meta['elevation']))
350 350 else:
351 351 title = 'Az={}$^\circ$'.format(self.data.meta['azimuth'])
352 352 label = 'A{:02d}'.format(int(self.data.meta['azimuth']))
353 353
354 354 self.save_labels = ['{}-{}'.format(lbl, label) for lbl in self.labels]
355 355 self.titles = ['{} {}'.format(
356 356 self.data.parameters[x], title) for x in self.channels]
357 357
Show file before | Show file after | Hide comments
@@ -1,702 +1,703
1 1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 2 # All rights reserved.
3 3 #
4 4 # Distributed under the terms of the BSD 3-clause license.
5 5 """Classes to plot Spectra data
6 6
7 7 """
8 8
9 9 import os
10 10 import numpy
11 11
12 12 from schainpy.model.graphics.jroplot_base import Plot, plt, log
13 13
14 14
15 15 class SpectraPlot(Plot):
16 16 '''
17 17 Plot for Spectra data
18 18 '''
19 19
20 CODE = 'spc'
20 CODE = 'spc_moments'
21 21 colormap = 'jet'
22 22 plot_type = 'pcolor'
23 23 buffering = False
24 24
25 25 def setup(self):
26 26 self.nplots = len(self.data.channels)
27 27 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
28 28 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
29 29 self.height = 2.6 * self.nrows
30 30 self.cb_label = 'dB'
31 31 if self.showprofile:
32 32 self.width = 4 * self.ncols
33 33 else:
34 34 self.width = 3.5 * self.ncols
35 35 self.plots_adjust.update({'wspace': 0.4, 'hspace':0.4, 'left': 0.1, 'right': 0.9, 'bottom': 0.08})
36 36 self.ylabel = 'Range [km]'
37 37
38 38 def update(self, dataOut):
39 39
40 40 data = {}
41 41 meta = {}
42 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
42 spc = 10 * numpy.log10(dataOut.data_spc / dataOut.normFactor)
43 43 data['spc'] = spc
44 44 data['rti'] = dataOut.getPower()
45 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
46 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
45 data['noise'] = 10 * numpy.log10(dataOut.getNoise() / dataOut.normFactor)
46 meta['xrange'] = (dataOut.getFreqRange(1) / 1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
47 47 if self.CODE == 'spc_moments':
48 48 data['moments'] = dataOut.moments
49 49
50 50 return data, meta
51 51
52 52 def plot(self):
53 53 if self.xaxis == "frequency":
54 54 x = self.data.xrange[0]
55 55 self.xlabel = "Frequency (kHz)"
56 56 elif self.xaxis == "time":
57 57 x = self.data.xrange[1]
58 58 self.xlabel = "Time (ms)"
59 59 else:
60 60 x = self.data.xrange[2]
61 61 self.xlabel = "Velocity (m/s)"
62 62
63 63 if self.CODE == 'spc_moments':
64 64 x = self.data.xrange[2]
65 65 self.xlabel = "Velocity (m/s)"
66 66
67 67 self.titles = []
68 68
69 69 y = self.data.yrange
70 70 self.y = y
71 71
72 72 data = self.data[-1]
73 73 z = data['spc']
74
74 #self.CODE = 'spc_moments'
75 75 for n, ax in enumerate(self.axes):
76 76 noise = data['noise'][n]
77 print(n,self.CODE)
77 78 if self.CODE == 'spc_moments':
78 mean = data['moments'][n, 1]
79 mean = data['moments'][n,1]
79 80 if ax.firsttime:
80 81 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
81 82 self.xmin = self.xmin if self.xmin else -self.xmax
82 83 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
83 84 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
84 85 ax.plt = ax.pcolormesh(x, y, z[n].T,
85 86 vmin=self.zmin,
86 87 vmax=self.zmax,
87 88 cmap=plt.get_cmap(self.colormap)
88 89 )
89 90
90 91 if self.showprofile:
91 92 ax.plt_profile = self.pf_axes[n].plot(
92 93 data['rti'][n], y)[0]
93 94 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
94 95 color="k", linestyle="dashed", lw=1)[0]
95 96 if self.CODE == 'spc_moments':
96 97 ax.plt_mean = ax.plot(mean, y, color='k')[0]
97 98 else:
98 99 ax.plt.set_array(z[n].T.ravel())
99 100 if self.showprofile:
100 101 ax.plt_profile.set_data(data['rti'][n], y)
101 102 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
102 103 if self.CODE == 'spc_moments':
103 104 ax.plt_mean.set_data(mean, y)
104 105 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
105 106
106 107
107 108 class CrossSpectraPlot(Plot):
108 109
109 110 CODE = 'cspc'
110 111 colormap = 'jet'
111 112 plot_type = 'pcolor'
112 113 zmin_coh = None
113 114 zmax_coh = None
114 115 zmin_phase = None
115 116 zmax_phase = None
116 117
117 118 def setup(self):
118 119
119 120 self.ncols = 4
120 121 self.nplots = len(self.data.pairs) * 2
121 122 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
122 123 self.width = 3.1 * self.ncols
123 124 self.height = 2.6 * self.nrows
124 125 self.ylabel = 'Range [km]'
125 126 self.showprofile = False
126 127 self.plots_adjust.update({'left': 0.08, 'right': 0.92, 'wspace': 0.5, 'hspace':0.4, 'top':0.95, 'bottom': 0.08})
127 128
128 129 def update(self, dataOut):
129 130
130 131 data = {}
131 132 meta = {}
132 133
133 134 spc = dataOut.data_spc
134 135 cspc = dataOut.data_cspc
135 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
136 meta['xrange'] = (dataOut.getFreqRange(1) / 1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
136 137 meta['pairs'] = dataOut.pairsList
137 138
138 139 tmp = []
139 140
140 141 for n, pair in enumerate(meta['pairs']):
141 142 out = cspc[n] / numpy.sqrt(spc[pair[0]] * spc[pair[1]])
142 143 coh = numpy.abs(out)
143 144 phase = numpy.arctan2(out.imag, out.real) * 180 / numpy.pi
144 145 tmp.append(coh)
145 146 tmp.append(phase)
146 147
147 148 data['cspc'] = numpy.array(tmp)
148 149
149 150 return data, meta
150 151
151 152 def plot(self):
152 153
153 154 if self.xaxis == "frequency":
154 155 x = self.data.xrange[0]
155 156 self.xlabel = "Frequency (kHz)"
156 157 elif self.xaxis == "time":
157 158 x = self.data.xrange[1]
158 159 self.xlabel = "Time (ms)"
159 160 else:
160 161 x = self.data.xrange[2]
161 162 self.xlabel = "Velocity (m/s)"
162 163
163 164 self.titles = []
164 165
165 166 y = self.data.yrange
166 167 self.y = y
167 168
168 169 data = self.data[-1]
169 170 cspc = data['cspc']
170 171
171 172 for n in range(len(self.data.pairs)):
172 173 pair = self.data.pairs[n]
173 coh = cspc[n*2]
174 phase = cspc[n*2+1]
174 coh = cspc[n * 2]
175 phase = cspc[n * 2 + 1]
175 176 ax = self.axes[2 * n]
176 177 if ax.firsttime:
177 178 ax.plt = ax.pcolormesh(x, y, coh.T,
178 179 vmin=0,
179 180 vmax=1,
180 181 cmap=plt.get_cmap(self.colormap_coh)
181 182 )
182 183 else:
183 184 ax.plt.set_array(coh.T.ravel())
184 185 self.titles.append(
185 186 'Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
186 187
187 188 ax = self.axes[2 * n + 1]
188 189 if ax.firsttime:
189 190 ax.plt = ax.pcolormesh(x, y, phase.T,
190 191 vmin=-180,
191 192 vmax=180,
192 193 cmap=plt.get_cmap(self.colormap_phase)
193 194 )
194 195 else:
195 196 ax.plt.set_array(phase.T.ravel())
196 197 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
197 198
198 199
199 200 class RTIPlot(Plot):
200 201 '''
201 202 Plot for RTI data
202 203 '''
203 204
204 205 CODE = 'rti'
205 206 colormap = 'jet'
206 207 plot_type = 'pcolorbuffer'
207 208
208 209 def setup(self):
209 210 self.xaxis = 'time'
210 211 self.ncols = 1
211 212 self.nrows = len(self.data.channels)
212 213 self.nplots = len(self.data.channels)
213 214 self.ylabel = 'Range [km]'
214 215 self.xlabel = 'Time'
215 216 self.cb_label = 'dB'
216 217 self.plots_adjust.update({'hspace':0.8, 'left': 0.1, 'bottom': 0.08, 'right':0.95})
217 218 self.titles = ['{} Channel {}'.format(
218 219 self.CODE.upper(), x) for x in range(self.nrows)]
219 220
220 221 def update(self, dataOut):
221 222
222 223 data = {}
223 224 meta = {}
224 225 data['rti'] = dataOut.getPower()
225 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
226 data['noise'] = 10 * numpy.log10(dataOut.getNoise() / dataOut.normFactor)
226 227
227 228 return data, meta
228 229
229 230 def plot(self):
230 231 self.x = self.data.times
231 232 self.y = self.data.yrange
232 233 self.z = self.data[self.CODE]
233 234 self.z = numpy.ma.masked_invalid(self.z)
234 235
235 236 if self.decimation is None:
236 237 x, y, z = self.fill_gaps(self.x, self.y, self.z)
237 238 else:
238 239 x, y, z = self.fill_gaps(*self.decimate())
239 240
240 241 for n, ax in enumerate(self.axes):
241 242 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
242 243 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
243 244 data = self.data[-1]
244 245 if ax.firsttime:
245 246 ax.plt = ax.pcolormesh(x, y, z[n].T,
246 247 vmin=self.zmin,
247 248 vmax=self.zmax,
248 249 cmap=plt.get_cmap(self.colormap)
249 250 )
250 251 if self.showprofile:
251 252 ax.plot_profile = self.pf_axes[n].plot(
252 253 data['rti'][n], self.y)[0]
253 254 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(data['noise'][n], len(self.y)), self.y,
254 255 color="k", linestyle="dashed", lw=1)[0]
255 256 else:
256 257 ax.collections.remove(ax.collections[0])
257 258 ax.plt = ax.pcolormesh(x, y, z[n].T,
258 259 vmin=self.zmin,
259 260 vmax=self.zmax,
260 261 cmap=plt.get_cmap(self.colormap)
261 262 )
262 263 if self.showprofile:
263 264 ax.plot_profile.set_data(data['rti'][n], self.y)
264 265 ax.plot_noise.set_data(numpy.repeat(
265 266 data['noise'][n], len(self.y)), self.y)
266 267
267 268
268 269 class CoherencePlot(RTIPlot):
269 270 '''
270 271 Plot for Coherence data
271 272 '''
272 273
273 274 CODE = 'coh'
274 275
275 276 def setup(self):
276 277 self.xaxis = 'time'
277 278 self.ncols = 1
278 279 self.nrows = len(self.data.pairs)
279 280 self.nplots = len(self.data.pairs)
280 281 self.ylabel = 'Range [km]'
281 282 self.xlabel = 'Time'
282 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1,'right':0.95})
283 self.plots_adjust.update({'hspace':0.6, 'left': 0.1, 'bottom': 0.1, 'right':0.95})
283 284 if self.CODE == 'coh':
284 285 self.cb_label = ''
285 286 self.titles = [
286 287 'Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
287 288 else:
288 289 self.cb_label = 'Degrees'
289 290 self.titles = [
290 291 'Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
291 292
292 293 def update(self, dataOut):
293 294
294 295 data = {}
295 296 meta = {}
296 297 data['coh'] = dataOut.getCoherence()
297 298 meta['pairs'] = dataOut.pairsList
298 299
299 300 return data, meta
300 301
301 302 class PhasePlot(CoherencePlot):
302 303 '''
303 304 Plot for Phase map data
304 305 '''
305 306
306 307 CODE = 'phase'
307 308 colormap = 'seismic'
308 309
309 310 def update(self, dataOut):
310 311
311 312 data = {}
312 313 meta = {}
313 314 data['phase'] = dataOut.getCoherence(phase=True)
314 315 meta['pairs'] = dataOut.pairsList
315 316
316 317 return data, meta
317 318
318 319 class NoisePlot(Plot):
319 320 '''
320 321 Plot for noise
321 322 '''
322 323
323 324 CODE = 'noise'
324 325 plot_type = 'scatterbuffer'
325 326
326 327 def setup(self):
327 328 self.xaxis = 'time'
328 329 self.ncols = 1
329 330 self.nrows = 1
330 331 self.nplots = 1
331 332 self.ylabel = 'Intensity [dB]'
332 333 self.xlabel = 'Time'
333 334 self.titles = ['Noise']
334 335 self.colorbar = False
335 336 self.plots_adjust.update({'right': 0.85 })
336 337
337 338 def update(self, dataOut):
338 339
339 340 data = {}
340 341 meta = {}
341 data['noise'] = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor).reshape(dataOut.nChannels, 1)
342 data['noise'] = 10 * numpy.log10(dataOut.getNoise() / dataOut.normFactor).reshape(dataOut.nChannels, 1)
342 343 meta['yrange'] = numpy.array([])
343 344
344 345 return data, meta
345 346
346 347 def plot(self):
347 348
348 349 x = self.data.times
349 350 xmin = self.data.min_time
350 351 xmax = xmin + self.xrange * 60 * 60
351 352 Y = self.data['noise']
352 353
353 354 if self.axes[0].firsttime:
354 355 self.ymin = numpy.nanmin(Y) - 5
355 356 self.ymax = numpy.nanmax(Y) + 5
356 357 for ch in self.data.channels:
357 358 y = Y[ch]
358 359 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
359 360 plt.legend(bbox_to_anchor=(1.18, 1.0))
360 361 else:
361 362 for ch in self.data.channels:
362 363 y = Y[ch]
363 364 self.axes[0].lines[ch].set_data(x, y)
364 365
365 366
366 367 class PowerProfilePlot(Plot):
367 368
368 369 CODE = 'pow_profile'
369 370 plot_type = 'scatter'
370 371
371 372 def setup(self):
372 373
373 374 self.ncols = 1
374 375 self.nrows = 1
375 376 self.nplots = 1
376 377 self.height = 4
377 378 self.width = 3
378 379 self.ylabel = 'Range [km]'
379 380 self.xlabel = 'Intensity [dB]'
380 381 self.titles = ['Power Profile']
381 382 self.colorbar = False
382 383
383 384 def update(self, dataOut):
384 385
385 386 data = {}
386 387 meta = {}
387 388 data[self.CODE] = dataOut.getPower()
388 389
389 390 return data, meta
390 391
391 392 def plot(self):
392 393
393 394 y = self.data.yrange
394 395 self.y = y
395 396
396 397 x = self.data[-1][self.CODE]
397 398
398 if self.xmin is None: self.xmin = numpy.nanmin(x)*0.9
399 if self.xmax is None: self.xmax = numpy.nanmax(x)*1.1
399 if self.xmin is None: self.xmin = numpy.nanmin(x) * 0.9
400 if self.xmax is None: self.xmax = numpy.nanmax(x) * 1.1
400 401
401 402 if self.axes[0].firsttime:
402 403 for ch in self.data.channels:
403 404 self.axes[0].plot(x[ch], y, lw=1, label='Ch{}'.format(ch))
404 405 plt.legend()
405 406 else:
406 407 for ch in self.data.channels:
407 408 self.axes[0].lines[ch].set_data(x[ch], y)
408 409
409 410
410 411 class SpectraCutPlot(Plot):
411 412
412 413 CODE = 'spc_cut'
413 414 plot_type = 'scatter'
414 415 buffering = False
415 416
416 417 def setup(self):
417 418
418 419 self.nplots = len(self.data.channels)
419 420 self.ncols = int(numpy.sqrt(self.nplots) + 0.9)
420 421 self.nrows = int((1.0 * self.nplots / self.ncols) + 0.9)
421 422 self.width = 3.4 * self.ncols + 1.5
422 423 self.height = 3 * self.nrows
423 424 self.ylabel = 'Power [dB]'
424 425 self.colorbar = False
425 426 self.plots_adjust.update({'left':0.1, 'hspace':0.3, 'right': 0.75, 'bottom':0.08})
426 427
427 428 def update(self, dataOut):
428 429
429 430 data = {}
430 431 meta = {}
431 spc = 10*numpy.log10(dataOut.data_spc/dataOut.normFactor)
432 spc = 10 * numpy.log10(dataOut.data_spc / dataOut.normFactor)
432 433 data['spc'] = spc
433 meta['xrange'] = (dataOut.getFreqRange(1)/1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
434 meta['xrange'] = (dataOut.getFreqRange(1) / 1000., dataOut.getAcfRange(1), dataOut.getVelRange(1))
434 435
435 436 return data, meta
436 437
437 438 def plot(self):
438 439 if self.xaxis == "frequency":
439 440 x = self.data.xrange[0][1:]
440 441 self.xlabel = "Frequency (kHz)"
441 442 elif self.xaxis == "time":
442 443 x = self.data.xrange[1]
443 444 self.xlabel = "Time (ms)"
444 445 else:
445 446 x = self.data.xrange[2]
446 447 self.xlabel = "Velocity (m/s)"
447 448
448 449 self.titles = []
449 450
450 451 y = self.data.yrange
451 452 z = self.data[-1]['spc']
452 453
453 454 if self.height_index:
454 455 index = numpy.array(self.height_index)
455 456 else:
456 index = numpy.arange(0, len(y), int((len(y))/9))
457 index = numpy.arange(0, len(y), int((len(y)) / 9))
457 458
458 459 for n, ax in enumerate(self.axes):
459 460 if ax.firsttime:
460 461 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
461 462 self.xmin = self.xmin if self.xmin else -self.xmax
462 463 self.ymin = self.ymin if self.ymin else numpy.nanmin(z)
463 464 self.ymax = self.ymax if self.ymax else numpy.nanmax(z)
464 465 ax.plt = ax.plot(x, z[n, :, index].T)
465 466 labels = ['Range = {:2.1f}km'.format(y[i]) for i in index]
466 467 self.figures[0].legend(ax.plt, labels, loc='center right')
467 468 else:
468 469 for i, line in enumerate(ax.plt):
469 470 line.set_data(x, z[n, :, index[i]])
470 471 self.titles.append('CH {}'.format(n))
471 472
472 473
473 474 class BeaconPhase(Plot):
474 475
475 476 __isConfig = None
476 477 __nsubplots = None
477 478
478 479 PREFIX = 'beacon_phase'
479 480
480 481 def __init__(self):
481 482 Plot.__init__(self)
482 self.timerange = 24*60*60
483 self.timerange = 24 * 60 * 60
483 484 self.isConfig = False
484 485 self.__nsubplots = 1
485 486 self.counter_imagwr = 0
486 487 self.WIDTH = 800
487 488 self.HEIGHT = 400
488 489 self.WIDTHPROF = 120
489 490 self.HEIGHTPROF = 0
490 491 self.xdata = None
491 492 self.ydata = None
492 493
493 494 self.PLOT_CODE = BEACON_CODE
494 495
495 496 self.FTP_WEI = None
496 497 self.EXP_CODE = None
497 498 self.SUB_EXP_CODE = None
498 499 self.PLOT_POS = None
499 500
500 501 self.filename_phase = None
501 502
502 503 self.figfile = None
503 504
504 505 self.xmin = None
505 506 self.xmax = None
506 507
507 508 def getSubplots(self):
508 509
509 510 ncol = 1
510 511 nrow = 1
511 512
512 513 return nrow, ncol
513 514
514 515 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
515 516
516 517 self.__showprofile = showprofile
517 518 self.nplots = nplots
518 519
519 520 ncolspan = 7
520 521 colspan = 6
521 522 self.__nsubplots = 2
522 523
523 self.createFigure(id = id,
524 wintitle = wintitle,
525 widthplot = self.WIDTH+self.WIDTHPROF,
526 heightplot = self.HEIGHT+self.HEIGHTPROF,
524 self.createFigure(id=id,
525 wintitle=wintitle,
526 widthplot=self.WIDTH + self.WIDTHPROF,
527 heightplot=self.HEIGHT + self.HEIGHTPROF,
527 528 show=show)
528 529
529 530 nrow, ncol = self.getSubplots()
530 531
531 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
532 self.addAxes(nrow, ncol * ncolspan, 0, 0, colspan, 1)
532 533
533 534 def save_phase(self, filename_phase):
534 f = open(filename_phase,'w+')
535 f = open(filename_phase, 'w+')
535 536 f.write('\n\n')
536 537 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
537 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
538 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n')
538 539 f.close()
539 540
540 541 def save_data(self, filename_phase, data, data_datetime):
541 f=open(filename_phase,'a')
542 f = open(filename_phase, 'a')
542 543 timetuple_data = data_datetime.timetuple()
543 544 day = str(timetuple_data.tm_mday)
544 545 month = str(timetuple_data.tm_mon)
545 546 year = str(timetuple_data.tm_year)
546 547 hour = str(timetuple_data.tm_hour)
547 548 minute = str(timetuple_data.tm_min)
548 549 second = str(timetuple_data.tm_sec)
549 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
550 f.write(day + ' ' + month + ' ' + year + ' ' + hour + ' ' + minute + ' ' + second + ' ' + str(data[0]) + ' ' + str(data[1]) + ' ' + str(data[2]) + ' ' + str(data[3]) + '\n')
550 551 f.close()
551 552
552 553 def plot(self):
553 554 log.warning('TODO: Not yet implemented...')
554 555
555 556 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
556 557 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
557 558 timerange=None,
558 559 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
559 560 server=None, folder=None, username=None, password=None,
560 561 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
561 562
562 563 if dataOut.flagNoData:
563 564 return dataOut
564 565
565 566 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
566 567 return
567 568
568 569 if pairsList == None:
569 570 pairsIndexList = dataOut.pairsIndexList[:10]
570 571 else:
571 572 pairsIndexList = []
572 573 for pair in pairsList:
573 574 if pair not in dataOut.pairsList:
574 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
575 raise ValueError("Pair %s is not in dataOut.pairsList" % (pair))
575 576 pairsIndexList.append(dataOut.pairsList.index(pair))
576 577
577 578 if pairsIndexList == []:
578 579 return
579 580
580 581 # if len(pairsIndexList) > 4:
581 582 # pairsIndexList = pairsIndexList[0:4]
582 583
583 584 hmin_index = None
584 585 hmax_index = None
585 586
586 587 if hmin != None and hmax != None:
587 588 indexes = numpy.arange(dataOut.nHeights)
588 589 hmin_list = indexes[dataOut.heightList >= hmin]
589 590 hmax_list = indexes[dataOut.heightList <= hmax]
590 591
591 592 if hmin_list.any():
592 593 hmin_index = hmin_list[0]
593 594
594 595 if hmax_list.any():
595 hmax_index = hmax_list[-1]+1
596 hmax_index = hmax_list[-1] + 1
596 597
597 598 x = dataOut.getTimeRange()
598 599
599 600 thisDatetime = dataOut.datatime
600 601
601 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
602 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
602 603 xlabel = "Local Time"
603 604 ylabel = "Phase (degrees)"
604 605
605 606 update_figfile = False
606 607
607 608 nplots = len(pairsIndexList)
608 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
609 # phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
609 610 phase_beacon = numpy.zeros(len(pairsIndexList))
610 611 for i in range(nplots):
611 612 pair = dataOut.pairsList[pairsIndexList[i]]
612 613 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
613 614 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
614 615 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
615 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
616 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
616 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
617 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real) * 180 / numpy.pi
617 618
618 619 if dataOut.beacon_heiIndexList:
619 620 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
620 621 else:
621 622 phase_beacon[i] = numpy.average(phase)
622 623
623 624 if not self.isConfig:
624 625
625 626 nplots = len(pairsIndexList)
626 627
627 628 self.setup(id=id,
628 629 nplots=nplots,
629 630 wintitle=wintitle,
630 631 showprofile=showprofile,
631 632 show=show)
632 633
633 634 if timerange != None:
634 635 self.timerange = timerange
635 636
636 637 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
637 638
638 639 if ymin == None: ymin = 0
639 640 if ymax == None: ymax = 360
640 641
641 642 self.FTP_WEI = ftp_wei
642 643 self.EXP_CODE = exp_code
643 644 self.SUB_EXP_CODE = sub_exp_code
644 645 self.PLOT_POS = plot_pos
645 646
646 647 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
647 648 self.isConfig = True
648 649 self.figfile = figfile
649 650 self.xdata = numpy.array([])
650 651 self.ydata = numpy.array([])
651 652
652 653 update_figfile = True
653 654
654 #open file beacon phase
655 path = '%s%03d' %(self.PREFIX, self.id)
656 beacon_file = os.path.join(path,'%s.txt'%self.name)
657 self.filename_phase = os.path.join(figpath,beacon_file)
658 #self.save_phase(self.filename_phase)
655 # open file beacon phase
656 path = '%s%03d' % (self.PREFIX, self.id)
657 beacon_file = os.path.join(path, '%s.txt' % self.name)
658 self.filename_phase = os.path.join(figpath, beacon_file)
659 # self.save_phase(self.filename_phase)
659 660
660 661
661 #store data beacon phase
662 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
662 # store data beacon phase
663 # self.save_data(self.filename_phase, phase_beacon, thisDatetime)
663 664
664 665 self.setWinTitle(title)
665 666
666 667
667 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
668 title = "Phase Plot %s" % (thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
668 669
669 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
670 legendlabels = ["Pair (%d,%d)" % (pair[0], pair[1]) for pair in dataOut.pairsList]
670 671
671 672 axes = self.axesList[0]
672 673
673 674 self.xdata = numpy.hstack((self.xdata, x[0:1]))
674 675
675 if len(self.ydata)==0:
676 self.ydata = phase_beacon.reshape(-1,1)
676 if len(self.ydata) == 0:
677 self.ydata = phase_beacon.reshape(-1, 1)
677 678 else:
678 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
679 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1, 1)))
679 680
680 681
681 682 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
682 683 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
683 684 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
684 685 XAxisAsTime=True, grid='both'
685 686 )
686 687
687 688 self.draw()
688 689
689 690 if dataOut.ltctime >= self.xmax:
690 691 self.counter_imagwr = wr_period
691 692 self.isConfig = False
692 693 update_figfile = True
693 694
694 695 self.save(figpath=figpath,
695 696 figfile=figfile,
696 697 save=save,
697 698 ftp=ftp,
698 699 wr_period=wr_period,
699 700 thisDatetime=thisDatetime,
700 701 update_figfile=update_figfile)
701 702
702 return dataOut No newline at end of file
703 return dataOut
Show file before | Show file after | Hide comments
@@ -1,302 +1,302
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from schainpy.model.graphics.jroplot_base import Plot, plt
11 11
12 12
13 13 class ScopePlot(Plot):
14 14
15 15 '''
16 16 Plot for Scope
17 17 '''
18 18
19 19 CODE = 'scope'
20 20 plot_type = 'scatter'
21 21
22 22 def setup(self):
23 23
24 24 self.xaxis = 'Range (Km)'
25 25 self.ncols = 1
26 26 self.nrows = 1
27 27 self.nplots = 1
28 28 self.ylabel = 'Intensity [dB]'
29 29 self.titles = ['Scope']
30 30 self.colorbar = False
31 31 self.width = 6
32 32 self.height = 4
33 33
34 34 def update(self, dataOut):
35 35
36 36 data = {}
37 37 meta = {
38 38 'nProfiles': dataOut.nProfiles,
39 39 'flagDataAsBlock': dataOut.flagDataAsBlock,
40 40 'profileIndex': dataOut.profileIndex,
41 41 }
42 42 if self.CODE == 'scope':
43 43 data[self.CODE] = dataOut.data
44 44 elif self.CODE == 'pp_power':
45 45 data[self.CODE] = dataOut.dataPP_POWER
46 46 elif self.CODE == 'pp_signal':
47 47 data[self.CODE] = dataOut.dataPP_POW
48 48 elif self.CODE == 'pp_velocity':
49 49 data[self.CODE] = dataOut.dataPP_DOP
50 50 elif self.CODE == 'pp_specwidth':
51 51 data[self.CODE] = dataOut.dataPP_WIDTH
52 52
53 53 return data, meta
54 54
55 55 def plot_iq(self, x, y, channelIndexList, thisDatetime, wintitle):
56 56
57 yreal = y[channelIndexList,:].real
58 yimag = y[channelIndexList,:].imag
59 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y"))
57 yreal = y[channelIndexList, :].real
58 yimag = y[channelIndexList, :].imag
59 title = wintitle + " Scope: %s" % (thisDatetime.strftime("%d-%b-%Y"))
60 60 self.xlabel = "Range (Km)"
61 61 self.ylabel = "Intensity - IQ"
62 62
63 63 self.y = yreal
64 64 self.x = x
65 65
66 66 self.titles[0] = title
67 67
68 for i,ax in enumerate(self.axes):
69 title = "Channel %d" %(i)
68 for i, ax in enumerate(self.axes):
69 title = "Channel %d" % (i)
70 70 if ax.firsttime:
71 71 self.xmin = min(x)
72 72 self.xmax = max(x)
73 ax.plt_r = ax.plot(x, yreal[i,:], color='b')[0]
74 ax.plt_i = ax.plot(x, yimag[i,:], color='r')[0]
73 ax.plt_r = ax.plot(x, yreal[i, :], color='b')[0]
74 ax.plt_i = ax.plot(x, yimag[i, :], color='r')[0]
75 75 else:
76 ax.plt_r.set_data(x, yreal[i,:])
77 ax.plt_i.set_data(x, yimag[i,:])
76 ax.plt_r.set_data(x, yreal[i, :])
77 ax.plt_i.set_data(x, yimag[i, :])
78 78
79 79 def plot_power(self, x, y, channelIndexList, thisDatetime, wintitle):
80 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
80 y = y[channelIndexList, :] * numpy.conjugate(y[channelIndexList, :])
81 81 yreal = y.real
82 yreal = 10*numpy.log10(yreal)
82 yreal = 10 * numpy.log10(yreal)
83 83 self.y = yreal
84 title = wintitle + " Power: %s" %(thisDatetime.strftime("%d-%b-%Y"))
84 title = wintitle + " Power: %s" % (thisDatetime.strftime("%d-%b-%Y"))
85 85 self.xlabel = "Range (Km)"
86 86 self.ylabel = "Intensity [dB]"
87 87
88 88
89 89 self.titles[0] = title
90 90
91 for i,ax in enumerate(self.axes):
92 title = "Channel %d" %(i)
93 ychannel = yreal[i,:]
91 for i, ax in enumerate(self.axes):
92 title = "Channel %d" % (i)
93 ychannel = yreal[i, :]
94 94
95 95 if ax.firsttime:
96 96 self.xmin = min(x)
97 97 self.xmax = max(x)
98 98 ax.plt_r = ax.plot(x, ychannel)[0]
99 99 else:
100 100 ax.plt_r.set_data(x, ychannel)
101 101
102 102 def plot_weatherpower(self, x, y, channelIndexList, thisDatetime, wintitle):
103 103
104 104
105 y = y[channelIndexList,:]
106 yreal = y.real
107 yreal = 10*numpy.log10(yreal)
105 y = y[channelIndexList, :]
106 yreal = y.real
107 yreal = 10 * numpy.log10(yreal)
108 108 self.y = yreal
109 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
109 title = wintitle + " Scope: %s" % (thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
110 110 self.xlabel = "Range (Km)"
111 111 self.ylabel = "Intensity"
112 self.xmin = min(x)
113 self.xmax = max(x)
112 self.xmin = min(x)
113 self.xmax = max(x)
114 114
115 self.titles[0] =title
116 for i,ax in enumerate(self.axes):
117 title = "Channel %d" %(i)
115 self.titles[0] = title
116 for i, ax in enumerate(self.axes):
117 title = "Channel %d" % (i)
118 118
119 ychannel = yreal[i,:]
119 ychannel = yreal[i, :]
120 120
121 121 if ax.firsttime:
122 122 ax.plt_r = ax.plot(x, ychannel)[0]
123 123 else:
124 #pass
124 # pass
125 125 ax.plt_r.set_data(x, ychannel)
126 126
127 127 def plot_weathervelocity(self, x, y, channelIndexList, thisDatetime, wintitle):
128 128
129 x = x[channelIndexList,:]
130 yreal = y
129 x = x[channelIndexList, :]
130 yreal = y
131 131 self.y = yreal
132 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
132 title = wintitle + " Scope: %s" % (thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
133 133 self.xlabel = "Velocity (m/s)"
134 134 self.ylabel = "Range (Km)"
135 self.xmin = numpy.min(x)
136 self.xmax = numpy.max(x)
137 self.titles[0] =title
138 for i,ax in enumerate(self.axes):
139 title = "Channel %d" %(i)
140 xchannel = x[i,:]
135 self.xmin = numpy.min(x)
136 self.xmax = numpy.max(x)
137 self.titles[0] = title
138 for i, ax in enumerate(self.axes):
139 title = "Channel %d" % (i)
140 xchannel = x[i, :]
141 141 if ax.firsttime:
142 142 ax.plt_r = ax.plot(xchannel, yreal)[0]
143 143 else:
144 #pass
144 # pass
145 145 ax.plt_r.set_data(xchannel, yreal)
146 146
147 147 def plot_weatherspecwidth(self, x, y, channelIndexList, thisDatetime, wintitle):
148 148
149 x = x[channelIndexList,:]
150 yreal = y
149 x = x[channelIndexList, :]
150 yreal = y
151 151 self.y = yreal
152 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
152 title = wintitle + " Scope: %s" % (thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
153 153 self.xlabel = "width "
154 154 self.ylabel = "Range (Km)"
155 self.xmin = numpy.min(x)
156 self.xmax = numpy.max(x)
157 self.titles[0] =title
158 for i,ax in enumerate(self.axes):
159 title = "Channel %d" %(i)
160 xchannel = x[i,:]
155 self.xmin = numpy.min(x)
156 self.xmax = numpy.max(x)
157 self.titles[0] = title
158 for i, ax in enumerate(self.axes):
159 title = "Channel %d" % (i)
160 xchannel = x[i, :]
161 161 if ax.firsttime:
162 162 ax.plt_r = ax.plot(xchannel, yreal)[0]
163 163 else:
164 #pass
164 # pass
165 165 ax.plt_r.set_data(xchannel, yreal)
166 166
167 167 def plot(self):
168 168 if self.channels:
169 169 channels = self.channels
170 170 else:
171 171 channels = self.data.channels
172 172
173 173 thisDatetime = datetime.datetime.utcfromtimestamp(self.data.times[-1])
174 174
175 175 scope = self.data[-1][self.CODE]
176 176
177 177 if self.data.flagDataAsBlock:
178 178
179 179 for i in range(self.data.nProfiles):
180 180
181 wintitle1 = " [Profile = %d] " %i
182 if self.CODE =="scope":
181 wintitle1 = " [Profile = %d] " % i
182 if self.CODE == "scope":
183 183 if self.type == "power":
184 184 self.plot_power(self.data.yrange,
185 scope[:,i,:],
185 scope[:, i, :],
186 186 channels,
187 187 thisDatetime,
188 188 wintitle1
189 189 )
190 190
191 191 if self.type == "iq":
192 192 self.plot_iq(self.data.yrange,
193 scope[:,i,:],
193 scope[:, i, :],
194 194 channels,
195 195 thisDatetime,
196 196 wintitle1
197 197 )
198 if self.CODE=="pp_power":
198 if self.CODE == "pp_power":
199 199 self.plot_weatherpower(self.data.yrange,
200 scope[:,i,:],
200 scope[:, i, :],
201 201 channels,
202 202 thisDatetime,
203 203 wintitle
204 204 )
205 if self.CODE=="pp_signal":
205 if self.CODE == "pp_signal":
206 206 self.plot_weatherpower(self.data.yrange,
207 scope[:,i,:],
207 scope[:, i, :],
208 208 channels,
209 209 thisDatetime,
210 210 wintitle
211 211 )
212 if self.CODE=="pp_velocity":
213 self.plot_weathervelocity(scope[:,i,:],
212 if self.CODE == "pp_velocity":
213 self.plot_weathervelocity(scope[:, i, :],
214 214 self.data.yrange,
215 215 channels,
216 216 thisDatetime,
217 217 wintitle
218 218 )
219 if self.CODE=="pp_spcwidth":
220 self.plot_weatherspecwidth(scope[:,i,:],
219 if self.CODE == "pp_spcwidth":
220 self.plot_weatherspecwidth(scope[:, i, :],
221 221 self.data.yrange,
222 222 channels,
223 223 thisDatetime,
224 224 wintitle
225 225 )
226 226 else:
227 wintitle = " [Profile = %d] " %self.data.profileIndex
228 if self.CODE== "scope":
227 wintitle = " [Profile = %d] " % self.data.profileIndex
228 if self.CODE == "scope":
229 229 if self.type == "power":
230 230 self.plot_power(self.data.yrange,
231 231 scope,
232 232 channels,
233 233 thisDatetime,
234 234 wintitle
235 235 )
236 236
237 237 if self.type == "iq":
238 238 self.plot_iq(self.data.yrange,
239 239 scope,
240 240 channels,
241 241 thisDatetime,
242 242 wintitle
243 243 )
244 if self.CODE=="pp_power":
244 if self.CODE == "pp_power":
245 245 self.plot_weatherpower(self.data.yrange,
246 246 scope,
247 247 channels,
248 248 thisDatetime,
249 249 wintitle
250 250 )
251 if self.CODE=="pp_signal":
251 if self.CODE == "pp_signal":
252 252 self.plot_weatherpower(self.data.yrange,
253 253 scope,
254 254 channels,
255 255 thisDatetime,
256 256 wintitle
257 257 )
258 if self.CODE=="pp_velocity":
258 if self.CODE == "pp_velocity":
259 259 self.plot_weathervelocity(scope,
260 260 self.data.yrange,
261 261 channels,
262 262 thisDatetime,
263 263 wintitle
264 264 )
265 if self.CODE=="pp_specwidth":
265 if self.CODE == "pp_specwidth":
266 266 self.plot_weatherspecwidth(scope,
267 267 self.data.yrange,
268 268 channels,
269 269 thisDatetime,
270 270 wintitle
271 271 )
272 272
273 273
274 274 class PulsepairPowerPlot(ScopePlot):
275 275 '''
276 276 Plot for P= S+N
277 277 '''
278 278
279 279 CODE = 'pp_power'
280 280 plot_type = 'scatter'
281 281
282 282 class PulsepairVelocityPlot(ScopePlot):
283 283 '''
284 284 Plot for VELOCITY
285 285 '''
286 286 CODE = 'pp_velocity'
287 287 plot_type = 'scatter'
288 288
289 289 class PulsepairSpecwidthPlot(ScopePlot):
290 290 '''
291 291 Plot for WIDTH
292 292 '''
293 293 CODE = 'pp_specwidth'
294 294 plot_type = 'scatter'
295 295
296 296 class PulsepairSignalPlot(ScopePlot):
297 297 '''
298 298 Plot for S
299 299 '''
300 300
301 301 CODE = 'pp_signal'
302 302 plot_type = 'scatter'
Show file before | Show file after | Hide comments
@@ -1,28 +1,28
1 1 '''
2 2 @author: roj-idl71
3 3 '''
4 #USED IN jroplot_spectra.py
5 RTI_CODE = 0 #Range time intensity (RTI).
6 SPEC_CODE = 1 #Spectra (and Cross-spectra) information.
7 CROSS_CODE = 2 #Cross-Correlation information.
8 COH_CODE = 3 #Coherence map.
9 BASE_CODE = 4 #Base lines graphic.
10 ROW_CODE = 5 #Row Spectra.
11 TOTAL_CODE = 6 #Total Power.
12 DRIFT_CODE = 7 #Drifts graphics.
13 HEIGHT_CODE = 8 #Height profile.
14 PHASE_CODE = 9 #Signal Phase.
4 # USED IN jroplot_spectra.py
5 RTI_CODE = 0 # Range time intensity (RTI).
6 SPEC_CODE = 1 # Spectra (and Cross-spectra) information.
7 CROSS_CODE = 2 # Cross-Correlation information.
8 COH_CODE = 3 # Coherence map.
9 BASE_CODE = 4 # Base lines graphic.
10 ROW_CODE = 5 # Row Spectra.
11 TOTAL_CODE = 6 # Total Power.
12 DRIFT_CODE = 7 # Drifts graphics.
13 HEIGHT_CODE = 8 # Height profile.
14 PHASE_CODE = 9 # Signal Phase.
15 15
16 16 POWER_CODE = 16
17 17 NOISE_CODE = 17
18 18 BEACON_CODE = 18
19 19
20 #USED IN jroplot_parameters.py
20 # USED IN jroplot_parameters.py
21 21 WIND_CODE = 22
22 22 MSKYMAP_CODE = 23
23 23 MPHASE_CODE = 24
24 24
25 25 MOMENTS_CODE = 25
26 26 PARMS_CODE = 26
27 27 SPECFIT_CODE = 27
28 28 EWDRIFT_CODE = 28
Show file before | Show file after | Hide comments
@@ -1,331 +1,331
1 1 import os
2 2 import sys
3 3 import glob
4 4 import fnmatch
5 5 import datetime
6 6 import time
7 7 import re
8 8 import h5py
9 9 import numpy
10 10
11 11 from scipy.optimize import curve_fit
12 12 from scipy import asarray as ar, exp
13 13 from scipy import stats
14 14
15 15 from duplicity.path import Path
16 16 from numpy.ma.core import getdata
17 17
18 18 SPEED_OF_LIGHT = 299792458
19 19 SPEED_OF_LIGHT = 3e8
20 20
21 21 try:
22 22 from gevent import sleep
23 23 except:
24 24 from time import sleep
25 25
26 26 from schainpy.model.data.jrodata import Spectra
27 #from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader
27 # from schainpy.model.data.BLTRheaderIO import FileHeader, RecordHeader
28 28 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
29 #from schainpy.model.io.jroIO_bltr import BLTRReader
29 # from schainpy.model.io.jroIO_bltr import BLTRReader
30 30 from numpy import imag, shape, NaN
31 31
32 32
33 33 startFp = open(
34 34 '/home/erick/Documents/MIRA35C/20160117/20160117_0000.zspc', "rb")
35 35
36 36
37 37 FILE_HEADER = numpy.dtype([ # HEADER 1024bytes
38 38 ('Hname', numpy.str_, 32), # Original file name
39 39 # Date and time when the file was created
40 40 ('Htime', numpy.str_, 32),
41 41 # Name of operator who created the file
42 42 ('Hoper', numpy.str_, 64),
43 43 # Place where the measurements was carried out
44 44 ('Hplace', numpy.str_, 128),
45 45 # Description of measurements
46 46 ('Hdescr', numpy.str_, 256),
47 47 ('Hdummy', numpy.str_, 512), # Reserved space
48 48 # Main chunk
49 49 ('Msign', '<i4'), # Main chunk signature FZKF or NUIG
50 50 ('MsizeData', '<i4'), # Size of data block main chunk
51 51 # Processing DSP parameters
52 52 ('PPARsign', '<i4'), # PPAR signature
53 53 ('PPARsize', '<i4'), # PPAR size of block
54 54 ('PPARprf', '<i4'), # Pulse repetition frequency
55 55 ('PPARpdr', '<i4'), # Pulse duration
56 56 ('PPARsft', '<i4'), # FFT length
57 57 # Number of spectral (in-coherent) averages
58 58 ('PPARavc', '<i4'),
59 59 # Number of lowest range gate for moment estimation
60 60 ('PPARihp', '<i4'),
61 61 # Count for gates for moment estimation
62 62 ('PPARchg', '<i4'),
63 63 # switch on/off polarimetric measurements. Should be 1.
64 64 ('PPARpol', '<i4'),
65 65 # Service DSP parameters
66 66 # STC attenuation on the lowest ranges on/off
67 67 ('SPARatt', '<i4'),
68 68 ('SPARtx', '<i4'), # OBSOLETE
69 69 ('SPARaddGain0', '<f4'), # OBSOLETE
70 70 ('SPARaddGain1', '<f4'), # OBSOLETE
71 71 # Debug only. It normal mode it is 0.
72 72 ('SPARwnd', '<i4'),
73 73 # Delay between sync pulse and tx pulse for phase corr, ns
74 74 ('SPARpos', '<i4'),
75 75 # "add to pulse" to compensate for delay between the leading edge of driver pulse and envelope of the RF signal.
76 76 ('SPARadd', '<i4'),
77 77 # Time for measuring txn pulse phase. OBSOLETE
78 78 ('SPARlen', '<i4'),
79 79 ('SPARcal', '<i4'), # OBSOLETE
80 80 ('SPARnos', '<i4'), # OBSOLETE
81 81 ('SPARof0', '<i4'), # detection threshold
82 82 ('SPARof1', '<i4'), # OBSOLETE
83 83 ('SPARswt', '<i4'), # 2nd moment estimation threshold
84 84 ('SPARsum', '<i4'), # OBSOLETE
85 85 ('SPARosc', '<i4'), # flag Oscillosgram mode
86 86 ('SPARtst', '<i4'), # OBSOLETE
87 87 ('SPARcor', '<i4'), # OBSOLETE
88 88 ('SPARofs', '<i4'), # OBSOLETE
89 89 # Hildebrand div noise detection on noise gate
90 90 ('SPARhsn', '<i4'),
91 91 # Hildebrand div noise detection on all gates
92 92 ('SPARhsa', '<f4'),
93 93 ('SPARcalibPow_M', '<f4'), # OBSOLETE
94 94 ('SPARcalibSNR_M', '<f4'), # OBSOLETE
95 95 ('SPARcalibPow_S', '<f4'), # OBSOLETE
96 96 ('SPARcalibSNR_S', '<f4'), # OBSOLETE
97 97 # Lowest range gate for spectra saving Raw_Gate1 >=5
98 98 ('SPARrawGate1', '<i4'),
99 99 # Number of range gates with atmospheric signal
100 100 ('SPARrawGate2', '<i4'),
101 101 # flag - IQ or spectra saving on/off
102 102 ('SPARraw', '<i4'),
103 103 ('SPARprc', '<i4'), ]) # flag - Moment estimation switched on/off
104 104
105 105
106 106 self.Hname = None
107 107 self.Htime = None
108 108 self.Hoper = None
109 109 self.Hplace = None
110 110 self.Hdescr = None
111 111 self.Hdummy = None
112 112
113 113 self.Msign = None
114 114 self.MsizeData = None
115 115
116 116 self.PPARsign = None
117 117 self.PPARsize = None
118 118 self.PPARprf = None
119 119 self.PPARpdr = None
120 120 self.PPARsft = None
121 121 self.PPARavc = None
122 122 self.PPARihp = None
123 123 self.PPARchg = None
124 124 self.PPARpol = None
125 125 # Service DSP parameters
126 126 self.SPARatt = None
127 127 self.SPARtx = None
128 128 self.SPARaddGain0 = None
129 129 self.SPARaddGain1 = None
130 130 self.SPARwnd = None
131 131 self.SPARpos = None
132 132 self.SPARadd = None
133 133 self.SPARlen = None
134 134 self.SPARcal = None
135 135 self.SPARnos = None
136 136 self.SPARof0 = None
137 137 self.SPARof1 = None
138 138 self.SPARswt = None
139 139 self.SPARsum = None
140 140 self.SPARosc = None
141 141 self.SPARtst = None
142 142 self.SPARcor = None
143 143 self.SPARofs = None
144 144 self.SPARhsn = None
145 145 self.SPARhsa = None
146 146 self.SPARcalibPow_M = None
147 147 self.SPARcalibSNR_M = None
148 148 self.SPARcalibPow_S = None
149 149 self.SPARcalibSNR_S = None
150 150 self.SPARrawGate1 = None
151 151 self.SPARrawGate2 = None
152 152 self.SPARraw = None
153 153 self.SPARprc = None
154 154
155 155
156 156 header = numpy.fromfile(fp, FILE_HEADER, 1)
157 157 ''' numpy.fromfile(file, dtype, count, sep='')
158 158 file : file or str
159 159 Open file object or filename.
160 160
161 161 dtype : data-type
162 162 Data type of the returned array. For binary files, it is used to determine
163 163 the size and byte-order of the items in the file.
164 164
165 165 count : int
166 166 Number of items to read. -1 means all items (i.e., the complete file).
167 167
168 168 sep : str
169 169 Separator between items if file is a text file. Empty ("") separator means
170 170 the file should be treated as binary. Spaces (" ") in the separator match zero
171 171 or more whitespace characters. A separator consisting only of spaces must match
172 172 at least one whitespace.
173 173
174 174 '''
175 175
176 176 Hname = str(header['Hname'][0])
177 177 Htime = str(header['Htime'][0])
178 178 Hoper = str(header['Hoper'][0])
179 179 Hplace = str(header['Hplace'][0])
180 180 Hdescr = str(header['Hdescr'][0])
181 181 Hdummy = str(header['Hdummy'][0])
182 182
183 183 Msign = header['Msign'][0]
184 184 MsizeData = header['MsizeData'][0]
185 185
186 186 PPARsign = header['PPARsign'][0]
187 187 PPARsize = header['PPARsize'][0]
188 188 PPARprf = header['PPARprf'][0]
189 189 PPARpdr = header['PPARpdr'][0]
190 190 PPARsft = header['PPARsft'][0]
191 191 PPARavc = header['PPARavc'][0]
192 192 PPARihp = header['PPARihp'][0]
193 193 PPARchg = header['PPARchg'][0]
194 194 PPARpol = header['PPARpol'][0]
195 195 # Service DSP parameters
196 196 SPARatt = header['SPARatt'][0]
197 197 SPARtx = header['SPARtx'][0]
198 198 SPARaddGain0 = header['SPARaddGain0'][0]
199 199 SPARaddGain1 = header['SPARaddGain1'][0]
200 200 SPARwnd = header['SPARwnd'][0]
201 201 SPARpos = header['SPARpos'][0]
202 202 SPARadd = header['SPARadd'][0]
203 203 SPARlen = header['SPARlen'][0]
204 204 SPARcal = header['SPARcal'][0]
205 205 SPARnos = header['SPARnos'][0]
206 206 SPARof0 = header['SPARof0'][0]
207 207 SPARof1 = header['SPARof1'][0]
208 208 SPARswt = header['SPARswt'][0]
209 209 SPARsum = header['SPARsum'][0]
210 210 SPARosc = header['SPARosc'][0]
211 211 SPARtst = header['SPARtst'][0]
212 212 SPARcor = header['SPARcor'][0]
213 213 SPARofs = header['SPARofs'][0]
214 214 SPARhsn = header['SPARhsn'][0]
215 215 SPARhsa = header['SPARhsa'][0]
216 216 SPARcalibPow_M = header['SPARcalibPow_M'][0]
217 217 SPARcalibSNR_M = header['SPARcalibSNR_M'][0]
218 218 SPARcalibPow_S = header['SPARcalibPow_S'][0]
219 219 SPARcalibSNR_S = header['SPARcalibSNR_S'][0]
220 220 SPARrawGate1 = header['SPARrawGate1'][0]
221 221 SPARrawGate2 = header['SPARrawGate2'][0]
222 222 SPARraw = header['SPARraw'][0]
223 223 SPARprc = header['SPARprc'][0]
224 224
225 225
226 226 SRVI_STRUCTURE = numpy.dtype([
227 227 ('frame_cnt', '<u4'),
228 ('time_t', '<u4'), #
229 ('tpow', '<f4'), #
230 ('npw1', '<f4'), #
231 ('npw2', '<f4'), #
232 ('cpw1', '<f4'), #
233 ('pcw2', '<f4'), #
234 ('ps_err', '<u4'), #
235 ('te_err', '<u4'), #
236 ('rc_err', '<u4'), #
237 ('grs1', '<u4'), #
238 ('grs2', '<u4'), #
239 ('azipos', '<f4'), #
240 ('azivel', '<f4'), #
241 ('elvpos', '<f4'), #
242 ('elvvel', '<f4'), #
228 ('time_t', '<u4'), #
229 ('tpow', '<f4'), #
230 ('npw1', '<f4'), #
231 ('npw2', '<f4'), #
232 ('cpw1', '<f4'), #
233 ('pcw2', '<f4'), #
234 ('ps_err', '<u4'), #
235 ('te_err', '<u4'), #
236 ('rc_err', '<u4'), #
237 ('grs1', '<u4'), #
238 ('grs2', '<u4'), #
239 ('azipos', '<f4'), #
240 ('azivel', '<f4'), #
241 ('elvpos', '<f4'), #
242 ('elvvel', '<f4'), #
243 243 ('northAngle', '<f4'),
244 ('microsec', '<u4'), #
244 ('microsec', '<u4'), #
245 245 ('azisetvel', '<f4'), #
246 246 ('elvsetpos', '<f4'), #
247 ('RadarConst', '<f4'), ]) #
247 ('RadarConst', '<f4'), ]) #
248 248
249 249 JUMP_STRUCTURE = numpy.dtype([
250 250 ('jump', '<u140'),
251 251 ('SizeOfDataBlock1', numpy.str_, 32),
252 252 ('jump', '<i4'),
253 253 ('DataBlockTitleSRVI1', numpy.str_, 32),
254 254 ('SizeOfSRVI1', '<i4'), ])
255 255
256 256
257 257 # frame_cnt=0, time_t= 0, tpow=0, npw1=0, npw2=0,
258 258 # cpw1=0, pcw2=0, ps_err=0, te_err=0, rc_err=0, grs1=0,
259 259 # grs2=0, azipos=0, azivel=0, elvpos=0, elvvel=0, northangle=0,
260 260 # microsec=0, azisetvel=0, elvsetpos=0, RadarConst=0
261 261
262 262
263 263 frame_cnt = frame_cnt
264 264 dwell = time_t
265 265 tpow = tpow
266 266 npw1 = npw1
267 267 npw2 = npw2
268 268 cpw1 = cpw1
269 269 pcw2 = pcw2
270 270 ps_err = ps_err
271 271 te_err = te_err
272 272 rc_err = rc_err
273 273 grs1 = grs1
274 274 grs2 = grs2
275 275 azipos = azipos
276 276 azivel = azivel
277 277 elvpos = elvpos
278 278 elvvel = elvvel
279 279 northAngle = northAngle
280 280 microsec = microsec
281 281 azisetvel = azisetvel
282 282 elvsetpos = elvsetpos
283 283 RadarConst5 = RadarConst
284 284
285 285
286 286 # print fp
287 287 # startFp = open('/home/erick/Documents/Data/huancayo.20161019.22.fdt',"rb") #The method tell() returns the current position of the file read/write pointer within the file.
288 288 # startFp = open(fp,"rb") #The method tell() returns the current position of the file read/write pointer within the file.
289 289 # RecCounter=0
290 290 # Off2StartNxtRec=811248
291 291 # print 'OffsetStartHeader ',self.OffsetStartHeader,'RecCounter ', self.RecCounter, 'Off2StartNxtRec ' , self.Off2StartNxtRec
292 #OffRHeader= self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec
293 #startFp.seek(OffRHeader, os.SEEK_SET)
292 # OffRHeader= self.OffsetStartHeader + self.RecCounter*self.Off2StartNxtRec
293 # startFp.seek(OffRHeader, os.SEEK_SET)
294 294 print('debe ser 48, RecCounter*811248', self.OffsetStartHeader, self.RecCounter, self.Off2StartNxtRec)
295 295 print('Posicion del bloque: ', OffRHeader)
296 296
297 297 header = numpy.fromfile(startFp, SRVI_STRUCTURE, 1)
298 298
299 299 self.frame_cnt = header['frame_cnt'][0]
300 self.time_t = header['frame_cnt'][0] #
301 self.tpow = header['frame_cnt'][0] #
302 self.npw1 = header['frame_cnt'][0] #
303 self.npw2 = header['frame_cnt'][0] #
304 self.cpw1 = header['frame_cnt'][0] #
305 self.pcw2 = header['frame_cnt'][0] #
306 self.ps_err = header['frame_cnt'][0] #
307 self.te_err = header['frame_cnt'][0] #
308 self.rc_err = header['frame_cnt'][0] #
309 self.grs1 = header['frame_cnt'][0] #
310 self.grs2 = header['frame_cnt'][0] #
311 self.azipos = header['frame_cnt'][0] #
312 self.azivel = header['frame_cnt'][0] #
313 self.elvpos = header['frame_cnt'][0] #
314 self.elvvel = header['frame_cnt'][0] #
315 self.northAngle = header['frame_cnt'][0] #
316 self.microsec = header['frame_cnt'][0] #
317 self.azisetvel = header['frame_cnt'][0] #
318 self.elvsetpos = header['frame_cnt'][0] #
319 self.RadarConst = header['frame_cnt'][0] #
300 self.time_t = header['frame_cnt'][0] #
301 self.tpow = header['frame_cnt'][0] #
302 self.npw1 = header['frame_cnt'][0] #
303 self.npw2 = header['frame_cnt'][0] #
304 self.cpw1 = header['frame_cnt'][0] #
305 self.pcw2 = header['frame_cnt'][0] #
306 self.ps_err = header['frame_cnt'][0] #
307 self.te_err = header['frame_cnt'][0] #
308 self.rc_err = header['frame_cnt'][0] #
309 self.grs1 = header['frame_cnt'][0] #
310 self.grs2 = header['frame_cnt'][0] #
311 self.azipos = header['frame_cnt'][0] #
312 self.azivel = header['frame_cnt'][0] #
313 self.elvpos = header['frame_cnt'][0] #
314 self.elvvel = header['frame_cnt'][0] #
315 self.northAngle = header['frame_cnt'][0] #
316 self.microsec = header['frame_cnt'][0] #
317 self.azisetvel = header['frame_cnt'][0] #
318 self.elvsetpos = header['frame_cnt'][0] #
319 self.RadarConst = header['frame_cnt'][0] #
320 320
321 321
322 322 self.ipp = 0.5 * (SPEED_OF_LIGHT / self.PRFhz)
323 323
324 324 self.RHsize = 180 + 20 * self.nChannels
325 325 self.Datasize = self.nProfiles * self.nChannels * self.nHeights * 2 * 4
326 326 # print 'Datasize',self.Datasize
327 327 endFp = self.OffsetStartHeader + self.RecCounter * self.Off2StartNxtRec
328 328
329 329 print('==============================================')
330 330
331 print('==============================================') No newline at end of file
331 print('==============================================')
Show file before | Show file after | Hide comments
@@ -1,24 +1,24
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 from .jroIO_voltage import *
8 8 from .jroIO_spectra import *
9 9 from .jroIO_heispectra import *
10 10 from .jroIO_usrp import *
11 11 from .jroIO_digitalRF import *
12 12 from .jroIO_kamisr import *
13 13 from .jroIO_param import *
14 14 from .jroIO_hf import *
15 15
16 16 from .jroIO_madrigal import *
17 17
18 18 from .bltrIO_param import *
19 19 from .bltrIO_spectra import *
20 20 from .jroIO_mira35c import *
21 21 from .julIO_param import *
22 22
23 23 from .pxIO_param import *
24 from .jroIO_simulator import * No newline at end of file
24 from .jroIO_simulator import *
Show file before | Show file after | Hide comments
@@ -1,355 +1,355
1 1 '''
2 2 Created on Nov 9, 2016
3 3
4 4 @author: roj- LouVD
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 16 import schainpy.admin
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
18 18 from schainpy.model.data.jrodata import Parameters
19 19 from schainpy.model.io.jroIO_base import Reader
20 20 from schainpy.utils import log
21 21
22 22 FILE_HEADER_STRUCTURE = numpy.dtype([
23 23 ('FMN', '<u4'),
24 24 ('nrec', '<u4'),
25 25 ('fr_offset', '<u4'),
26 26 ('id', '<u4'),
27 27 ('site', 'u1', (32,))
28 28 ])
29 29
30 30 REC_HEADER_STRUCTURE = numpy.dtype([
31 31 ('rmn', '<u4'),
32 32 ('rcounter', '<u4'),
33 33 ('nr_offset', '<u4'),
34 34 ('tr_offset', '<u4'),
35 35 ('time', '<u4'),
36 36 ('time_msec', '<u4'),
37 37 ('tag', 'u1', (32,)),
38 38 ('comments', 'u1', (32,)),
39 39 ('lat', '<f4'),
40 40 ('lon', '<f4'),
41 41 ('gps_status', '<u4'),
42 42 ('freq', '<u4'),
43 43 ('freq0', '<u4'),
44 44 ('nchan', '<u4'),
45 45 ('delta_r', '<u4'),
46 46 ('nranges', '<u4'),
47 47 ('r0', '<u4'),
48 48 ('prf', '<u4'),
49 49 ('ncoh', '<u4'),
50 50 ('npoints', '<u4'),
51 51 ('polarization', '<i4'),
52 52 ('rx_filter', '<u4'),
53 53 ('nmodes', '<u4'),
54 54 ('dmode_index', '<u4'),
55 55 ('dmode_rngcorr', '<u4'),
56 56 ('nrxs', '<u4'),
57 57 ('acf_length', '<u4'),
58 58 ('acf_lags', '<u4'),
59 59 ('sea_to_atmos', '<f4'),
60 60 ('sea_notch', '<u4'),
61 61 ('lh_sea', '<u4'),
62 62 ('hh_sea', '<u4'),
63 63 ('nbins_sea', '<u4'),
64 64 ('min_snr', '<f4'),
65 65 ('min_cc', '<f4'),
66 66 ('max_time_diff', '<f4')
67 67 ])
68 68
69 69 DATA_STRUCTURE = numpy.dtype([
70 70 ('range', '<u4'),
71 71 ('status', '<u4'),
72 72 ('zonal', '<f4'),
73 73 ('meridional', '<f4'),
74 74 ('vertical', '<f4'),
75 75 ('zonal_a', '<f4'),
76 76 ('meridional_a', '<f4'),
77 77 ('corrected_fading', '<f4'), # seconds
78 78 ('uncorrected_fading', '<f4'), # seconds
79 79 ('time_diff', '<f4'),
80 80 ('major_axis', '<f4'),
81 81 ('axial_ratio', '<f4'),
82 82 ('orientation', '<f4'),
83 83 ('sea_power', '<u4'),
84 84 ('sea_algorithm', '<u4')
85 85 ])
86 86
87 87
88 88 class BLTRParamReader(Reader, ProcessingUnit):
89 89 '''
90 90 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
91 91 from *.sswma files
92 92 '''
93 93
94 94 ext = '.sswma'
95 95
96 96 def __init__(self):
97 97
98 98 ProcessingUnit.__init__(self)
99 99
100 100 self.dataOut = Parameters()
101 101 self.dataOut.timezone = 300
102 102 self.counter_records = 0
103 103 self.flagNoMoreFiles = 0
104 104 self.isConfig = False
105 105 self.filename = None
106 106 self.status_value = 0
107 self.datatime = datetime.datetime(1900,1,1)
107 self.datatime = datetime.datetime(1900, 1, 1)
108 108 self.filefmt = "*********%Y%m%d******"
109 109
110 110 def setup(self, **kwargs):
111 111
112 112 self.set_kwargs(**kwargs)
113 113
114 114 if self.path is None:
115 115 raise ValueError("The path is not valid")
116 116
117 117 if self.online:
118 118 log.log("Searching files in online mode...", self.name)
119 119
120 120 for nTries in range(self.nTries):
121 121 fullpath = self.searchFilesOnLine(self.path, self.startDate,
122 self.endDate, self.expLabel, self.ext, self.walk,
122 self.endDate, self.expLabel, self.ext, self.walk,
123 123 self.filefmt, self.folderfmt)
124 124 try:
125 125 fullpath = next(fullpath)
126 126 except:
127 127 fullpath = None
128 128
129 129 if fullpath:
130 130 self.fileSize = os.path.getsize(fullpath)
131 131 self.filename = fullpath
132 132 self.flagIsNewFile = 1
133 133 if self.fp != None:
134 134 self.fp.close()
135 135 self.fp = self.open_file(fullpath, self.open_mode)
136 136 self.flagNoMoreFiles = 0
137 137 break
138 138
139 139 log.warning(
140 140 'Waiting {} sec for a valid file in {}: try {} ...'.format(
141 self.delay, self.path, nTries + 1),
141 self.delay, self.path, nTries + 1),
142 142 self.name)
143 143 time.sleep(self.delay)
144 144
145 145 if not(fullpath):
146 146 raise schainpy.admin.SchainError(
147 147 'There isn\'t any valid file in {}'.format(self.path))
148 148 self.readFirstHeader()
149 149 else:
150 150 log.log("Searching files in {}".format(self.path), self.name)
151 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
151 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
152 152 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
153 153 self.setNextFile()
154 154
155 155 def checkForRealPath(self, nextFile, nextDay):
156 156 '''
157 157 '''
158 158
159 159 dt = self.datatime + datetime.timedelta(1)
160 160 filename = '{}.{}{}'.format(self.siteFile, dt.strftime('%Y%m%d'), self.ext)
161 161 fullfilename = os.path.join(self.path, filename)
162 162 if os.path.exists(fullfilename):
163 163 return fullfilename, filename
164 164 return None, filename
165 165
166 166
167 167 def readFirstHeader(self):
168 168 '''
169 169 '''
170 170
171 171 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
172 172 self.siteFile = self.filename.split('/')[-1].split('.')[0]
173 173 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
174 174 self.nrecords = self.header_file['nrec'][0]
175 175 self.counter_records = 0
176 176 self.flagIsNewFile = 0
177 177 self.fileIndex += 1
178 178
179 179 def readNextBlock(self):
180 180
181 181 while True:
182 182 if not self.online and self.counter_records == self.nrecords:
183 183 self.flagIsNewFile = 1
184 184 if not self.setNextFile():
185 185 return 0
186 186 try:
187 187 pointer = self.fp.tell()
188 188 self.readBlock()
189 189 except:
190 190 if self.online and self.waitDataBlock(pointer, 38512) == 1:
191 191 continue
192 192 else:
193 193 if not self.setNextFile():
194 194 return 0
195 195
196 196 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
197 197 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
198 198 log.warning(
199 199 'Reading Record No. {}/{} -> {} [Skipping]'.format(
200 200 self.counter_records,
201 201 self.nrecords,
202 202 self.datatime.ctime()),
203 203 'BLTRParamReader')
204 204 continue
205 205 break
206 206
207 207 log.log('Reading Record No. {} -> {}'.format(
208 208 self.counter_records,
209 209 self.datatime.ctime()), 'BLTRParamReader')
210 210
211 211 return 1
212 212
213 213 def readBlock(self):
214 214
215 215 pointer = self.fp.tell()
216 216 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
217 217 self.nchannels = int(header_rec['nchan'][0] / 2)
218 218 self.kchan = header_rec['nrxs'][0]
219 219 self.nmodes = header_rec['nmodes'][0]
220 220 self.nranges = header_rec['nranges'][0]
221 221 self.fp.seek(pointer)
222 222 self.height = numpy.empty((self.nmodes, self.nranges))
223 223 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
224 224 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
225 225 self.flagDiscontinuousBlock = 0
226 226
227 227 for mode in range(self.nmodes):
228 228 self.readHeader()
229 229 data = self.readData()
230 230 self.height[mode] = (data[0] - self.correction) / 1000.
231 231 self.buffer[mode] = data[1]
232 232 self.snr[mode] = data[2]
233 233
234 234 self.counter_records = self.counter_records + self.nmodes
235 235
236 236 return
237 237
238 238 def readHeader(self):
239 239 '''
240 240 RecordHeader of BLTR rawdata file
241 241 '''
242 242
243 243 header_structure = numpy.dtype(
244 244 REC_HEADER_STRUCTURE.descr + [
245 245 ('antenna_coord', 'f4', (2, int(self.nchannels))),
246 246 ('rx_gains', 'u4', (int(self.nchannels),)),
247 247 ('rx_analysis', 'u4', (int(self.nchannels),))
248 248 ]
249 249 )
250 250
251 251 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
252 252 self.lat = self.header_rec['lat'][0]
253 253 self.lon = self.header_rec['lon'][0]
254 254 self.delta = self.header_rec['delta_r'][0]
255 255 self.correction = self.header_rec['dmode_rngcorr'][0]
256 256 self.imode = self.header_rec['dmode_index'][0]
257 257 self.antenna = self.header_rec['antenna_coord']
258 258 self.rx_gains = self.header_rec['rx_gains']
259 259 self.time = self.header_rec['time'][0]
260 260 dt = datetime.datetime.utcfromtimestamp(self.time)
261 if dt.date()>self.datatime.date():
261 if dt.date() > self.datatime.date():
262 262 self.flagDiscontinuousBlock = 1
263 263 self.datatime = dt
264 264
265 265 def readData(self):
266 266 '''
267 267 Reading and filtering data block record of BLTR rawdata file,
268 268 filtering is according to status_value.
269 269
270 270 Input:
271 271 status_value - Array data is set to NAN for values that are not
272 272 equal to status_value
273 273
274 274 '''
275 275 self.nchannels = int(self.nchannels)
276 276
277 277 data_structure = numpy.dtype(
278 278 DATA_STRUCTURE.descr + [
279 279 ('rx_saturation', 'u4', (self.nchannels,)),
280 280 ('chan_offset', 'u4', (2 * self.nchannels,)),
281 281 ('rx_amp', 'u4', (self.nchannels,)),
282 282 ('rx_snr', 'f4', (self.nchannels,)),
283 283 ('cross_snr', 'f4', (self.kchan,)),
284 284 ('sea_power_relative', 'f4', (self.kchan,))]
285 285 )
286 286
287 287 data = numpy.fromfile(self.fp, data_structure, self.nranges)
288 288
289 289 height = data['range']
290 290 winds = numpy.array(
291 291 (data['zonal'], data['meridional'], data['vertical']))
292 292 snr = data['rx_snr'].T
293 293
294 294 winds[numpy.where(winds == -9999.)] = numpy.nan
295 295 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
296 296 snr[numpy.where(snr == -9999.)] = numpy.nan
297 297 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
298 298 snr = numpy.power(10, snr / 10)
299 299
300 300 return height, winds, snr
301 301
302 302 def set_output(self):
303 303 '''
304 304 Storing data from databuffer to dataOut object
305 305 '''
306 306
307 307 self.dataOut.data_snr = self.snr
308 308 self.dataOut.height = self.height
309 309 self.dataOut.data = self.buffer
310 310 self.dataOut.utctimeInit = self.time
311 311 self.dataOut.utctime = self.dataOut.utctimeInit
312 312 self.dataOut.useLocalTime = False
313 313 self.dataOut.paramInterval = 157
314 314 self.dataOut.site = self.siteFile
315 315 self.dataOut.nrecords = self.nrecords / self.nmodes
316 316 self.dataOut.lat = self.lat
317 317 self.dataOut.lon = self.lon
318 318 self.dataOut.channelList = list(range(self.nchannels))
319 319 self.dataOut.kchan = self.kchan
320 320 self.dataOut.delta = self.delta
321 321 self.dataOut.correction = self.correction
322 322 self.dataOut.nmodes = self.nmodes
323 323 self.dataOut.imode = self.imode
324 324 self.dataOut.antenna = self.antenna
325 325 self.dataOut.rx_gains = self.rx_gains
326 326 self.dataOut.flagNoData = False
327 327 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
328 328
329 329 def getData(self):
330 330 '''
331 331 Storing data from databuffer to dataOut object
332 332 '''
333 333 if self.flagNoMoreFiles:
334 334 self.dataOut.flagNoData = True
335 335 return 0
336 336
337 337 if not self.readNextBlock():
338 338 self.dataOut.flagNoData = True
339 339 return 0
340 340
341 341 self.set_output()
342 342
343 343 return 1
344 344
345 345 def run(self, **kwargs):
346 346 '''
347 347 '''
348 348
349 349 if not(self.isConfig):
350 350 self.setup(**kwargs)
351 351 self.isConfig = True
352 352
353 353 self.getData()
354 354
355 return No newline at end of file
355 return
Show file before | Show file after | Hide comments
@@ -1,453 +1,453
1 1 import os
2 2 import sys
3 3 import glob
4 4 import numpy
5 5
6 6
7 7 SPEED_OF_LIGHT = 299792458
8 8 SPEED_OF_LIGHT = 3e8
9 9
10 10 from .utils import folder_in_range
11 11
12 12 import schainpy.admin
13 13 from schainpy.model.data.jrodata import Spectra
14 14 from schainpy.model.proc.jroproc_base import ProcessingUnit
15 15 from schainpy.utils import log
16 16
17 17
18 18 def pol2cart(rho, phi):
19 19 x = rho * numpy.cos(phi)
20 20 y = rho * numpy.sin(phi)
21 21 return(x, y)
22 22
23 23 FILE_STRUCTURE = numpy.dtype([ # HEADER 48bytes
24 24 ('FileMgcNumber', '<u4'), # 0x23020100
25 25 ('nFDTdataRecors', '<u4'),
26 26 ('OffsetStartHeader', '<u4'),
27 27 ('RadarUnitId', '<u4'),
28 28 ('SiteName', 'S32'), # Null terminated
29 29 ])
30 30
31 31
32 32 class FileHeaderBLTR():
33 33
34 34 def __init__(self, fo):
35 35
36 36 self.fo = fo
37 37 self.size = 48
38 38 self.read()
39 39
40 40 def read(self):
41 41
42 42 header = numpy.fromfile(self.fo, FILE_STRUCTURE, 1)
43 43 self.FileMgcNumber = hex(header['FileMgcNumber'][0])
44 44 self.nFDTdataRecors = int(header['nFDTdataRecors'][0])
45 45 self.RadarUnitId = int(header['RadarUnitId'][0])
46 46 self.OffsetStartHeader = int(header['OffsetStartHeader'][0])
47 47 self.SiteName = header['SiteName'][0]
48 48
49 49 def write(self, fp):
50 50
51 51 headerTuple = (self.FileMgcNumber,
52 52 self.nFDTdataRecors,
53 53 self.RadarUnitId,
54 54 self.SiteName,
55 55 self.size)
56 56
57 57 header = numpy.array(headerTuple, FILE_STRUCTURE)
58 58 header.tofile(fp)
59 59 ''' ndarray.tofile(fid, sep, format) Write array to a file as text or binary (default).
60 60
61 61 fid : file or str
62 62 An open file object, or a string containing a filename.
63 63
64 64 sep : str
65 65 Separator between array items for text output. If "" (empty), a binary file is written,
66 66 equivalent to file.write(a.tobytes()).
67 67
68 68 format : str
69 69 Format string for text file output. Each entry in the array is formatted to text by
70 70 first converting it to the closest Python type, and then using "format" % item.
71 71
72 72 '''
73 73
74 74 return 1
75 75
76 76
77 77 RECORD_STRUCTURE = numpy.dtype([ # RECORD HEADER 180+20N bytes
78 78 ('RecMgcNumber', '<u4'), # 0x23030001
79 79 ('RecCounter', '<u4'), # Record counter(0,1, ...)
80 80 # Offset to start of next record form start of this record
81 81 ('Off2StartNxtRec', '<u4'),
82 82 # Offset to start of data from start of this record
83 83 ('Off2StartData', '<u4'),
84 84 # Epoch time stamp of start of acquisition (seconds)
85 85 ('nUtime', '<i4'),
86 86 # Millisecond component of time stamp (0,...,999)
87 87 ('nMilisec', '<u4'),
88 88 # Experiment tag name (null terminated)
89 89 ('ExpTagName', 'S32'),
90 90 # Experiment comment (null terminated)
91 91 ('ExpComment', 'S32'),
92 92 # Site latitude (from GPS) in degrees (positive implies North)
93 93 ('SiteLatDegrees', '<f4'),
94 94 # Site longitude (from GPS) in degrees (positive implies East)
95 95 ('SiteLongDegrees', '<f4'),
96 96 # RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
97 97 ('RTCgpsStatus', '<u4'),
98 98 ('TransmitFrec', '<u4'), # Transmit frequency (Hz)
99 99 ('ReceiveFrec', '<u4'), # Receive frequency
100 100 # First local oscillator frequency (Hz)
101 101 ('FirstOsciFrec', '<u4'),
102 102 # (0="O", 1="E", 2="linear 1", 3="linear2")
103 103 ('Polarisation', '<u4'),
104 104 # Receiver filter settings (0,1,2,3)
105 105 ('ReceiverFiltSett', '<u4'),
106 106 # Number of modes in use (1 or 2)
107 107 ('nModesInUse', '<u4'),
108 108 # Dual Mode index number for these data (0 or 1)
109 109 ('DualModeIndex', '<u4'),
110 110 # Dual Mode range correction for these data (m)
111 111 ('DualModeRange', '<u4'),
112 112 # Number of digital channels acquired (2*N)
113 113 ('nDigChannels', '<u4'),
114 114 # Sampling resolution (meters)
115 115 ('SampResolution', '<u4'),
116 116 # Number of range gates sampled
117 117 ('nHeights', '<u4'),
118 118 # Start range of sampling (meters)
119 119 ('StartRangeSamp', '<u4'),
120 120 ('PRFhz', '<u4'), # PRF (Hz)
121 121 ('nCohInt', '<u4'), # Integrations
122 122 # Number of data points transformed
123 123 ('nProfiles', '<u4'),
124 124 # Number of receive beams stored in file (1 or N)
125 125 ('nChannels', '<u4'),
126 126 ('nIncohInt', '<u4'), # Number of spectral averages
127 127 # FFT windowing index (0 = no window)
128 128 ('FFTwindowingInd', '<u4'),
129 129 # Beam steer angle (azimuth) in degrees (clockwise from true North)
130 130 ('BeamAngleAzim', '<f4'),
131 131 # Beam steer angle (zenith) in degrees (0=> vertical)
132 132 ('BeamAngleZen', '<f4'),
133 133 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
134 134 ('AntennaCoord0', '<f4'),
135 135 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
136 136 ('AntennaAngl0', '<f4'),
137 137 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
138 138 ('AntennaCoord1', '<f4'),
139 139 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
140 140 ('AntennaAngl1', '<f4'),
141 141 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
142 142 ('AntennaCoord2', '<f4'),
143 143 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
144 144 ('AntennaAngl2', '<f4'),
145 145 # Receiver phase calibration (degrees) - N values
146 146 ('RecPhaseCalibr0', '<f4'),
147 147 # Receiver phase calibration (degrees) - N values
148 148 ('RecPhaseCalibr1', '<f4'),
149 149 # Receiver phase calibration (degrees) - N values
150 150 ('RecPhaseCalibr2', '<f4'),
151 151 # Receiver amplitude calibration (ratio relative to receiver one) - N values
152 152 ('RecAmpCalibr0', '<f4'),
153 153 # Receiver amplitude calibration (ratio relative to receiver one) - N values
154 154 ('RecAmpCalibr1', '<f4'),
155 155 # Receiver amplitude calibration (ratio relative to receiver one) - N values
156 156 ('RecAmpCalibr2', '<f4'),
157 157 # Receiver gains in dB - N values
158 158 ('ReceiverGaindB0', '<i4'),
159 159 # Receiver gains in dB - N values
160 160 ('ReceiverGaindB1', '<i4'),
161 161 # Receiver gains in dB - N values
162 162 ('ReceiverGaindB2', '<i4'),
163 163 ])
164 164
165 165
166 166 class RecordHeaderBLTR():
167 167
168 168 def __init__(self, fo):
169 169
170 170 self.fo = fo
171 171 self.OffsetStartHeader = 48
172 172 self.Off2StartNxtRec = 811248
173 173
174 174 def read(self, block):
175 175 OffRHeader = self.OffsetStartHeader + block * self.Off2StartNxtRec
176 176 self.fo.seek(OffRHeader, os.SEEK_SET)
177 177 header = numpy.fromfile(self.fo, RECORD_STRUCTURE, 1)
178 178 self.RecMgcNumber = hex(header['RecMgcNumber'][0]) # 0x23030001
179 179 self.RecCounter = int(header['RecCounter'][0])
180 180 self.Off2StartNxtRec = int(header['Off2StartNxtRec'][0])
181 181 self.Off2StartData = int(header['Off2StartData'][0])
182 182 self.nUtime = header['nUtime'][0]
183 183 self.nMilisec = header['nMilisec'][0]
184 self.ExpTagName = '' # str(header['ExpTagName'][0])
185 self.ExpComment = '' # str(header['ExpComment'][0])
184 self.ExpTagName = '' # str(header['ExpTagName'][0])
185 self.ExpComment = '' # str(header['ExpComment'][0])
186 186 self.SiteLatDegrees = header['SiteLatDegrees'][0]
187 187 self.SiteLongDegrees = header['SiteLongDegrees'][0]
188 188 self.RTCgpsStatus = header['RTCgpsStatus'][0]
189 189 self.TransmitFrec = header['TransmitFrec'][0]
190 190 self.ReceiveFrec = header['ReceiveFrec'][0]
191 191 self.FirstOsciFrec = header['FirstOsciFrec'][0]
192 192 self.Polarisation = header['Polarisation'][0]
193 193 self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
194 194 self.nModesInUse = header['nModesInUse'][0]
195 195 self.DualModeIndex = header['DualModeIndex'][0]
196 196 self.DualModeRange = header['DualModeRange'][0]
197 197 self.nDigChannels = header['nDigChannels'][0]
198 198 self.SampResolution = header['SampResolution'][0]
199 199 self.nHeights = header['nHeights'][0]
200 200 self.StartRangeSamp = header['StartRangeSamp'][0]
201 201 self.PRFhz = header['PRFhz'][0]
202 202 self.nCohInt = header['nCohInt'][0]
203 203 self.nProfiles = header['nProfiles'][0]
204 204 self.nChannels = header['nChannels'][0]
205 205 self.nIncohInt = header['nIncohInt'][0]
206 206 self.FFTwindowingInd = header['FFTwindowingInd'][0]
207 207 self.BeamAngleAzim = header['BeamAngleAzim'][0]
208 208 self.BeamAngleZen = header['BeamAngleZen'][0]
209 209 self.AntennaCoord0 = header['AntennaCoord0'][0]
210 210 self.AntennaAngl0 = header['AntennaAngl0'][0]
211 211 self.AntennaCoord1 = header['AntennaCoord1'][0]
212 212 self.AntennaAngl1 = header['AntennaAngl1'][0]
213 213 self.AntennaCoord2 = header['AntennaCoord2'][0]
214 214 self.AntennaAngl2 = header['AntennaAngl2'][0]
215 215 self.RecPhaseCalibr0 = header['RecPhaseCalibr0'][0]
216 216 self.RecPhaseCalibr1 = header['RecPhaseCalibr1'][0]
217 217 self.RecPhaseCalibr2 = header['RecPhaseCalibr2'][0]
218 218 self.RecAmpCalibr0 = header['RecAmpCalibr0'][0]
219 219 self.RecAmpCalibr1 = header['RecAmpCalibr1'][0]
220 220 self.RecAmpCalibr2 = header['RecAmpCalibr2'][0]
221 221 self.ReceiverGaindB0 = header['ReceiverGaindB0'][0]
222 222 self.ReceiverGaindB1 = header['ReceiverGaindB1'][0]
223 223 self.ReceiverGaindB2 = header['ReceiverGaindB2'][0]
224 224 self.ipp = 0.5 * (SPEED_OF_LIGHT / self.PRFhz)
225 225 self.RHsize = 180 + 20 * self.nChannels
226 226 self.Datasize = self.nProfiles * self.nChannels * self.nHeights * 2 * 4
227 227 endFp = self.OffsetStartHeader + self.RecCounter * self.Off2StartNxtRec
228 228
229 229
230 230 if OffRHeader > endFp:
231 231 sys.stderr.write(
232 232 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp)
233 233 return 0
234 234
235 235 if OffRHeader < endFp:
236 236 sys.stderr.write(
237 237 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp)
238 238 return 0
239 239
240 240 return 1
241 241
242 242
243 243 class BLTRSpectraReader (ProcessingUnit):
244 244
245 245 def __init__(self):
246 246
247 247 ProcessingUnit.__init__(self)
248 248
249 249 self.ext = ".fdt"
250 250 self.optchar = "P"
251 251 self.fpFile = None
252 252 self.fp = None
253 253 self.BlockCounter = 0
254 254 self.fileSizeByHeader = None
255 255 self.filenameList = []
256 256 self.fileSelector = 0
257 257 self.Off2StartNxtRec = 0
258 258 self.RecCounter = 0
259 259 self.flagNoMoreFiles = 0
260 260 self.data_spc = None
261 261 self.data_cspc = None
262 262 self.path = None
263 263 self.OffsetStartHeader = 0
264 264 self.Off2StartData = 0
265 265 self.ipp = 0
266 266 self.nFDTdataRecors = 0
267 267 self.blocksize = 0
268 268 self.dataOut = Spectra()
269 269 self.dataOut.flagNoData = False
270 270
271 271 def search_files(self):
272 272 '''
273 273 Function that indicates the number of .fdt files that exist in the folder to be read.
274 274 It also creates an organized list with the names of the files to read.
275 275 '''
276 276
277 277 files = glob.glob(os.path.join(self.path, '*{}'.format(self.ext)))
278 278 files = sorted(files)
279 279 for f in files:
280 280 filename = f.split('/')[-1]
281 281 if folder_in_range(filename.split('.')[1], self.startDate, self.endDate, '%Y%m%d'):
282 282 self.filenameList.append(f)
283 283
284 284 def run(self, **kwargs):
285 285 '''
286 286 This method will be the one that will initiate the data entry, will be called constantly.
287 287 You should first verify that your Setup () is set up and then continue to acquire
288 288 the data to be processed with getData ().
289 289 '''
290 290 if not self.isConfig:
291 291 self.setup(**kwargs)
292 292 self.isConfig = True
293 293
294 294 self.getData()
295 295
296 def setup(self,
296 def setup(self,
297 297 path=None,
298 298 startDate=None,
299 299 endDate=None,
300 300 startTime=None,
301 301 endTime=None,
302 302 walk=True,
303 303 code=None,
304 304 online=False,
305 305 mode=None,
306 306 **kwargs):
307 307
308 308 self.isConfig = True
309 309
310 310 self.path = path
311 311 self.startDate = startDate
312 312 self.endDate = endDate
313 313 self.startTime = startTime
314 314 self.endTime = endTime
315 315 self.walk = walk
316 316 self.mode = int(mode)
317 317 self.search_files()
318 318 if self.filenameList:
319 319 self.readFile()
320 320
321 321 def getData(self):
322 322 '''
323 323 Before starting this function, you should check that there is still an unread file,
324 324 If there are still blocks to read or if the data block is empty.
325 325
326 326 You should call the file "read".
327 327
328 328 '''
329 329
330 330 if self.flagNoMoreFiles:
331 331 self.dataOut.flagNoData = True
332 332 raise schainpy.admin.SchainError('No more files')
333 333
334 334 self.readBlock()
335 335
336 336 def readFile(self):
337 337 '''
338 338 You must indicate if you are reading in Online or Offline mode and load the
339 339 The parameters for this file reading mode.
340 340
341 341 Then you must do 2 actions:
342 342
343 343 1. Get the BLTR FileHeader.
344 344 2. Start reading the first block.
345 345 '''
346 346
347 347 if self.fileSelector < len(self.filenameList):
348 348 log.success('Opening file: {}'.format(self.filenameList[self.fileSelector]), self.name)
349 349 self.fp = open(self.filenameList[self.fileSelector])
350 350 self.fheader = FileHeaderBLTR(self.fp)
351 351 self.rheader = RecordHeaderBLTR(self.fp)
352 352 self.nFDTdataRecors = self.fheader.nFDTdataRecors
353 353 self.fileSelector += 1
354 354 self.BlockCounter = 0
355 355 return 1
356 356 else:
357 357 self.flagNoMoreFiles = True
358 358 self.dataOut.flagNoData = True
359 359 return 0
360 360
361 361 def readBlock(self):
362 362 '''
363 363 It should be checked if the block has data, if it is not passed to the next file.
364 364
365 365 Then the following is done:
366 366
367 367 1. Read the RecordHeader
368 368 2. Fill the buffer with the current block number.
369 369
370 370 '''
371 371
372 372 if self.BlockCounter == self.nFDTdataRecors:
373 373 if not self.readFile():
374 374 return
375 375
376 376 if self.mode == 1:
377 self.rheader.read(self.BlockCounter+1)
377 self.rheader.read(self.BlockCounter + 1)
378 378 elif self.mode == 0:
379 379 self.rheader.read(self.BlockCounter)
380 380
381 381 self.RecCounter = self.rheader.RecCounter
382 382 self.OffsetStartHeader = self.rheader.OffsetStartHeader
383 383 self.Off2StartNxtRec = self.rheader.Off2StartNxtRec
384 384 self.Off2StartData = self.rheader.Off2StartData
385 385 self.nProfiles = self.rheader.nProfiles
386 386 self.nChannels = self.rheader.nChannels
387 387 self.nHeights = self.rheader.nHeights
388 388 self.frequency = self.rheader.TransmitFrec
389 389 self.DualModeIndex = self.rheader.DualModeIndex
390 390 self.pairsList = [(0, 1), (0, 2), (1, 2)]
391 391 self.dataOut.pairsList = self.pairsList
392 392 self.nRdPairs = len(self.dataOut.pairsList)
393 393 self.dataOut.nRdPairs = self.nRdPairs
394 394 self.dataOut.heightList = (self.rheader.StartRangeSamp + numpy.arange(self.nHeights) * self.rheader.SampResolution) / 1000.
395 395 self.dataOut.channelList = range(self.nChannels)
396 self.dataOut.nProfiles=self.rheader.nProfiles
397 self.dataOut.nIncohInt=self.rheader.nIncohInt
398 self.dataOut.nCohInt=self.rheader.nCohInt
399 self.dataOut.ippSeconds= 1/float(self.rheader.PRFhz)
400 self.dataOut.PRF=self.rheader.PRFhz
401 self.dataOut.nFFTPoints=self.rheader.nProfiles
402 self.dataOut.utctime = self.rheader.nUtime + self.rheader.nMilisec/1000.
396 self.dataOut.nProfiles = self.rheader.nProfiles
397 self.dataOut.nIncohInt = self.rheader.nIncohInt
398 self.dataOut.nCohInt = self.rheader.nCohInt
399 self.dataOut.ippSeconds = 1 / float(self.rheader.PRFhz)
400 self.dataOut.PRF = self.rheader.PRFhz
401 self.dataOut.nFFTPoints = self.rheader.nProfiles
402 self.dataOut.utctime = self.rheader.nUtime + self.rheader.nMilisec / 1000.
403 403 self.dataOut.timeZone = 0
404 404 self.dataOut.useLocalTime = False
405 405 self.dataOut.nmodes = 2
406 406 log.log('Reading block {} - {}'.format(self.BlockCounter, self.dataOut.datatime), self.name)
407 407 OffDATA = self.OffsetStartHeader + self.RecCounter * \
408 408 self.Off2StartNxtRec + self.Off2StartData
409 409 self.fp.seek(OffDATA, os.SEEK_SET)
410 410
411 self.data_fft = numpy.fromfile(self.fp, [('complex','<c8')], self.nProfiles*self.nChannels*self.nHeights )
411 self.data_fft = numpy.fromfile(self.fp, [('complex', '<c8')], self.nProfiles * self.nChannels * self.nHeights)
412 412 self.data_fft = self.data_fft.astype(numpy.dtype('complex'))
413 self.data_block = numpy.reshape(self.data_fft,(self.nHeights, self.nChannels, self.nProfiles))
414 self.data_block = numpy.transpose(self.data_block, (1,2,0))
413 self.data_block = numpy.reshape(self.data_fft, (self.nHeights, self.nChannels, self.nProfiles))
414 self.data_block = numpy.transpose(self.data_block, (1, 2, 0))
415 415 copy = self.data_block.copy()
416 416 spc = copy * numpy.conjugate(copy)
417 417 self.data_spc = numpy.absolute(spc) # valor absoluto o magnitud
418 418
419 419 cspc = self.data_block.copy()
420 420 self.data_cspc = self.data_block.copy()
421 421
422 422 for i in range(self.nRdPairs):
423 423
424 424 chan_index0 = self.dataOut.pairsList[i][0]
425 425 chan_index1 = self.dataOut.pairsList[i][1]
426 426
427 427 self.data_cspc[i, :, :] = cspc[chan_index0, :, :] * numpy.conjugate(cspc[chan_index1, :, :])
428 428
429 429 '''Getting Eij and Nij'''
430 430 (AntennaX0, AntennaY0) = pol2cart(
431 431 self.rheader.AntennaCoord0, self.rheader.AntennaAngl0 * numpy.pi / 180)
432 432 (AntennaX1, AntennaY1) = pol2cart(
433 433 self.rheader.AntennaCoord1, self.rheader.AntennaAngl1 * numpy.pi / 180)
434 434 (AntennaX2, AntennaY2) = pol2cart(
435 435 self.rheader.AntennaCoord2, self.rheader.AntennaAngl2 * numpy.pi / 180)
436 436
437 437 E01 = AntennaX0 - AntennaX1
438 438 N01 = AntennaY0 - AntennaY1
439 439
440 440 E02 = AntennaX0 - AntennaX2
441 441 N02 = AntennaY0 - AntennaY2
442 442
443 443 E12 = AntennaX1 - AntennaX2
444 444 N12 = AntennaY1 - AntennaY2
445 445
446 446 self.ChanDist = numpy.array(
447 447 [[E01, N01], [E02, N02], [E12, N12]])
448 448
449 449 self.dataOut.ChanDist = self.ChanDist
450 450
451 451 self.BlockCounter += 2
452 452 self.dataOut.data_spc = self.data_spc
453 self.dataOut.data_cspc =self.data_cspc
453 self.dataOut.data_cspc = self.data_cspc
Show file before | Show file after | Hide comments
@@ -1,691 +1,691
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4
5 5 import os
6 6 import sys
7 7 import glob
8 8 import fnmatch
9 9 import datetime
10 10 import time
11 11 import re
12 12 import h5py
13 13 import numpy
14 14
15 15 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
16 16 from schainpy.model.data.jroamisr import AMISR
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 class RadacHeader():
24 24 def __init__(self, fp):
25 25 header = 'Raw11/Data/RadacHeader'
26 self.beamCodeByPulse = fp.get(header+'/BeamCode')
26 self.beamCodeByPulse = fp.get(header + '/BeamCode')
27 27 self.beamCode = fp.get('Raw11/Data/Beamcodes')
28 self.code = fp.get(header+'/Code')
29 self.frameCount = fp.get(header+'/FrameCount')
30 self.modeGroup = fp.get(header+'/ModeGroup')
31 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
32 self.pulseCount = fp.get(header+'/PulseCount')
33 self.radacTime = fp.get(header+'/RadacTime')
34 self.timeCount = fp.get(header+'/TimeCount')
35 self.timeStatus = fp.get(header+'/TimeStatus')
36
37 self.nrecords = self.pulseCount.shape[0] #nblocks
38 self.npulses = self.pulseCount.shape[1] #nprofile
39 self.nsamples = self.nsamplesPulse[0,0] #ngates
28 self.code = fp.get(header + '/Code')
29 self.frameCount = fp.get(header + '/FrameCount')
30 self.modeGroup = fp.get(header + '/ModeGroup')
31 self.nsamplesPulse = fp.get(header + '/NSamplesPulse')
32 self.pulseCount = fp.get(header + '/PulseCount')
33 self.radacTime = fp.get(header + '/RadacTime')
34 self.timeCount = fp.get(header + '/TimeCount')
35 self.timeStatus = fp.get(header + '/TimeStatus')
36
37 self.nrecords = self.pulseCount.shape[0] # nblocks
38 self.npulses = self.pulseCount.shape[1] # nprofile
39 self.nsamples = self.nsamplesPulse[0, 0] # ngates
40 40 self.nbeams = self.beamCode.shape[1]
41 41
42 42
43 43 def getIndexRangeToPulse(self, idrecord=0):
44 #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
45 #startPulseCountId = indexToZero[0][0]
46 #endPulseCountId = startPulseCountId - 1
47 #range1 = numpy.arange(startPulseCountId,self.npulses,1)
48 #range2 = numpy.arange(0,startPulseCountId,1)
49 #return range1, range2
44 # indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
45 # startPulseCountId = indexToZero[0][0]
46 # endPulseCountId = startPulseCountId - 1
47 # range1 = numpy.arange(startPulseCountId,self.npulses,1)
48 # range2 = numpy.arange(0,startPulseCountId,1)
49 # return range1, range2
50 50 zero = 0
51 npulse = max(self.pulseCount[0,:]+1)-1
52 looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
51 npulse = max(self.pulseCount[0, :] + 1) - 1
52 looking_index = numpy.where(self.pulseCount.value[idrecord, :] == npulse)[0]
53 53 getLastIndex = looking_index[-1]
54 index_data = numpy.arange(0,getLastIndex+1,1)
55 index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
54 index_data = numpy.arange(0, getLastIndex + 1, 1)
55 index_buffer = numpy.arange(getLastIndex + 1, self.npulses, 1)
56 56 return index_data, index_buffer
57 57
58 58 class AMISRReader(ProcessingUnit):
59 59
60 60 path = None
61 61 startDate = None
62 62 endDate = None
63 63 startTime = None
64 64 endTime = None
65 65 walk = None
66 66 isConfig = False
67 67
68 68 def __init__(self):
69 69 self.set = None
70 70 self.subset = None
71 71 self.extension_file = '.h5'
72 72 self.dtc_str = 'dtc'
73 73 self.dtc_id = 0
74 74 self.status = True
75 75 self.isConfig = False
76 76 self.dirnameList = []
77 77 self.filenameList = []
78 78 self.fileIndex = None
79 79 self.flagNoMoreFiles = False
80 80 self.flagIsNewFile = 0
81 81 self.filename = ''
82 82 self.amisrFilePointer = None
83 83 self.radacHeaderObj = None
84 84 self.dataOut = self.__createObjByDefault()
85 85 self.datablock = None
86 86 self.rest_datablock = None
87 87 self.range = None
88 88 self.idrecord_count = 0
89 89 self.profileIndex = 0
90 90 self.index_amisr_sample = None
91 91 self.index_amisr_buffer = None
92 92 self.beamCodeByFrame = None
93 93 self.radacTimeByFrame = None
94 #atributos originales tal y como esta en el archivo de datos
94 # atributos originales tal y como esta en el archivo de datos
95 95 self.beamCodesFromFile = None
96 96 self.radacTimeFromFile = None
97 97 self.rangeFromFile = None
98 98 self.dataByFrame = None
99 99 self.dataset = None
100 100
101 101 self.beamCodeDict = {}
102 102 self.beamRangeDict = {}
103 103
104 #experiment cgf file
104 # experiment cgf file
105 105 self.npulsesint_fromfile = None
106 106 self.recordsperfile_fromfile = None
107 107 self.nbeamcodes_fromfile = None
108 108 self.ngates_fromfile = None
109 109 self.ippSeconds_fromfile = None
110 110 self.frequency_h5file = None
111 111
112 112
113 113 self.__firstFile = True
114 114 self.buffer_radactime = None
115 115
116 116 self.index4_schain_datablock = None
117 117 self.index4_buffer = None
118 118 self.schain_datablock = None
119 119 self.buffer = None
120 120 self.linear_pulseCount = None
121 121 self.npulseByFrame = None
122 122 self.profileIndex_offset = None
123 123 self.timezone = 'ut'
124 124
125 125 self.__waitForNewFile = 20
126 126 self.__filename_online = None
127 127
128 128 def __createObjByDefault(self):
129 129
130 130 dataObj = AMISR()
131 131
132 132 return dataObj
133 133
134 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
134 def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''):
135 135 self.path = path
136 136 self.startDate = startDate
137 137 self.endDate = endDate
138 138 self.startTime = startTime
139 139 self.endTime = endTime
140 140 self.walk = walk
141 141
142 142 def __checkPath(self):
143 143 if os.path.exists(self.path):
144 144 self.status = 1
145 145 else:
146 146 self.status = 0
147 print('Path:%s does not exists'%self.path)
147 print('Path:%s does not exists' % self.path)
148 148
149 149 return
150 150
151 151 def __selDates(self, amisr_dirname_format):
152 152 try:
153 153 year = int(amisr_dirname_format[0:4])
154 154 month = int(amisr_dirname_format[4:6])
155 155 dom = int(amisr_dirname_format[6:8])
156 thisDate = datetime.date(year,month,dom)
156 thisDate = datetime.date(year, month, dom)
157 157
158 if (thisDate>=self.startDate and thisDate <= self.endDate):
158 if (thisDate >= self.startDate and thisDate <= self.endDate):
159 159 return amisr_dirname_format
160 160 except:
161 161 return None
162 162
163 def __findDataForDates(self,online=False):
163 def __findDataForDates(self, online=False):
164 164
165 165
166 166
167 167 if not(self.status):
168 168 return None
169 169
170 170 pat = '\d+.\d+'
171 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
172 dirnameList = [x for x in dirnameList if x!=None]
171 dirnameList = [re.search(pat, x) for x in os.listdir(self.path)]
172 dirnameList = [x for x in dirnameList if x != None]
173 173 dirnameList = [x.string for x in dirnameList]
174 174 if not(online):
175 175 dirnameList = [self.__selDates(x) for x in dirnameList]
176 dirnameList = [x for x in dirnameList if x!=None]
177 if len(dirnameList)>0:
176 dirnameList = [x for x in dirnameList if x != None]
177 if len(dirnameList) > 0:
178 178 self.status = 1
179 179 self.dirnameList = dirnameList
180 180 self.dirnameList.sort()
181 181 else:
182 182 self.status = 0
183 183 return None
184 184
185 185 def __getTimeFromData(self):
186 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
187 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
186 startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime)
187 endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime)
188 188
189 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
189 print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader))
190 190 print('........................................')
191 191 filter_filenameList = []
192 192 self.filenameList.sort()
193 for i in range(len(self.filenameList)-1):
193 for i in range(len(self.filenameList) - 1):
194 194 filename = self.filenameList[i]
195 fp = h5py.File(filename,'r')
195 fp = h5py.File(filename, 'r')
196 196 time_str = fp.get('Time/RadacTimeString')
197 197
198 198 startDateTimeStr_File = time_str[0][0].split('.')[0]
199 199 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
200 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
200 startDateTime_File = datetime.datetime(junk.tm_year, junk.tm_mon, junk.tm_mday, junk.tm_hour, junk.tm_min, junk.tm_sec)
201 201
202 202 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
203 203 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
204 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
204 endDateTime_File = datetime.datetime(junk.tm_year, junk.tm_mon, junk.tm_mday, junk.tm_hour, junk.tm_min, junk.tm_sec)
205 205
206 206 fp.close()
207 207
208 208 if self.timezone == 'lt':
209 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
210 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
209 startDateTime_File = startDateTime_File - datetime.timedelta(minutes=300)
210 endDateTime_File = endDateTime_File - datetime.timedelta(minutes=300)
211 211
212 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
213 #self.filenameList.remove(filename)
212 if (endDateTime_File >= startDateTime_Reader and endDateTime_File < endDateTime_Reader):
213 # self.filenameList.remove(filename)
214 214 filter_filenameList.append(filename)
215 215
216 216 filter_filenameList.sort()
217 217 self.filenameList = filter_filenameList
218 218 return 1
219 219
220 220 def __filterByGlob1(self, dirName):
221 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
221 filter_files = glob.glob1(dirName, '*.*%s' % self.extension_file)
222 222 filterDict = {}
223 223 filterDict.setdefault(dirName)
224 224 filterDict[dirName] = filter_files
225 225 return filterDict
226 226
227 227 def __getFilenameList(self, fileListInKeys, dirList):
228 228 for value in fileListInKeys:
229 229 dirName = list(value.keys())[0]
230 230 for file in value[dirName]:
231 231 filename = os.path.join(dirName, file)
232 232 self.filenameList.append(filename)
233 233
234 234
235 235 def __selectDataForTimes(self, online=False):
236 #aun no esta implementado el filtro for tiempo
236 # aun no esta implementado el filtro for tiempo
237 237 if not(self.status):
238 238 return None
239 239
240 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
240 dirList = [os.path.join(self.path, x) for x in self.dirnameList]
241 241
242 242 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
243 243
244 244 self.__getFilenameList(fileListInKeys, dirList)
245 245 if not(online):
246 #filtro por tiempo
246 # filtro por tiempo
247 247 if not(self.all):
248 248 self.__getTimeFromData()
249 249
250 if len(self.filenameList)>0:
250 if len(self.filenameList) > 0:
251 251 self.status = 1
252 252 self.filenameList.sort()
253 253 else:
254 254 self.status = 0
255 255 return None
256 256
257 257 else:
258 #get the last file - 1
258 # get the last file - 1
259 259 self.filenameList = [self.filenameList[-2]]
260 260
261 261 new_dirnameList = []
262 262 for dirname in self.dirnameList:
263 263 junk = numpy.array([dirname in x for x in self.filenameList])
264 264 junk_sum = junk.sum()
265 265 if junk_sum > 0:
266 266 new_dirnameList.append(dirname)
267 267 self.dirnameList = new_dirnameList
268 268 return 1
269 269
270 270 def searchFilesOnLine(self,
271 271 path,
272 272 walk=True):
273 273
274 274 startDate = datetime.datetime.utcnow().date()
275 275 endDate = datetime.datetime.utcnow().date()
276 276
277 277 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
278 278
279 279 self.__checkPath()
280 280
281 281 self.__findDataForDates(online=True)
282 282
283 283 self.dirnameList = [self.dirnameList[-1]]
284 284
285 285 self.__selectDataForTimes(online=True)
286 286
287 287 return
288 288
289 289
290 290 def searchFilesOffLine(self,
291 291 path,
292 292 startDate,
293 293 endDate,
294 startTime=datetime.time(0,0,0),
295 endTime=datetime.time(23,59,59),
294 startTime=datetime.time(0, 0, 0),
295 endTime=datetime.time(23, 59, 59),
296 296 walk=True):
297 297
298 298 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
299 299
300 300 self.__checkPath()
301 301
302 302 self.__findDataForDates()
303 303
304 304 self.__selectDataForTimes()
305 305
306 306 for i in range(len(self.filenameList)):
307 print("%s" %(self.filenameList[i]))
307 print("%s" % (self.filenameList[i]))
308 308
309 309 return
310 310
311 311 def __setNextFileOffline(self):
312 312 idFile = self.fileIndex
313 313
314 314 while (True):
315 315 idFile += 1
316 316 if not(idFile < len(self.filenameList)):
317 317 self.flagNoMoreFiles = 1
318 318 print("No more Files")
319 319 return 0
320 320
321 321 filename = self.filenameList[idFile]
322 322
323 amisrFilePointer = h5py.File(filename,'r')
323 amisrFilePointer = h5py.File(filename, 'r')
324 324
325 325 break
326 326
327 327 self.flagIsNewFile = 1
328 328 self.fileIndex = idFile
329 329 self.filename = filename
330 330
331 331 self.amisrFilePointer = amisrFilePointer
332 332
333 print("Setting the file: %s"%self.filename)
333 print("Setting the file: %s" % self.filename)
334 334
335 335 return 1
336 336
337 337
338 338 def __setNextFileOnline(self):
339 339 filename = self.filenameList[0]
340 340 if self.__filename_online != None:
341 341 self.__selectDataForTimes(online=True)
342 342 filename = self.filenameList[0]
343 343 while self.__filename_online == filename:
344 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
344 print('waiting %d seconds to get a new file...' % (self.__waitForNewFile))
345 345 sleep(self.__waitForNewFile)
346 346 self.__selectDataForTimes(online=True)
347 347 filename = self.filenameList[0]
348 348
349 349 self.__filename_online = filename
350 350
351 self.amisrFilePointer = h5py.File(filename,'r')
351 self.amisrFilePointer = h5py.File(filename, 'r')
352 352 self.flagIsNewFile = 1
353 353 self.filename = filename
354 print("Setting the file: %s"%self.filename)
354 print("Setting the file: %s" % self.filename)
355 355 return 1
356 356
357 357
358 358 def __readHeader(self):
359 359 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
360 360
361 #update values from experiment cfg file
361 # update values from experiment cfg file
362 362 if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
363 363 self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
364 364 self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
365 365 self.radacHeaderObj.npulses = self.npulsesint_fromfile
366 366 self.radacHeaderObj.nsamples = self.ngates_fromfile
367 367
368 #looking index list for data
369 start_index = self.radacHeaderObj.pulseCount[0,:][0]
368 # looking index list for data
369 start_index = self.radacHeaderObj.pulseCount[0, :][0]
370 370 end_index = self.radacHeaderObj.npulses
371 371 range4data = list(range(start_index, end_index))
372 372 self.index4_schain_datablock = numpy.array(range4data)
373 373
374 374 buffer_start_index = 0
375 buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
375 buffer_end_index = self.radacHeaderObj.pulseCount[0, :][0]
376 376 range4buffer = list(range(buffer_start_index, buffer_end_index))
377 377 self.index4_buffer = numpy.array(range4buffer)
378 378
379 379 self.linear_pulseCount = numpy.array(range4data + range4buffer)
380 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
380 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0, :] + 1)
381 381
382 #get tuning frequency
383 frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
384 self.frequency_h5file = frequency_h5file_dataset[0,0]
382 # get tuning frequency
383 frequency_h5file_dataset = self.amisrFilePointer.get('Rx' + '/TuningFrequency')
384 self.frequency_h5file = frequency_h5file_dataset[0, 0]
385 385
386 386 self.flagIsNewFile = 1
387 387
388 388 def __getBeamCode(self):
389 389 self.beamCodeDict = {}
390 390 self.beamRangeDict = {}
391 391
392 392 beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
393 393
394 for i in range(len(self.radacHeaderObj.beamCode[0,:])):
394 for i in range(len(self.radacHeaderObj.beamCode[0, :])):
395 395 self.beamCodeDict.setdefault(i)
396 396 self.beamRangeDict.setdefault(i)
397 beamcodeValue = self.radacHeaderObj.beamCode[0,i]
398 beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
397 beamcodeValue = self.radacHeaderObj.beamCode[0, i]
398 beamcodeIndex = numpy.where(beamCodeMap[:, 0] == beamcodeValue)[0][0]
399 399 x = beamCodeMap[beamcodeIndex][1]
400 400 y = beamCodeMap[beamcodeIndex][2]
401 401 z = beamCodeMap[beamcodeIndex][3]
402 402 self.beamCodeDict[i] = [beamcodeValue, x, y, z]
403 403
404 just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
404 just4record0 = self.radacHeaderObj.beamCodeByPulse[0, :]
405 405
406 406 for i in range(len(list(self.beamCodeDict.values()))):
407 xx = numpy.where(just4record0==list(self.beamCodeDict.values())[i][0])
407 xx = numpy.where(just4record0 == list(self.beamCodeDict.values())[i][0])
408 408 indexPulseByBeam = self.linear_pulseCount[xx[0]]
409 409 self.beamRangeDict[i] = indexPulseByBeam
410 410
411 411 def __getExpParameters(self):
412 412 if not(self.status):
413 413 return None
414 414
415 415 experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
416 416
417 expFinder = glob.glob1(experimentCfgPath,'*.exp')
418 if len(expFinder)== 0:
417 expFinder = glob.glob1(experimentCfgPath, '*.exp')
418 if len(expFinder) == 0:
419 419 self.status = 0
420 420 return None
421 421
422 experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
422 experimentFilename = os.path.join(experimentCfgPath, expFinder[0])
423 423
424 424 f = open(experimentFilename)
425 425 lines = f.readlines()
426 426 f.close()
427 427
428 parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
428 parmsList = ['npulsesint*', 'recordsperfile*', 'nbeamcodes*', 'ngates*']
429 429 filterList = [fnmatch.filter(lines, x) for x in parmsList]
430 430
431 431
432 values = [re.sub(r'\D',"",x[0]) for x in filterList]
432 values = [re.sub(r'\D', "", x[0]) for x in filterList]
433 433
434 434 self.npulsesint_fromfile = int(values[0])
435 435 self.recordsperfile_fromfile = int(values[1])
436 436 self.nbeamcodes_fromfile = int(values[2])
437 437 self.ngates_fromfile = int(values[3])
438 438
439 439 tufileFinder = fnmatch.filter(lines, 'tufile=*')
440 440 tufile = tufileFinder[0].split('=')[1].split('\n')[0]
441 441 tufile = tufile.split('\r')[0]
442 tufilename = os.path.join(experimentCfgPath,tufile)
442 tufilename = os.path.join(experimentCfgPath, tufile)
443 443
444 444 f = open(tufilename)
445 445 lines = f.readlines()
446 446 f.close()
447 self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
447 self.ippSeconds_fromfile = float(lines[1].split()[2]) / 1E6
448 448
449 449
450 450 self.status = 1
451 451
452 452 def __setIdsAndArrays(self):
453 453 self.dataByFrame = self.__setDataByFrame()
454 454 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
455 455 self.readRanges()
456 456 self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
457 457 self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
458 458 if len(self.index_amisr_buffer) > 0:
459 459 self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
460 460
461 461
462 def __setNextFile(self,online=False):
462 def __setNextFile(self, online=False):
463 463
464 464 if not(online):
465 465 newFile = self.__setNextFileOffline()
466 466 else:
467 467 newFile = self.__setNextFileOnline()
468 468
469 469 if not(newFile):
470 470 return 0
471 471
472 472 self.__readHeader()
473 473
474 474 if self.__firstFile:
475 475 self.__setIdsAndArrays()
476 476 self.__firstFile = False
477 477
478 478 self.__getBeamCode()
479 479 self.readDataBlock()
480 480
481 481
482 def setup(self,path=None,
483 startDate=None,
484 endDate=None,
485 startTime=datetime.time(0,0,0),
486 endTime=datetime.time(23,59,59),
482 def setup(self, path=None,
483 startDate=None,
484 endDate=None,
485 startTime=datetime.time(0, 0, 0),
486 endTime=datetime.time(23, 59, 59),
487 487 walk=True,
488 488 timezone='ut',
489 489 all=0,
490 490 online=False):
491 491
492 492 self.timezone = timezone
493 493 self.all = all
494 494 self.online = online
495 495 if not(online):
496 #Busqueda de archivos offline
496 # Busqueda de archivos offline
497 497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 498 else:
499 499 self.searchFilesOnLine(path, walk)
500 500
501 501 if not(self.filenameList):
502 print("There is no files into the folder: %s"%(path))
502 print("There is no files into the folder: %s" % (path))
503 503
504 504 sys.exit(-1)
505 505
506 506 self.__getExpParameters()
507 507
508 508 self.fileIndex = -1
509 509
510 510 self.__setNextFile(online)
511 511
512 512 # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
513 513 # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
514 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
514 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0, :][0]
515 515 self.profileIndex = self.profileIndex_offset
516 516
517 517 def readRanges(self):
518 518 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
519 519
520 self.rangeFromFile = numpy.reshape(dataset.value,(-1))
520 self.rangeFromFile = numpy.reshape(dataset.value, (-1))
521 521 return self.rangeFromFile
522 522
523 523
524 def readRadacTime(self,idrecord, range1, range2):
524 def readRadacTime(self, idrecord, range1, range2):
525 525 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
526 526
527 527 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
528 #radacTimeByFrame = dataset[idrecord - 1,range1]
529 #radacTimeByFrame = dataset[idrecord,range2]
528 # radacTimeByFrame = dataset[idrecord - 1,range1]
529 # radacTimeByFrame = dataset[idrecord,range2]
530 530
531 531 return radacTimeByFrame
532 532
533 533 def readBeamCode(self, idrecord, range1, range2):
534 534 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
535 535 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
536 536 self.beamCodesFromFile = dataset.value
537 537
538 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
539 #beamcodeByFrame[range2] = dataset[idrecord, range2]
538 # beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
539 # beamcodeByFrame[range2] = dataset[idrecord, range2]
540 540 beamcodeByFrame[range1] = dataset[idrecord, range1]
541 541 beamcodeByFrame[range2] = dataset[idrecord, range2]
542 542
543 543 return beamcodeByFrame
544 544
545 545
546 546 def __setDataByFrame(self):
547 ndata = 2 # porque es complejo
547 ndata = 2 # porque es complejo
548 548 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
549 549 return dataByFrame
550 550
551 551 def __readDataSet(self):
552 552 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
553 553 return dataset
554 554
555 555 def __setDataBlock(self,):
556 real = self.dataByFrame[:,:,0] #asumo que 0 es real
557 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
558 datablock = real + imag*1j #armo el complejo
556 real = self.dataByFrame[:, :, 0] # asumo que 0 es real
557 imag = self.dataByFrame[:, :, 1] # asumo que 1 es imaginario
558 datablock = real + imag * 1j # armo el complejo
559 559 return datablock
560 560
561 def readSamples_version1(self,idrecord):
562 #estas tres primeras lineas solo se deben ejecutar una vez
561 def readSamples_version1(self, idrecord):
562 # estas tres primeras lineas solo se deben ejecutar una vez
563 563 if self.flagIsNewFile:
564 #reading dataset
564 # reading dataset
565 565 self.dataset = self.__readDataSet()
566 566 self.flagIsNewFile = 0
567 567
568 568 if idrecord == 0:
569 self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
569 self.dataByFrame[self.index4_schain_datablock, : , :] = self.dataset[0, self.index_amisr_sample, :, :]
570 570 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
571 571 datablock = self.__setDataBlock()
572 572 if len(self.index_amisr_buffer) > 0:
573 self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
573 self.buffer = self.dataset[0, self.index_amisr_buffer, :, :]
574 574 self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
575 575
576 576 return datablock
577 577 if len(self.index_amisr_buffer) > 0:
578 self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
578 self.dataByFrame[self.index4_buffer, :, :] = self.buffer.copy()
579 579 self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
580 self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
580 self.dataByFrame[self.index4_schain_datablock, :, :] = self.dataset[idrecord, self.index_amisr_sample, :, :]
581 581 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
582 582 datablock = self.__setDataBlock()
583 583 if len(self.index_amisr_buffer) > 0:
584 584 self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
585 585 self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
586 586
587 587 return datablock
588 588
589 589
590 def readSamples(self,idrecord):
590 def readSamples(self, idrecord):
591 591 if self.flagIsNewFile:
592 592 self.dataByFrame = self.__setDataByFrame()
593 593 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
594 594
595 #reading ranges
595 # reading ranges
596 596 self.readRanges()
597 #reading dataset
597 # reading dataset
598 598 self.dataset = self.__readDataSet()
599 599
600 600 self.flagIsNewFile = 0
601 601 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
602 602 self.dataByFrame = self.dataset[idrecord, :, :, :]
603 603 datablock = self.__setDataBlock()
604 604 return datablock
605 605
606 606
607 607 def readDataBlock(self):
608 608
609 609 self.datablock = self.readSamples_version1(self.idrecord_count)
610 #self.datablock = self.readSamples(self.idrecord_count)
611 #print 'record:', self.idrecord_count
610 # self.datablock = self.readSamples(self.idrecord_count)
611 # print 'record:', self.idrecord_count
612 612
613 613 self.idrecord_count += 1
614 614 self.profileIndex = 0
615 615
616 616 if self.idrecord_count >= self.radacHeaderObj.nrecords:
617 617 self.idrecord_count = 0
618 618 self.flagIsNewFile = 1
619 619
620 620 def readNextBlock(self):
621 621
622 622 self.readDataBlock()
623 623
624 624 if self.flagIsNewFile:
625 625 self.__setNextFile(self.online)
626 626 pass
627 627
628 628 def __hasNotDataInBuffer(self):
629 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
629 # self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
630 630 if self.profileIndex >= self.radacHeaderObj.npulses:
631 631 return 1
632 632 return 0
633 633
634 634 def printUTC(self):
635 635 print(self.dataOut.utctime)
636 636 print('')
637 637
638 638 def setObjProperties(self):
639 639
640 self.dataOut.heightList = self.rangeFromFile/1000.0 #km
640 self.dataOut.heightList = self.rangeFromFile / 1000.0 # km
641 641 self.dataOut.nProfiles = self.radacHeaderObj.npulses
642 642 self.dataOut.nRecords = self.radacHeaderObj.nrecords
643 643 self.dataOut.nBeams = self.radacHeaderObj.nbeams
644 644 self.dataOut.ippSeconds = self.ippSeconds_fromfile
645 645 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
646 646 self.dataOut.frequency = self.frequency_h5file
647 647 self.dataOut.npulseByFrame = self.npulseByFrame
648 648 self.dataOut.nBaud = None
649 649 self.dataOut.nCode = None
650 650 self.dataOut.code = None
651 651
652 652 self.dataOut.beamCodeDict = self.beamCodeDict
653 653 self.dataOut.beamRangeDict = self.beamRangeDict
654 654
655 655 if self.timezone == 'lt':
656 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
656 self.dataOut.timeZone = time.timezone / 60. # get the timezone in minutes
657 657 else:
658 self.dataOut.timeZone = 0 #by default time is UTC
658 self.dataOut.timeZone = 0 # by default time is UTC
659 659
660 660 def getData(self):
661 661
662 662 if self.flagNoMoreFiles:
663 663 self.dataOut.flagNoData = True
664 664 return 0
665 665
666 666 if self.__hasNotDataInBuffer():
667 667 self.readNextBlock()
668 668
669 669
670 if self.datablock is None: # setear esta condicion cuando no hayan datos por leers
670 if self.datablock is None: # setear esta condicion cuando no hayan datos por leers
671 671 self.dataOut.flagNoData = True
672 672 return 0
673 673
674 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
674 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex, :], (1, -1))
675 675
676 676 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
677 677 self.dataOut.profileIndex = self.profileIndex
678 678 self.dataOut.flagNoData = False
679 679
680 680 self.profileIndex += 1
681 681
682 682 return self.dataOut.data
683 683
684 684
685 685 def run(self, **kwargs):
686 686 if not(self.isConfig):
687 687 self.setup(**kwargs)
688 688 self.setObjProperties()
689 689 self.isConfig = True
690 690
691 self.getData() No newline at end of file
691 self.getData()
Show file before | Show file after | Hide comments
@@ -1,1575 +1,1575
1 1 """
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 """
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import zmq
16 16
17 17 from schainpy.model.proc.jroproc_base import Operation, MPDecorator
18 18 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
19 19 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
20 20 from schainpy.utils import log
21 21 import schainpy.admin
22 22
23 23 LOCALTIME = True
24 24 DT_DIRECTIVES = {
25 25 '%Y': 4,
26 26 '%y': 2,
27 27 '%m': 2,
28 28 '%d': 2,
29 29 '%j': 3,
30 30 '%H': 2,
31 31 '%M': 2,
32 32 '%S': 2,
33 33 '%f': 6
34 34 }
35 35
36 36
37 37 def isNumber(cad):
38 38 """
39 39 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
40 40
41 41 Excepciones:
42 42 Si un determinado string no puede ser convertido a numero
43 43 Input:
44 44 str, string al cual se le analiza para determinar si convertible a un numero o no
45 45
46 46 Return:
47 47 True : si el string es uno numerico
48 48 False : no es un string numerico
49 49 """
50 50 try:
51 51 float(cad)
52 52 return True
53 53 except:
54 54 return False
55 55
56 56
57 57 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
58 58 """
59 59 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
60 60
61 61 Inputs:
62 62 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
63 63
64 64 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
65 65 segundos contados desde 01/01/1970.
66 66 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
67 67 segundos contados desde 01/01/1970.
68 68
69 69 Return:
70 70 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
71 71 fecha especificado, de lo contrario retorna False.
72 72
73 73 Excepciones:
74 74 Si el archivo no existe o no puede ser abierto
75 75 Si la cabecera no puede ser leida.
76 76
77 77 """
78 78 basicHeaderObj = BasicHeader(LOCALTIME)
79 79
80 80 try:
81 81 fp = open(filename, 'rb')
82 82 except IOError:
83 83 print("The file %s can't be opened" % (filename))
84 84 return 0
85 85
86 86 sts = basicHeaderObj.read(fp)
87 87 fp.close()
88 88
89 89 if not(sts):
90 90 print("Skipping the file %s because it has not a valid header" % (filename))
91 91 return 0
92 92
93 93 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
94 94 return 0
95 95
96 96 return 1
97 97
98 98
99 99 def isTimeInRange(thisTime, startTime, endTime):
100 100 if endTime >= startTime:
101 101 if (thisTime < startTime) or (thisTime > endTime):
102 102 return 0
103 103 return 1
104 104 else:
105 105 if (thisTime < startTime) and (thisTime > endTime):
106 106 return 0
107 107 return 1
108 108
109 109
110 110 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
111 111 """
112 112 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
113 113
114 114 Inputs:
115 115 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
116 116
117 117 startDate : fecha inicial del rango seleccionado en formato datetime.date
118 118
119 119 endDate : fecha final del rango seleccionado en formato datetime.date
120 120
121 121 startTime : tiempo inicial del rango seleccionado en formato datetime.time
122 122
123 123 endTime : tiempo final del rango seleccionado en formato datetime.time
124 124
125 125 Return:
126 126 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
127 127 fecha especificado, de lo contrario retorna False.
128 128
129 129 Excepciones:
130 130 Si el archivo no existe o no puede ser abierto
131 131 Si la cabecera no puede ser leida.
132 132
133 133 """
134 134
135 135 try:
136 136 fp = open(filename, 'rb')
137 137 except IOError:
138 138 print("The file %s can't be opened" % (filename))
139 139 return None
140 140
141 141 firstBasicHeaderObj = BasicHeader(LOCALTIME)
142 142 systemHeaderObj = SystemHeader()
143 143 radarControllerHeaderObj = RadarControllerHeader()
144 144 processingHeaderObj = ProcessingHeader()
145 145
146 146 lastBasicHeaderObj = BasicHeader(LOCALTIME)
147 147
148 148 sts = firstBasicHeaderObj.read(fp)
149 149
150 150 if not(sts):
151 151 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
152 152 return None
153 153
154 154 if not systemHeaderObj.read(fp):
155 155 return None
156 156
157 157 if not radarControllerHeaderObj.read(fp):
158 158 return None
159 159
160 160 if not processingHeaderObj.read(fp):
161 161 return None
162 162
163 163 filesize = os.path.getsize(filename)
164 164
165 165 offset = processingHeaderObj.blockSize + 24 # header size
166 166
167 167 if filesize <= offset:
168 168 print("[Reading] %s: This file has not enough data" % filename)
169 169 return None
170 170
171 171 fp.seek(-offset, 2)
172 172
173 173 sts = lastBasicHeaderObj.read(fp)
174 174
175 175 fp.close()
176 176
177 177 thisDatetime = lastBasicHeaderObj.datatime
178 178 thisTime_last_block = thisDatetime.time()
179 179
180 180 thisDatetime = firstBasicHeaderObj.datatime
181 181 thisDate = thisDatetime.date()
182 182 thisTime_first_block = thisDatetime.time()
183 183
184 184 # General case
185 185 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
186 186 #-----------o----------------------------o-----------
187 187 # startTime endTime
188 188
189 189 if endTime >= startTime:
190 190 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
191 191 return None
192 192
193 193 return thisDatetime
194 194
195 195 # If endTime < startTime then endTime belongs to the next day
196 196
197 #<<<<<<<<<<<o o>>>>>>>>>>>
197 # <<<<<<<<<<<o o>>>>>>>>>>>
198 198 #-----------o----------------------------o-----------
199 199 # endTime startTime
200 200
201 201 if (thisDate == startDate) and (thisTime_last_block < startTime):
202 202 return None
203 203
204 204 if (thisDate == endDate) and (thisTime_first_block > endTime):
205 205 return None
206 206
207 207 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
208 208 return None
209 209
210 210 return thisDatetime
211 211
212 212
213 213 def isFolderInDateRange(folder, startDate=None, endDate=None):
214 214 """
215 215 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
216 216
217 217 Inputs:
218 218 folder : nombre completo del directorio.
219 219 Su formato deberia ser "/path_root/?YYYYDDD"
220 220
221 221 siendo:
222 222 YYYY : Anio (ejemplo 2015)
223 223 DDD : Dia del anio (ejemplo 305)
224 224
225 225 startDate : fecha inicial del rango seleccionado en formato datetime.date
226 226
227 227 endDate : fecha final del rango seleccionado en formato datetime.date
228 228
229 229 Return:
230 230 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
231 231 fecha especificado, de lo contrario retorna False.
232 232 Excepciones:
233 233 Si el directorio no tiene el formato adecuado
234 234 """
235 235
236 236 basename = os.path.basename(folder)
237 237
238 238 if not isRadarFolder(basename):
239 239 print("The folder %s has not the rigth format" % folder)
240 240 return 0
241 241
242 242 if startDate and endDate:
243 243 thisDate = getDateFromRadarFolder(basename)
244 244
245 245 if thisDate < startDate:
246 246 return 0
247 247
248 248 if thisDate > endDate:
249 249 return 0
250 250
251 251 return 1
252 252
253 253
254 254 def isFileInDateRange(filename, startDate=None, endDate=None):
255 255 """
256 256 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
257 257
258 258 Inputs:
259 259 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
260 260
261 261 Su formato deberia ser "?YYYYDDDsss"
262 262
263 263 siendo:
264 264 YYYY : Anio (ejemplo 2015)
265 265 DDD : Dia del anio (ejemplo 305)
266 266 sss : set
267 267
268 268 startDate : fecha inicial del rango seleccionado en formato datetime.date
269 269
270 270 endDate : fecha final del rango seleccionado en formato datetime.date
271 271
272 272 Return:
273 273 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
274 274 fecha especificado, de lo contrario retorna False.
275 275 Excepciones:
276 276 Si el archivo no tiene el formato adecuado
277 277 """
278 278
279 279 basename = os.path.basename(filename)
280 280
281 281 if not isRadarFile(basename):
282 282 print("The filename %s has not the rigth format" % filename)
283 283 return 0
284 284
285 285 if startDate and endDate:
286 286 thisDate = getDateFromRadarFile(basename)
287 287
288 288 if thisDate < startDate:
289 289 return 0
290 290
291 291 if thisDate > endDate:
292 292 return 0
293 293
294 294 return 1
295 295
296 296
297 297 def getFileFromSet(path, ext, set):
298 298 validFilelist = []
299 299 fileList = os.listdir(path)
300 300
301 301 # 0 1234 567 89A BCDE
302 302 # H YYYY DDD SSS .ext
303 303
304 304 for thisFile in fileList:
305 305 try:
306 306 year = int(thisFile[1:5])
307 307 doy = int(thisFile[5:8])
308 308 except:
309 309 continue
310 310
311 311 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
312 312 continue
313 313
314 314 validFilelist.append(thisFile)
315 315
316 316 myfile = fnmatch.filter(
317 317 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
318 318
319 319 if len(myfile) != 0:
320 320 return myfile[0]
321 321 else:
322 322 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
323 323 print('the filename %s does not exist' % filename)
324 324 print('...going to the last file: ')
325 325
326 326 if validFilelist:
327 327 validFilelist = sorted(validFilelist, key=str.lower)
328 328 return validFilelist[-1]
329 329
330 330 return None
331 331
332 332
333 333 def getlastFileFromPath(path, ext):
334 334 """
335 335 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
336 336 al final de la depuracion devuelve el ultimo file de la lista que quedo.
337 337
338 338 Input:
339 339 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
340 340 ext : extension de los files contenidos en una carpeta
341 341
342 342 Return:
343 343 El ultimo file de una determinada carpeta, no se considera el path.
344 344 """
345 345 validFilelist = []
346 346 fileList = os.listdir(path)
347 347
348 348 # 0 1234 567 89A BCDE
349 349 # H YYYY DDD SSS .ext
350 350
351 351 for thisFile in fileList:
352 352
353 353 year = thisFile[1:5]
354 354 if not isNumber(year):
355 355 continue
356 356
357 357 doy = thisFile[5:8]
358 358 if not isNumber(doy):
359 359 continue
360 360
361 361 year = int(year)
362 362 doy = int(doy)
363 363
364 364 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
365 365 continue
366 366
367 367 validFilelist.append(thisFile)
368 368
369 369 if validFilelist:
370 370 validFilelist = sorted(validFilelist, key=str.lower)
371 371 return validFilelist[-1]
372 372
373 373 return None
374 374
375 375
376 376 def isRadarFolder(folder):
377 377 try:
378 378 year = int(folder[1:5])
379 379 doy = int(folder[5:8])
380 380 except:
381 381 return 0
382 382
383 383 return 1
384 384
385 385
386 386 def isRadarFile(file):
387 387 try:
388 388 year = int(file[1:5])
389 389 doy = int(file[5:8])
390 390 set = int(file[8:11])
391 391 except:
392 392 return 0
393 393
394 394 return 1
395 395
396 396
397 397 def getDateFromRadarFile(file):
398 398 try:
399 399 year = int(file[1:5])
400 400 doy = int(file[5:8])
401 401 set = int(file[8:11])
402 402 except:
403 403 return None
404 404
405 405 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
406 406 return thisDate
407 407
408 408
409 409 def getDateFromRadarFolder(folder):
410 410 try:
411 411 year = int(folder[1:5])
412 412 doy = int(folder[5:8])
413 413 except:
414 414 return None
415 415
416 416 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
417 417 return thisDate
418 418
419 419 def parse_format(s, fmt):
420 420
421 421 for i in range(fmt.count('%')):
422 422 x = fmt.index('%')
423 d = DT_DIRECTIVES[fmt[x:x+2]]
424 fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
423 d = DT_DIRECTIVES[fmt[x:x + 2]]
424 fmt = fmt.replace(fmt[x:x + 2], s[x:x + d])
425 425 return fmt
426 426
427 427 class Reader(object):
428 428
429 429 c = 3E8
430 430 isConfig = False
431 431 dtype = None
432 432 pathList = []
433 433 filenameList = []
434 434 datetimeList = []
435 435 filename = None
436 436 ext = None
437 437 flagIsNewFile = 1
438 438 flagDiscontinuousBlock = 0
439 439 flagIsNewBlock = 0
440 440 flagNoMoreFiles = 0
441 441 fp = None
442 442 firstHeaderSize = 0
443 443 basicHeaderSize = 24
444 444 versionFile = 1103
445 445 fileSize = None
446 446 fileSizeByHeader = None
447 447 fileIndex = -1
448 448 profileIndex = None
449 449 blockIndex = 0
450 450 nTotalBlocks = 0
451 451 maxTimeStep = 30
452 452 lastUTTime = None
453 453 datablock = None
454 454 dataOut = None
455 455 getByBlock = False
456 456 path = None
457 457 startDate = None
458 458 endDate = None
459 459 startTime = datetime.time(0, 0, 0)
460 460 endTime = datetime.time(23, 59, 59)
461 461 set = None
462 462 expLabel = ""
463 463 online = False
464 464 delay = 60
465 465 nTries = 3 # quantity tries
466 466 nFiles = 3 # number of files for searching
467 467 walk = True
468 468 getblock = False
469 469 nTxs = 1
470 470 realtime = False
471 471 blocksize = 0
472 472 blocktime = None
473 473 warnings = True
474 474 verbose = True
475 475 server = None
476 476 format = None
477 477 oneDDict = None
478 478 twoDDict = None
479 479 independentParam = None
480 480 filefmt = None
481 481 folderfmt = None
482 482 open_file = open
483 483 open_mode = 'rb'
484 484
485 485 def run(self):
486 486
487 487 raise NotImplementedError
488 488
489 489 def getAllowedArgs(self):
490 490 if hasattr(self, '__attrs__'):
491 491 return self.__attrs__
492 492 else:
493 493 return inspect.getargspec(self.run).args
494 494
495 495 def set_kwargs(self, **kwargs):
496 496
497 497 for key, value in kwargs.items():
498 498 setattr(self, key, value)
499 499
500 500 def find_folders(self, path, startDate, endDate, folderfmt, last=False):
501 501
502 502 folders = [x for f in path.split(',')
503 503 for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
504 504 folders.sort()
505 505
506 506 if last:
507 507 folders = [folders[-1]]
508 508
509 509 for folder in folders:
510 510 try:
511 511 dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
512 512 if dt >= startDate and dt <= endDate:
513 513 yield os.path.join(path, folder)
514 514 else:
515 515 log.log('Skiping folder {}'.format(folder), self.name)
516 516 except Exception as e:
517 517 log.log('Skiping folder {}'.format(folder), self.name)
518 518 continue
519 519 return
520 520
521 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
521 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
522 522 expLabel='', last=False):
523 523
524 524 for path in folders:
525 525 files = glob.glob1(path, '*{}'.format(ext))
526 526 files.sort()
527 527 if last:
528 528 if files:
529 529 fo = files[-1]
530 530 try:
531 531 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
532 532 yield os.path.join(path, expLabel, fo)
533 533 except Exception as e:
534 534 pass
535 535 return
536 536 else:
537 537 return
538 538
539 539 for fo in files:
540 540 try:
541 541 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
542 542 if dt >= startDate and dt <= endDate:
543 543 yield os.path.join(path, expLabel, fo)
544 544 else:
545 545 log.log('Skiping file {}'.format(fo), self.name)
546 546 except Exception as e:
547 547 log.log('Skiping file {}'.format(fo), self.name)
548 548 continue
549 549
550 550 def searchFilesOffLine(self, path, startDate, endDate,
551 expLabel, ext, walk,
551 expLabel, ext, walk,
552 552 filefmt, folderfmt):
553 553 """Search files in offline mode for the given arguments
554 554
555 555 Return:
556 556 Generator of files
557 557 """
558 558
559 559 if walk:
560 560 folders = self.find_folders(
561 561 path, startDate, endDate, folderfmt)
562 562 else:
563 563 folders = path.split(',')
564 564
565 565 return self.find_files(
566 566 folders, ext, filefmt, startDate, endDate, expLabel)
567 567
568 568 def searchFilesOnLine(self, path, startDate, endDate,
569 expLabel, ext, walk,
569 expLabel, ext, walk,
570 570 filefmt, folderfmt):
571 571 """Search for the last file of the last folder
572 572
573 573 Arguments:
574 574 path : carpeta donde estan contenidos los files que contiene data
575 575 expLabel : Nombre del subexperimento (subfolder)
576 576 ext : extension de los files
577 577 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
578 578
579 579 Return:
580 580 generator with the full path of last filename
581 581 """
582 582
583 583 if walk:
584 584 folders = self.find_folders(
585 585 path, startDate, endDate, folderfmt, last=True)
586 586 else:
587 587 folders = path.split(',')
588 588
589 589 return self.find_files(
590 590 folders, ext, filefmt, startDate, endDate, expLabel, last=True)
591 591
592 592 def setNextFile(self):
593 593 """Set the next file to be readed open it and parse de file header"""
594 594
595 595 while True:
596 596 if self.fp != None:
597 597 self.fp.close()
598 598
599 599 if self.online:
600 600 newFile = self.setNextFileOnline()
601 601 else:
602 602 newFile = self.setNextFileOffline()
603 603
604 604 if not(newFile):
605 605 if self.online:
606 606 raise schainpy.admin.SchainError('Time to wait for new files reach')
607 607 else:
608 608 if self.fileIndex == -1:
609 609 raise schainpy.admin.SchainWarning('No files found in the given path')
610 610 else:
611 611 raise schainpy.admin.SchainWarning('No more files to read')
612 612
613 613 if self.verifyFile(self.filename):
614 614 break
615 615
616 616 log.log('Opening file: %s' % self.filename, self.name)
617 617
618 618 self.readFirstHeader()
619 619 self.nReadBlocks = 0
620 620
621 621 def setNextFileOnline(self):
622 622 """Check for the next file to be readed in online mode.
623 623
624 624 Set:
625 625 self.filename
626 626 self.fp
627 627 self.filesize
628 628
629 629 Return:
630 630 boolean
631 631
632 632 """
633 633 nextFile = True
634 634 nextDay = False
635 635
636 for nFiles in range(self.nFiles+1):
636 for nFiles in range(self.nFiles + 1):
637 637 for nTries in range(self.nTries):
638 638 fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
639 639 if fullfilename is not None:
640 640 break
641 641 log.warning(
642 642 "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
643 643 self.name)
644 644 time.sleep(self.delay)
645 645 nextFile = False
646 646 continue
647 647
648 648 if fullfilename is not None:
649 649 break
650 650
651 651 self.nTries = 1
652 652 nextFile = True
653 653
654 654 if nFiles == (self.nFiles - 1):
655 655 log.log('Trying with next day...', self.name)
656 656 nextDay = True
657 657 self.nTries = 3
658 658
659 659 if fullfilename:
660 660 self.fileSize = os.path.getsize(fullfilename)
661 661 self.filename = fullfilename
662 662 self.flagIsNewFile = 1
663 663 if self.fp != None:
664 664 self.fp.close()
665 665 self.fp = self.open_file(fullfilename, self.open_mode)
666 666 self.flagNoMoreFiles = 0
667 667 self.fileIndex += 1
668 668 return 1
669 669 else:
670 670 return 0
671 671
672 672 def setNextFileOffline(self):
673 673 """Open the next file to be readed in offline mode"""
674 674
675 675 try:
676 676 filename = next(self.filenameList)
677 self.fileIndex +=1
677 self.fileIndex += 1
678 678 except StopIteration:
679 679 self.flagNoMoreFiles = 1
680 680 return 0
681 681
682 682 self.filename = filename
683 683 self.fileSize = os.path.getsize(filename)
684 684 self.fp = self.open_file(filename, self.open_mode)
685 685 self.flagIsNewFile = 1
686 686
687 687 return 1
688 688
689 689 @staticmethod
690 690 def isDateTimeInRange(dt, startDate, endDate, startTime, endTime):
691 691 """Check if the given datetime is in range"""
692 692
693 693 if startDate <= dt.date() <= endDate:
694 694 if startTime <= dt.time() <= endTime:
695 695 return True
696 696 return False
697 697
698 698 def verifyFile(self, filename):
699 699 """Check for a valid file
700 700
701 701 Arguments:
702 702 filename -- full path filename
703 703
704 704 Return:
705 705 boolean
706 706 """
707 707
708 708 return True
709 709
710 710 def checkForRealPath(self, nextFile, nextDay):
711 711 """Check if the next file to be readed exists"""
712 712
713 713 raise NotImplementedError
714 714
715 715 def readFirstHeader(self):
716 716 """Parse the file header"""
717 717
718 718 pass
719 719
720 720 def waitDataBlock(self, pointer_location, blocksize=None):
721 721 """
722 722 """
723 723
724 724 currentPointer = pointer_location
725 725 if blocksize is None:
726 726 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
727 727 else:
728 728 neededSize = blocksize
729 729
730 730 for nTries in range(self.nTries):
731 731 self.fp.close()
732 732 self.fp = open(self.filename, 'rb')
733 733 self.fp.seek(currentPointer)
734 734
735 735 self.fileSize = os.path.getsize(self.filename)
736 736 currentSize = self.fileSize - currentPointer
737 737
738 738 if (currentSize >= neededSize):
739 739 return 1
740 740
741 741 log.warning(
742 742 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
743 743 self.name
744 744 )
745 745 time.sleep(self.delay)
746 746
747 747 return 0
748 748
749 749 class JRODataReader(Reader):
750 750
751 751 utc = 0
752 752 nReadBlocks = 0
753 753 foldercounter = 0
754 754 firstHeaderSize = 0
755 755 basicHeaderSize = 24
756 756 __isFirstTimeOnline = 1
757 757 filefmt = "*%Y%j***"
758 758 folderfmt = "*%Y%j"
759 759 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'online', 'delay', 'walk']
760 760
761 761 def getDtypeWidth(self):
762 762
763 763 dtype_index = get_dtype_index(self.dtype)
764 764 dtype_width = get_dtype_width(dtype_index)
765 765
766 766 return dtype_width
767 767
768 768 def checkForRealPath(self, nextFile, nextDay):
769 769 """Check if the next file to be readed exists.
770 770
771 771 Example :
772 772 nombre correcto del file es .../.../D2009307/P2009307367.ext
773 773
774 774 Entonces la funcion prueba con las siguientes combinaciones
775 775 .../.../y2009307367.ext
776 776 .../.../Y2009307367.ext
777 777 .../.../x2009307/y2009307367.ext
778 778 .../.../x2009307/Y2009307367.ext
779 779 .../.../X2009307/y2009307367.ext
780 780 .../.../X2009307/Y2009307367.ext
781 781 siendo para este caso, la ultima combinacion de letras, identica al file buscado
782 782
783 783 Return:
784 784 str -- fullpath of the file
785 785 """
786 786
787 787
788 788 if nextFile:
789 789 self.set += 1
790 790 if nextDay:
791 791 self.set = 0
792 792 self.doy += 1
793 793 foldercounter = 0
794 794 prefixDirList = [None, 'd', 'D']
795 795 if self.ext.lower() == ".r": # voltage
796 796 prefixFileList = ['d', 'D']
797 797 elif self.ext.lower() == ".pdata": # spectra
798 798 prefixFileList = ['p', 'P']
799 799
800 800 # barrido por las combinaciones posibles
801 801 for prefixDir in prefixDirList:
802 802 thispath = self.path
803 803 if prefixDir != None:
804 804 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
805 805 if foldercounter == 0:
806 thispath = os.path.join(self.path, "%s%04d%03d" %
806 thispath = os.path.join(self.path, "%s%04d%03d" %
807 807 (prefixDir, self.year, self.doy))
808 808 else:
809 809 thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
810 810 prefixDir, self.year, self.doy, foldercounter))
811 811 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
812 812 # formo el nombre del file xYYYYDDDSSS.ext
813 813 filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
814 814 fullfilename = os.path.join(
815 815 thispath, filename)
816 816
817 817 if os.path.exists(fullfilename):
818 818 return fullfilename, filename
819 819
820 820 return None, filename
821 821
822 822 def __waitNewBlock(self):
823 823 """
824 824 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
825 825
826 826 Si el modo de lectura es OffLine siempre retorn 0
827 827 """
828 828 if not self.online:
829 829 return 0
830 830
831 831 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
832 832 return 0
833 833
834 834 currentPointer = self.fp.tell()
835 835
836 836 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
837 837
838 838 for nTries in range(self.nTries):
839 839
840 840 self.fp.close()
841 841 self.fp = open(self.filename, 'rb')
842 842 self.fp.seek(currentPointer)
843 843
844 844 self.fileSize = os.path.getsize(self.filename)
845 845 currentSize = self.fileSize - currentPointer
846 846
847 847 if (currentSize >= neededSize):
848 848 self.basicHeaderObj.read(self.fp)
849 849 return 1
850 850
851 851 if self.fileSize == self.fileSizeByHeader:
852 852 # self.flagEoF = True
853 853 return 0
854 854
855 855 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
856 856 time.sleep(self.delay)
857 857
858 858 return 0
859 859
860 860 def __setNewBlock(self):
861 861
862 862 if self.fp == None:
863 863 return 0
864 864
865 865 if self.flagIsNewFile:
866 866 self.lastUTTime = self.basicHeaderObj.utc
867 867 return 1
868 868
869 869 if self.realtime:
870 870 self.flagDiscontinuousBlock = 1
871 871 if not(self.setNextFile()):
872 872 return 0
873 873 else:
874 874 return 1
875 875
876 876 currentSize = self.fileSize - self.fp.tell()
877 877 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
878 878
879 879 if (currentSize >= neededSize):
880 880 self.basicHeaderObj.read(self.fp)
881 881 self.lastUTTime = self.basicHeaderObj.utc
882 882 return 1
883 883
884 884 if self.__waitNewBlock():
885 885 self.lastUTTime = self.basicHeaderObj.utc
886 886 return 1
887 887
888 888 if not(self.setNextFile()):
889 889 return 0
890 890
891 891 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
892 892 self.lastUTTime = self.basicHeaderObj.utc
893 893
894 894 self.flagDiscontinuousBlock = 0
895 895
896 896 if deltaTime > self.maxTimeStep:
897 897 self.flagDiscontinuousBlock = 1
898 898
899 899 return 1
900 900
901 901 def readNextBlock(self):
902 902
903 903 while True:
904 904 if not(self.__setNewBlock()):
905 905 continue
906 906
907 907 if not(self.readBlock()):
908 908 return 0
909 909
910 910 self.getBasicHeader()
911 911
912 912 if not self.isDateTimeInRange(self.dataOut.datatime, self.startDate, self.endDate, self.startTime, self.endTime):
913 913 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
914 914 self.processingHeaderObj.dataBlocksPerFile,
915 915 self.dataOut.datatime.ctime()))
916 916 continue
917 917
918 918 break
919 919
920 920 if self.verbose:
921 921 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
922 922 self.processingHeaderObj.dataBlocksPerFile,
923 923 self.dataOut.datatime.ctime()))
924 924 return 1
925 925
926 926 def readFirstHeader(self):
927 927
928 928 self.basicHeaderObj.read(self.fp)
929 929 self.systemHeaderObj.read(self.fp)
930 930 self.radarControllerHeaderObj.read(self.fp)
931 931 self.processingHeaderObj.read(self.fp)
932 932 self.firstHeaderSize = self.basicHeaderObj.size
933 933
934 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
934 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
935 935 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
936 936 if datatype == 0:
937 937 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
938 938 elif datatype == 1:
939 939 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
940 940 elif datatype == 2:
941 941 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
942 942 elif datatype == 3:
943 943 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
944 944 elif datatype == 4:
945 945 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
946 946 elif datatype == 5:
947 947 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
948 948 else:
949 949 raise ValueError('Data type was not defined')
950 950
951 951 self.dtype = datatype_str
952 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
952 # self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
953 953 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
954 954 self.firstHeaderSize + self.basicHeaderSize * \
955 955 (self.processingHeaderObj.dataBlocksPerFile - 1)
956 956 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
957 957 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
958 958 self.getBlockDimension()
959 959
960 960 def verifyFile(self, filename):
961 961
962 962 flag = True
963 963
964 964 try:
965 965 fp = open(filename, 'rb')
966 966 except IOError:
967 967 log.error("File {} can't be opened".format(filename), self.name)
968 968 return False
969 969
970 970 if self.online and self.waitDataBlock(0):
971 971 pass
972 972
973 973 basicHeaderObj = BasicHeader(LOCALTIME)
974 974 systemHeaderObj = SystemHeader()
975 975 radarControllerHeaderObj = RadarControllerHeader()
976 976 processingHeaderObj = ProcessingHeader()
977 977
978 978 if not(basicHeaderObj.read(fp)):
979 979 flag = False
980 980 if not(systemHeaderObj.read(fp)):
981 981 flag = False
982 982 if not(radarControllerHeaderObj.read(fp)):
983 983 flag = False
984 984 if not(processingHeaderObj.read(fp)):
985 985 flag = False
986 986 if not self.online:
987 987 dt1 = basicHeaderObj.datatime
988 pos = self.fileSize-processingHeaderObj.blockSize-24
989 if pos<0:
988 pos = self.fileSize - processingHeaderObj.blockSize - 24
989 if pos < 0:
990 990 flag = False
991 991 log.error('Invalid size for file: {}'.format(self.filename), self.name)
992 992 else:
993 993 fp.seek(pos)
994 994 if not(basicHeaderObj.read(fp)):
995 995 flag = False
996 996 dt2 = basicHeaderObj.datatime
997 997 if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \
998 998 self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime):
999 999 flag = False
1000 1000
1001 1001 fp.close()
1002 1002 return flag
1003 1003
1004 1004 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1005 1005
1006 1006 path_empty = True
1007 1007
1008 1008 dateList = []
1009 1009 pathList = []
1010 1010
1011 1011 multi_path = path.split(',')
1012 1012
1013 1013 if not walk:
1014 1014
1015 1015 for single_path in multi_path:
1016 1016
1017 1017 if not os.path.isdir(single_path):
1018 1018 continue
1019 1019
1020 1020 fileList = glob.glob1(single_path, "*" + ext)
1021 1021
1022 1022 if not fileList:
1023 1023 continue
1024 1024
1025 1025 path_empty = False
1026 1026
1027 1027 fileList.sort()
1028 1028
1029 1029 for thisFile in fileList:
1030 1030
1031 1031 if not os.path.isfile(os.path.join(single_path, thisFile)):
1032 1032 continue
1033 1033
1034 1034 if not isRadarFile(thisFile):
1035 1035 continue
1036 1036
1037 1037 if not isFileInDateRange(thisFile, startDate, endDate):
1038 1038 continue
1039 1039
1040 1040 thisDate = getDateFromRadarFile(thisFile)
1041 1041
1042 1042 if thisDate in dateList or single_path in pathList:
1043 1043 continue
1044 1044
1045 1045 dateList.append(thisDate)
1046 1046 pathList.append(single_path)
1047 1047
1048 1048 else:
1049 1049 for single_path in multi_path:
1050 1050
1051 1051 if not os.path.isdir(single_path):
1052 1052 continue
1053 1053
1054 1054 dirList = []
1055 1055
1056 1056 for thisPath in os.listdir(single_path):
1057 1057
1058 1058 if not os.path.isdir(os.path.join(single_path, thisPath)):
1059 1059 continue
1060 1060
1061 1061 if not isRadarFolder(thisPath):
1062 1062 continue
1063 1063
1064 1064 if not isFolderInDateRange(thisPath, startDate, endDate):
1065 1065 continue
1066 1066
1067 1067 dirList.append(thisPath)
1068 1068
1069 1069 if not dirList:
1070 1070 continue
1071 1071
1072 1072 dirList.sort()
1073 1073
1074 1074 for thisDir in dirList:
1075 1075
1076 1076 datapath = os.path.join(single_path, thisDir, expLabel)
1077 1077 fileList = glob.glob1(datapath, "*" + ext)
1078 1078
1079 1079 if not fileList:
1080 1080 continue
1081 1081
1082 1082 path_empty = False
1083 1083
1084 1084 thisDate = getDateFromRadarFolder(thisDir)
1085 1085
1086 1086 pathList.append(datapath)
1087 1087 dateList.append(thisDate)
1088 1088
1089 1089 dateList.sort()
1090 1090
1091 1091 if walk:
1092 1092 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1093 1093 else:
1094 1094 pattern_path = multi_path[0]
1095 1095
1096 1096 if path_empty:
1097 1097 raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1098 1098 else:
1099 1099 if not dateList:
1100 1100 raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1101 1101
1102 1102 if include_path:
1103 1103 return dateList, pathList
1104 1104
1105 1105 return dateList
1106 1106
1107 1107 def setup(self, **kwargs):
1108 1108
1109 1109 self.set_kwargs(**kwargs)
1110 1110 if not self.ext.startswith('.'):
1111 1111 self.ext = '.{}'.format(self.ext)
1112 1112
1113 1113 if self.server is not None:
1114 1114 if 'tcp://' in self.server:
1115 1115 address = server
1116 1116 else:
1117 1117 address = 'ipc:///tmp/%s' % self.server
1118 1118 self.server = address
1119 1119 self.context = zmq.Context()
1120 1120 self.receiver = self.context.socket(zmq.PULL)
1121 1121 self.receiver.connect(self.server)
1122 1122 time.sleep(0.5)
1123 1123 print('[Starting] ReceiverData from {}'.format(self.server))
1124 1124 else:
1125 1125 self.server = None
1126 1126 if self.path == None:
1127 1127 raise ValueError("[Reading] The path is not valid")
1128 1128
1129 1129 if self.online:
1130 1130 log.log("[Reading] Searching files in online mode...", self.name)
1131 1131
1132 1132 for nTries in range(self.nTries):
1133 1133 fullpath = self.searchFilesOnLine(self.path, self.startDate,
1134 self.endDate, self.expLabel, self.ext, self.walk,
1134 self.endDate, self.expLabel, self.ext, self.walk,
1135 1135 self.filefmt, self.folderfmt)
1136 1136
1137 1137 try:
1138 1138 fullpath = next(fullpath)
1139 1139 except:
1140 1140 fullpath = None
1141 1141
1142 1142 if fullpath:
1143 1143 break
1144 1144
1145 1145 log.warning(
1146 1146 'Waiting {} sec for a valid file in {}: try {} ...'.format(
1147 self.delay, self.path, nTries + 1),
1147 self.delay, self.path, nTries + 1),
1148 1148 self.name)
1149 1149 time.sleep(self.delay)
1150 1150
1151 1151 if not(fullpath):
1152 1152 raise schainpy.admin.SchainError(
1153 1153 'There isn\'t any valid file in {}'.format(self.path))
1154 1154
1155 1155 pathname, filename = os.path.split(fullpath)
1156 1156 self.year = int(filename[1:5])
1157 1157 self.doy = int(filename[5:8])
1158 1158 self.set = int(filename[8:11]) - 1
1159 1159 else:
1160 1160 log.log("Searching files in {}".format(self.path), self.name)
1161 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
1161 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
1162 1162 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
1163 1163
1164 1164 self.setNextFile()
1165 1165
1166 1166 return
1167 1167
1168 1168 def getBasicHeader(self):
1169 1169
1170 1170 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1171 1171 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1172 1172
1173 1173 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1174 1174
1175 1175 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1176 1176
1177 1177 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1178 1178
1179 1179 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1180 1180
1181 1181 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1182 1182
1183 1183 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1184 1184
1185 1185 def getFirstHeader(self):
1186 1186
1187 1187 raise NotImplementedError
1188 1188
1189 1189 def getData(self):
1190 1190
1191 1191 raise NotImplementedError
1192 1192
1193 1193 def hasNotDataInBuffer(self):
1194 1194
1195 1195 raise NotImplementedError
1196 1196
1197 1197 def readBlock(self):
1198 1198
1199 1199 raise NotImplementedError
1200 1200
1201 1201 def isEndProcess(self):
1202 1202
1203 1203 return self.flagNoMoreFiles
1204 1204
1205 1205 def printReadBlocks(self):
1206 1206
1207 1207 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1208 1208
1209 1209 def printTotalBlocks(self):
1210 1210
1211 1211 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1212 1212
1213 1213 def run(self, **kwargs):
1214 1214 """
1215 1215
1216 1216 Arguments:
1217 1217 path :
1218 1218 startDate :
1219 1219 endDate :
1220 1220 startTime :
1221 1221 endTime :
1222 1222 set :
1223 1223 expLabel :
1224 1224 ext :
1225 1225 online :
1226 1226 delay :
1227 1227 walk :
1228 1228 getblock :
1229 1229 nTxs :
1230 1230 realtime :
1231 1231 blocksize :
1232 1232 blocktime :
1233 1233 skip :
1234 1234 cursor :
1235 1235 warnings :
1236 1236 server :
1237 1237 verbose :
1238 1238 format :
1239 1239 oneDDict :
1240 1240 twoDDict :
1241 1241 independentParam :
1242 1242 """
1243 1243
1244 1244 if not(self.isConfig):
1245 1245 self.setup(**kwargs)
1246 1246 self.isConfig = True
1247 1247 if self.server is None:
1248 1248 self.getData()
1249 1249 else:
1250 1250 self.getFromServer()
1251 1251
1252 1252
1253 1253 class JRODataWriter(Reader):
1254 1254
1255 1255 """
1256 1256 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1257 1257 de los datos siempre se realiza por bloques.
1258 1258 """
1259 1259
1260 1260 setFile = None
1261 1261 profilesPerBlock = None
1262 1262 blocksPerFile = None
1263 1263 nWriteBlocks = 0
1264 1264 fileDate = None
1265 1265
1266 1266 def __init__(self, dataOut=None):
1267 1267 raise NotImplementedError
1268 1268
1269 1269 def hasAllDataInBuffer(self):
1270 1270 raise NotImplementedError
1271 1271
1272 1272 def setBlockDimension(self):
1273 1273 raise NotImplementedError
1274 1274
1275 1275 def writeBlock(self):
1276 1276 raise NotImplementedError
1277 1277
1278 1278 def putData(self):
1279 1279 raise NotImplementedError
1280 1280
1281 1281 def getDtypeWidth(self):
1282 1282
1283 1283 dtype_index = get_dtype_index(self.dtype)
1284 1284 dtype_width = get_dtype_width(dtype_index)
1285 1285
1286 1286 return dtype_width
1287 1287
1288 1288 def getProcessFlags(self):
1289 1289
1290 1290 processFlags = 0
1291 1291
1292 1292 dtype_index = get_dtype_index(self.dtype)
1293 1293 procflag_dtype = get_procflag_dtype(dtype_index)
1294 1294
1295 1295 processFlags += procflag_dtype
1296 1296
1297 1297 if self.dataOut.flagDecodeData:
1298 1298 processFlags += PROCFLAG.DECODE_DATA
1299 1299
1300 1300 if self.dataOut.flagDeflipData:
1301 1301 processFlags += PROCFLAG.DEFLIP_DATA
1302 1302
1303 1303 if self.dataOut.code is not None:
1304 1304 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1305 1305
1306 1306 if self.dataOut.nCohInt > 1:
1307 1307 processFlags += PROCFLAG.COHERENT_INTEGRATION
1308 1308
1309 1309 if self.dataOut.type == "Spectra":
1310 1310 if self.dataOut.nIncohInt > 1:
1311 1311 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1312 1312
1313 1313 if self.dataOut.data_dc is not None:
1314 1314 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1315 1315
1316 1316 if self.dataOut.flagShiftFFT:
1317 1317 processFlags += PROCFLAG.SHIFT_FFT_DATA
1318 1318
1319 1319 return processFlags
1320 1320
1321 1321 def setBasicHeader(self):
1322 1322
1323 1323 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1324 1324 self.basicHeaderObj.version = self.versionFile
1325 1325 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1326 1326 utc = numpy.floor(self.dataOut.utctime)
1327 1327 milisecond = (self.dataOut.utctime - utc) * 1000.0
1328 1328 self.basicHeaderObj.utc = utc
1329 1329 self.basicHeaderObj.miliSecond = milisecond
1330 1330 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1331 1331 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1332 1332 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1333 1333
1334 1334 def setFirstHeader(self):
1335 1335 """
1336 1336 Obtiene una copia del First Header
1337 1337
1338 1338 Affected:
1339 1339
1340 1340 self.basicHeaderObj
1341 1341 self.systemHeaderObj
1342 1342 self.radarControllerHeaderObj
1343 1343 self.processingHeaderObj self.
1344 1344
1345 1345 Return:
1346 1346 None
1347 1347 """
1348 1348
1349 1349 raise NotImplementedError
1350 1350
1351 1351 def __writeFirstHeader(self):
1352 1352 """
1353 1353 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1354 1354
1355 1355 Affected:
1356 1356 __dataType
1357 1357
1358 1358 Return:
1359 1359 None
1360 1360 """
1361 1361
1362 1362 # CALCULAR PARAMETROS
1363 1363
1364 1364 sizeLongHeader = self.systemHeaderObj.size + \
1365 1365 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1366 1366 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1367 1367
1368 1368 self.basicHeaderObj.write(self.fp)
1369 1369 self.systemHeaderObj.write(self.fp)
1370 1370 self.radarControllerHeaderObj.write(self.fp)
1371 1371 self.processingHeaderObj.write(self.fp)
1372 1372
1373 1373 def __setNewBlock(self):
1374 1374 """
1375 1375 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1376 1376
1377 1377 Return:
1378 1378 0 : si no pudo escribir nada
1379 1379 1 : Si escribio el Basic el First Header
1380 1380 """
1381 1381 if self.fp == None:
1382 1382 self.setNextFile()
1383 1383
1384 1384 if self.flagIsNewFile:
1385 1385 return 1
1386 1386
1387 1387 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1388 1388 self.basicHeaderObj.write(self.fp)
1389 1389 return 1
1390 1390
1391 1391 if not(self.setNextFile()):
1392 1392 return 0
1393 1393
1394 1394 return 1
1395 1395
1396 1396 def writeNextBlock(self):
1397 1397 """
1398 1398 Selecciona el bloque siguiente de datos y los escribe en un file
1399 1399
1400 1400 Return:
1401 1401 0 : Si no hizo pudo escribir el bloque de datos
1402 1402 1 : Si no pudo escribir el bloque de datos
1403 1403 """
1404 1404 if not(self.__setNewBlock()):
1405 1405 return 0
1406 1406
1407 1407 self.writeBlock()
1408 1408
1409 1409 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1410 1410 self.processingHeaderObj.dataBlocksPerFile))
1411 1411
1412 1412 return 1
1413 1413
1414 1414 def setNextFile(self):
1415 1415 """Determina el siguiente file que sera escrito
1416 1416
1417 1417 Affected:
1418 1418 self.filename
1419 1419 self.subfolder
1420 1420 self.fp
1421 1421 self.setFile
1422 1422 self.flagIsNewFile
1423 1423
1424 1424 Return:
1425 1425 0 : Si el archivo no puede ser escrito
1426 1426 1 : Si el archivo esta listo para ser escrito
1427 1427 """
1428 1428 ext = self.ext
1429 1429 path = self.path
1430 1430
1431 1431 if self.fp != None:
1432 1432 self.fp.close()
1433 1433
1434 1434 if not os.path.exists(path):
1435 1435 os.mkdir(path)
1436 1436
1437 1437 timeTuple = time.localtime(self.dataOut.utctime)
1438 1438 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1439 1439
1440 1440 fullpath = os.path.join(path, subfolder)
1441 1441 setFile = self.setFile
1442 1442
1443 1443 if not(os.path.exists(fullpath)):
1444 1444 os.mkdir(fullpath)
1445 1445 setFile = -1 # inicializo mi contador de seteo
1446 1446 else:
1447 1447 filesList = os.listdir(fullpath)
1448 1448 if len(filesList) > 0:
1449 1449 filesList = sorted(filesList, key=str.lower)
1450 1450 filen = filesList[-1]
1451 1451 # el filename debera tener el siguiente formato
1452 1452 # 0 1234 567 89A BCDE (hex)
1453 1453 # x YYYY DDD SSS .ext
1454 1454 if isNumber(filen[8:11]):
1455 1455 # inicializo mi contador de seteo al seteo del ultimo file
1456 1456 setFile = int(filen[8:11])
1457 1457 else:
1458 1458 setFile = -1
1459 1459 else:
1460 1460 setFile = -1 # inicializo mi contador de seteo
1461 1461
1462 1462 setFile += 1
1463 1463
1464 1464 # If this is a new day it resets some values
1465 1465 if self.dataOut.datatime.date() > self.fileDate:
1466 1466 setFile = 0
1467 1467 self.nTotalBlocks = 0
1468 1468
1469 1469 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1470 1470 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1471 1471
1472 1472 filename = os.path.join(path, subfolder, filen)
1473 1473
1474 1474 fp = open(filename, 'wb')
1475 1475
1476 1476 self.blockIndex = 0
1477 1477 self.filename = filename
1478 1478 self.subfolder = subfolder
1479 1479 self.fp = fp
1480 1480 self.setFile = setFile
1481 1481 self.flagIsNewFile = 1
1482 1482 self.fileDate = self.dataOut.datatime.date()
1483 1483 self.setFirstHeader()
1484 1484
1485 1485 print('[Writing] Opening file: %s' % self.filename)
1486 1486
1487 1487 self.__writeFirstHeader()
1488 1488
1489 1489 return 1
1490 1490
1491 1491 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1492 1492 """
1493 1493 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1494 1494
1495 1495 Inputs:
1496 1496 path : directory where data will be saved
1497 1497 profilesPerBlock : number of profiles per block
1498 1498 set : initial file set
1499 1499 datatype : An integer number that defines data type:
1500 1500 0 : int8 (1 byte)
1501 1501 1 : int16 (2 bytes)
1502 1502 2 : int32 (4 bytes)
1503 1503 3 : int64 (8 bytes)
1504 1504 4 : float32 (4 bytes)
1505 1505 5 : double64 (8 bytes)
1506 1506
1507 1507 Return:
1508 1508 0 : Si no realizo un buen seteo
1509 1509 1 : Si realizo un buen seteo
1510 1510 """
1511 1511
1512 1512 if ext == None:
1513 1513 ext = self.ext
1514 1514
1515 1515 self.ext = ext.lower()
1516 1516
1517 1517 self.path = path
1518 1518
1519 1519 if set is None:
1520 1520 self.setFile = -1
1521 1521 else:
1522 1522 self.setFile = set - 1
1523 1523
1524 1524 self.blocksPerFile = blocksPerFile
1525 1525 self.profilesPerBlock = profilesPerBlock
1526 1526 self.dataOut = dataOut
1527 1527 self.fileDate = self.dataOut.datatime.date()
1528 1528 self.dtype = self.dataOut.dtype
1529 1529
1530 1530 if datatype is not None:
1531 1531 self.dtype = get_numpy_dtype(datatype)
1532 1532
1533 1533 if not(self.setNextFile()):
1534 1534 print("[Writing] There isn't a next file")
1535 1535 return 0
1536 1536
1537 1537 self.setBlockDimension()
1538 1538
1539 1539 return 1
1540 1540
1541 1541 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1542 1542
1543 1543 if not(self.isConfig):
1544 1544
1545 1545 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1546 1546 set=set, ext=ext, datatype=datatype, **kwargs)
1547 1547 self.isConfig = True
1548 1548
1549 1549 self.dataOut = dataOut
1550 1550 self.putData()
1551 1551 return self.dataOut
1552 1552
1553 1553 @MPDecorator
1554 1554 class printInfo(Operation):
1555 1555
1556 1556 def __init__(self):
1557 1557
1558 1558 Operation.__init__(self)
1559 1559 self.__printInfo = True
1560 1560
1561 def run(self, dataOut, headers = ['systemHeaderObj', 'radarControllerHeaderObj', 'processingHeaderObj']):
1561 def run(self, dataOut, headers=['systemHeaderObj', 'radarControllerHeaderObj', 'processingHeaderObj']):
1562 1562 if self.__printInfo == False:
1563 1563 return
1564 1564
1565 1565 for header in headers:
1566 1566 if hasattr(dataOut, header):
1567 1567 obj = getattr(dataOut, header)
1568 1568 if hasattr(obj, 'printInfo'):
1569 1569 obj.printInfo()
1570 1570 else:
1571 1571 print(obj)
1572 1572 else:
1573 1573 log.warning('Header {} Not found in object'.format(header))
1574 1574
1575 1575 self.__printInfo = False
Show file before | Show file after | Hide comments
@@ -1,793 +1,793
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 # SUBCHANNELS EN VEZ DE CHANNELS
7 7 # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
8 8 # ACTUALIZACION DE VERSION
9 9 # HEADERS
10 10 # MODULO DE ESCRITURA
11 11 # METADATA
12 12
13 13 import os
14 14 import time
15 15 import datetime
16 16 import numpy
17 17 import timeit
18 18 from fractions import Fraction
19 19 from time import time
20 20 from time import sleep
21 21
22 22 import schainpy.admin
23 23 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 24 from schainpy.model.data.jrodata import Voltage
25 25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
26 26
27 27 import pickle
28 28 try:
29 29 import digital_rf
30 30 except:
31 31 pass
32 32
33 33
34 34 class DigitalRFReader(ProcessingUnit):
35 35 '''
36 36 classdocs
37 37 '''
38 38
39 39 def __init__(self):
40 40 '''
41 41 Constructor
42 42 '''
43 43
44 44 ProcessingUnit.__init__(self)
45 45
46 self.dataOut = Voltage()
47 self.__printInfo = True
46 self.dataOut = Voltage()
47 self.__printInfo = True
48 48 self.__flagDiscontinuousBlock = False
49 49 self.__bufferIndex = 9999999
50 self.__codeType = 0
51 self.__ippKm = None
52 self.__nCode = None
53 self.__nBaud = None
54 self.__code = None
55 self.dtype = None
56 self.oldAverage = None
57 self.path = None
50 self.__codeType = 0
51 self.__ippKm = None
52 self.__nCode = None
53 self.__nBaud = None
54 self.__code = None
55 self.dtype = None
56 self.oldAverage = None
57 self.path = None
58 58
59 59 def close(self):
60 60 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
61 61 return
62 62
63 63 def __getCurrentSecond(self):
64 64
65 65 return self.__thisUnixSample / self.__sample_rate
66 66
67 67 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
68 68
69 69 def __setFileHeader(self):
70 70 '''
71 71 In this method will be initialized every parameter of dataOut object (header, no data)
72 72 '''
73 73 ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
74 74
75 75 nProfiles = 1.0 / ippSeconds # Number of profiles in one second
76 76
77 77 try:
78 78 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
79 79 self.__radarControllerHeader)
80 80 except:
81 81 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
82 82 txA=0,
83 83 txB=0,
84 84 nWindows=1,
85 85 nHeights=self.__nSamples,
86 86 firstHeight=self.__firstHeigth,
87 87 deltaHeight=self.__deltaHeigth,
88 88 codeType=self.__codeType,
89 89 nCode=self.__nCode, nBaud=self.__nBaud,
90 90 code=self.__code)
91 91
92 92 try:
93 93 self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
94 94 except:
95 95 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
96 96 nProfiles=nProfiles,
97 97 nChannels=len(
98 98 self.__channelList),
99 99 adcResolution=14)
100 self.dataOut.type = "Voltage"
100 self.dataOut.type = "Voltage"
101 101
102 self.dataOut.data = None
102 self.dataOut.data = None
103 103
104 104 self.dataOut.dtype = self.dtype
105 105
106 106 # self.dataOut.nChannels = 0
107 107
108 108 # self.dataOut.nHeights = 0
109 109
110 self.dataOut.nProfiles = int(nProfiles)
110 self.dataOut.nProfiles = int(nProfiles)
111 111
112 self.dataOut.heightList = self.__firstHeigth + \
112 self.dataOut.heightList = self.__firstHeigth + \
113 113 numpy.arange(self.__nSamples, dtype=numpy.float) * \
114 114 self.__deltaHeigth
115 115
116 116 self.dataOut.channelList = list(range(self.__num_subchannels))
117 117
118 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
118 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
119 119
120 120 # self.dataOut.channelIndexList = None
121 121
122 self.dataOut.flagNoData = True
122 self.dataOut.flagNoData = True
123 123
124 124 self.dataOut.flagDataAsBlock = False
125 125 # Set to TRUE if the data is discontinuous
126 126 self.dataOut.flagDiscontinuousBlock = False
127 127
128 self.dataOut.utctime = None
128 self.dataOut.utctime = None
129 129
130 130 # timezone like jroheader, difference in minutes between UTC and localtime
131 self.dataOut.timeZone = self.__timezone / 60
131 self.dataOut.timeZone = self.__timezone / 60
132 132
133 self.dataOut.dstFlag = 0
133 self.dataOut.dstFlag = 0
134 134
135 self.dataOut.errorCount = 0
135 self.dataOut.errorCount = 0
136 136
137 137 try:
138 138 self.dataOut.nCohInt = self.fixed_metadata_dict.get(
139 139 'nCohInt', self.nCohInt)
140 140
141 141 # asumo que la data esta decodificada
142 142 self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
143 143 'flagDecodeData', self.flagDecodeData)
144 144
145 145 # asumo que la data esta sin flip
146 146 self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
147 147
148 self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
148 self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
149 149
150 self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
150 self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
151 151 except:
152 152 pass
153 153
154 154 self.dataOut.ippSeconds = ippSeconds
155 155
156 156 # Time interval between profiles
157 157 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
158 158
159 self.dataOut.frequency = self.__frequency
159 self.dataOut.frequency = self.__frequency
160 160
161 self.dataOut.realtime = self.__online
161 self.dataOut.realtime = self.__online
162 162
163 163 def findDatafiles(self, path, startDate=None, endDate=None):
164 164
165 165 if not os.path.isdir(path):
166 166 return []
167 167
168 168 try:
169 169 digitalReadObj = digital_rf.DigitalRFReader(
170 170 path, load_all_metadata=True)
171 171 except:
172 172 digitalReadObj = digital_rf.DigitalRFReader(path)
173 173
174 channelNameList = digitalReadObj.get_channels()
174 channelNameList = digitalReadObj.get_channels()
175 175
176 176 if not channelNameList:
177 177 return []
178 178
179 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
179 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
180 180
181 sample_rate = metadata_dict['sample_rate'][0]
181 sample_rate = metadata_dict['sample_rate'][0]
182 182
183 183 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
184 184
185 185 try:
186 timezone = this_metadata_file['timezone'].value
186 timezone = this_metadata_file['timezone'].value
187 187 except:
188 timezone = 0
188 timezone = 0
189 189
190 190 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
191 191 channelNameList[0]) / sample_rate - timezone
192 192
193 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
194 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
193 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
194 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
195 195
196 196 if not startDate:
197 startDate = startDatetime.date()
197 startDate = startDatetime.date()
198 198
199 199 if not endDate:
200 endDate = endDatatime.date()
200 endDate = endDatatime.date()
201 201
202 dateList = []
202 dateList = []
203 203
204 thisDatetime = startDatetime
204 thisDatetime = startDatetime
205 205
206 206 while(thisDatetime <= endDatatime):
207 207
208 thisDate = thisDatetime.date()
208 thisDate = thisDatetime.date()
209 209
210 if thisDate < startDate:
210 if thisDate < startDate:
211 211 continue
212 212
213 if thisDate > endDate:
213 if thisDate > endDate:
214 214 break
215 215
216 216 dateList.append(thisDate)
217 217 thisDatetime += datetime.timedelta(1)
218 218
219 219 return dateList
220 220
221 221 def setup(self, path=None,
222 222 startDate=None,
223 223 endDate=None,
224 224 startTime=datetime.time(0, 0, 0),
225 225 endTime=datetime.time(23, 59, 59),
226 226 channelList=None,
227 227 nSamples=None,
228 228 online=False,
229 229 delay=60,
230 230 buffer_size=1024,
231 231 ippKm=None,
232 232 nCohInt=1,
233 233 nCode=1,
234 234 nBaud=1,
235 235 flagDecodeData=False,
236 236 code=numpy.ones((1, 1), dtype=numpy.int),
237 237 **kwargs):
238 238 '''
239 239 In this method we should set all initial parameters.
240 240
241 241 Inputs:
242 242 path
243 243 startDate
244 244 endDate
245 245 startTime
246 246 endTime
247 247 set
248 248 expLabel
249 249 ext
250 250 online
251 251 delay
252 252 '''
253 self.path = path
254 self.nCohInt = nCohInt
253 self.path = path
254 self.nCohInt = nCohInt
255 255 self.flagDecodeData = flagDecodeData
256 self.i = 0
256 self.i = 0
257 257 if not os.path.isdir(path):
258 258 raise ValueError("[Reading] Directory %s does not exist" % path)
259 259
260 260 try:
261 261 self.digitalReadObj = digital_rf.DigitalRFReader(
262 262 path, load_all_metadata=True)
263 263 except:
264 264 self.digitalReadObj = digital_rf.DigitalRFReader(path)
265 265
266 channelNameList = self.digitalReadObj.get_channels()
266 channelNameList = self.digitalReadObj.get_channels()
267 267
268 268 if not channelNameList:
269 269 raise ValueError("[Reading] Directory %s does not have any files" % path)
270 270
271 271 if not channelList:
272 272 channelList = list(range(len(channelNameList)))
273 273
274 274 ########## Reading metadata ######################
275 275
276 top_properties = self.digitalReadObj.get_properties(
276 top_properties = self.digitalReadObj.get_properties(
277 277 channelNameList[channelList[0]])
278 278
279 self.__num_subchannels = top_properties['num_subchannels']
280 self.__sample_rate = 1.0 * \
279 self.__num_subchannels = top_properties['num_subchannels']
280 self.__sample_rate = 1.0 * \
281 281 top_properties['sample_rate_numerator'] / \
282 282 top_properties['sample_rate_denominator']
283 283 # self.__samples_per_file = top_properties['samples_per_file'][0]
284 self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate # why 0.15?
284 self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate # why 0.15?
285 285
286 this_metadata_file = self.digitalReadObj.get_digital_metadata(
286 this_metadata_file = self.digitalReadObj.get_digital_metadata(
287 287 channelNameList[channelList[0]])
288 metadata_bounds = this_metadata_file.get_bounds()
288 metadata_bounds = this_metadata_file.get_bounds()
289 289 self.fixed_metadata_dict = this_metadata_file.read(
290 290 metadata_bounds[0])[metadata_bounds[0]] # GET FIRST HEADER
291 291
292 292 try:
293 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
293 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
294 294 self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
295 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
296 self.dtype = pickle.loads(self.fixed_metadata_dict['dtype'])
295 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
296 self.dtype = pickle.loads(self.fixed_metadata_dict['dtype'])
297 297 except:
298 298 pass
299 299
300 300 self.__frequency = None
301 301
302 302 self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
303 303
304 304 self.__timezone = self.fixed_metadata_dict.get('timezone', 18000)
305 305
306 306 try:
307 307 nSamples = self.fixed_metadata_dict['nSamples']
308 308 except:
309 309 nSamples = None
310 310
311 311 self.__firstHeigth = 0
312 312
313 313 try:
314 codeType = self.__radarControllerHeader['codeType']
314 codeType = self.__radarControllerHeader['codeType']
315 315 except:
316 codeType = 0
316 codeType = 0
317 317
318 318 try:
319 319 if codeType:
320 320 nCode = self.__radarControllerHeader['nCode']
321 321 nBaud = self.__radarControllerHeader['nBaud']
322 code = self.__radarControllerHeader['code']
322 code = self.__radarControllerHeader['code']
323 323 except:
324 324 pass
325 325
326 326 if not ippKm:
327 327 try:
328 328 # seconds to km
329 329 ippKm = self.__radarControllerHeader['ipp']
330 330 except:
331 331 ippKm = None
332 332 ####################################################
333 self.__ippKm = ippKm
333 self.__ippKm = ippKm
334 334 startUTCSecond = None
335 endUTCSecond = None
335 endUTCSecond = None
336 336
337 337 if startDate:
338 startDatetime = datetime.datetime.combine(startDate, startTime)
338 startDatetime = datetime.datetime.combine(startDate, startTime)
339 339 startUTCSecond = (
340 340 startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
341 341
342 342 if endDate:
343 endDatetime = datetime.datetime.combine(endDate, endTime)
344 endUTCSecond = (endDatetime - datetime.datetime(1970,
343 endDatetime = datetime.datetime.combine(endDate, endTime)
344 endUTCSecond = (endDatetime - datetime.datetime(1970,
345 345 1, 1)).total_seconds() + self.__timezone
346 346
347 347 start_index, end_index = self.digitalReadObj.get_bounds(
348 348 channelNameList[channelList[0]])
349 349
350 350 if not startUTCSecond:
351 351 startUTCSecond = start_index / self.__sample_rate
352 352
353 if start_index > startUTCSecond * self.__sample_rate:
353 if start_index > startUTCSecond * self.__sample_rate:
354 354 startUTCSecond = start_index / self.__sample_rate
355 355
356 356 if not endUTCSecond:
357 endUTCSecond = end_index / self.__sample_rate
357 endUTCSecond = end_index / self.__sample_rate
358 358
359 if end_index < endUTCSecond * self.__sample_rate:
360 endUTCSecond = end_index / self.__sample_rate
359 if end_index < endUTCSecond * self.__sample_rate:
360 endUTCSecond = end_index / self.__sample_rate
361 361 if not nSamples:
362 362 if not ippKm:
363 363 raise ValueError("[Reading] nSamples or ippKm should be defined")
364 nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
365 channelBoundList = []
364 nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
365 channelBoundList = []
366 366 channelNameListFiltered = []
367 367
368 368 for thisIndexChannel in channelList:
369 thisChannelName = channelNameList[thisIndexChannel]
369 thisChannelName = channelNameList[thisIndexChannel]
370 370 start_index, end_index = self.digitalReadObj.get_bounds(
371 371 thisChannelName)
372 372 channelBoundList.append((start_index, end_index))
373 373 channelNameListFiltered.append(thisChannelName)
374 374
375 375 self.profileIndex = 0
376 self.i = 0
377 self.__delay = delay
378
379 self.__codeType = codeType
380 self.__nCode = nCode
381 self.__nBaud = nBaud
382 self.__code = code
383
384 self.__datapath = path
385 self.__online = online
386 self.__channelList = channelList
387 self.__channelNameList = channelNameListFiltered
376 self.i = 0
377 self.__delay = delay
378
379 self.__codeType = codeType
380 self.__nCode = nCode
381 self.__nBaud = nBaud
382 self.__code = code
383
384 self.__datapath = path
385 self.__online = online
386 self.__channelList = channelList
387 self.__channelNameList = channelNameListFiltered
388 388 self.__channelBoundList = channelBoundList
389 self.__nSamples = nSamples
390 self.__samples_to_read = int(nSamples) # FIJO: AHORA 40
391 self.__nChannels = len(self.__channelList)
389 self.__nSamples = nSamples
390 self.__samples_to_read = int(nSamples) # FIJO: AHORA 40
391 self.__nChannels = len(self.__channelList)
392 392
393 self.__startUTCSecond = startUTCSecond
394 self.__endUTCSecond = endUTCSecond
393 self.__startUTCSecond = startUTCSecond
394 self.__endUTCSecond = endUTCSecond
395 395
396 self.__timeInterval = 1.0 * self.__samples_to_read / \
396 self.__timeInterval = 1.0 * self.__samples_to_read / \
397 397 self.__sample_rate # Time interval
398 398
399 399 if online:
400 400 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
401 401 startUTCSecond = numpy.floor(endUTCSecond)
402 402
403 403 # por que en el otro metodo lo primero q se hace es sumar samplestoread
404 404 self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
405 405
406 self.__data_buffer = numpy.zeros(
406 self.__data_buffer = numpy.zeros(
407 407 (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
408 408
409 409 self.__setFileHeader()
410 410 self.isConfig = True
411 411
412 412 print("[Reading] Digital RF Data was found from %s to %s " % (
413 413 datetime.datetime.utcfromtimestamp(
414 414 self.__startUTCSecond - self.__timezone),
415 415 datetime.datetime.utcfromtimestamp(
416 416 self.__endUTCSecond - self.__timezone)
417 417 ))
418 418
419 419 print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
420 420 datetime.datetime.utcfromtimestamp(
421 421 endUTCSecond - self.__timezone)
422 422 ))
423 self.oldAverage = None
424 self.count = 0
423 self.oldAverage = None
424 self.count = 0
425 425 self.executionTime = 0
426 426
427 427 def __reload(self):
428 428 # print
429 429 # print "%s not in range [%s, %s]" %(
430 430 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
431 431 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
432 432 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
433 433 # )
434 434 print("[Reading] reloading metadata ...")
435 435
436 436 try:
437 437 self.digitalReadObj.reload(complete_update=True)
438 438 except:
439 439 self.digitalReadObj = digital_rf.DigitalRFReader(self.path)
440 440
441 start_index, end_index = self.digitalReadObj.get_bounds(
441 start_index, end_index = self.digitalReadObj.get_bounds(
442 442 self.__channelNameList[self.__channelList[0]])
443 443
444 if start_index > self.__startUTCSecond * self.__sample_rate:
444 if start_index > self.__startUTCSecond * self.__sample_rate:
445 445 self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
446 446
447 if end_index > self.__endUTCSecond * self.__sample_rate:
447 if end_index > self.__endUTCSecond * self.__sample_rate:
448 448 self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
449 449 print()
450 450 print("[Reading] New timerange found [%s, %s] " % (
451 451 datetime.datetime.utcfromtimestamp(
452 452 self.__startUTCSecond - self.__timezone),
453 453 datetime.datetime.utcfromtimestamp(
454 454 self.__endUTCSecond - self.__timezone)
455 455 ))
456 456
457 457 return True
458 458
459 459 return False
460 460
461 461 def timeit(self, toExecute):
462 t0 = time.time()
462 t0 = time.time()
463 463 toExecute()
464 self.executionTime = time.time() - t0
464 self.executionTime = time.time() - t0
465 465 if self.oldAverage is None:
466 466 self.oldAverage = self.executionTime
467 self.oldAverage = (self.executionTime + self.count *
467 self.oldAverage = (self.executionTime + self.count *
468 468 self.oldAverage) / (self.count + 1.0)
469 self.count = self.count + 1.0
469 self.count = self.count + 1.0
470 470 return
471 471
472 472 def __readNextBlock(self, seconds=30, volt_scale=1):
473 473 '''
474 474 '''
475 475
476 476 # Set the next data
477 477 self.__flagDiscontinuousBlock = False
478 self.__thisUnixSample += self.__samples_to_read
478 self.__thisUnixSample += self.__samples_to_read
479 479
480 480 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
481 481 print ("[Reading] There are no more data into selected time-range")
482 482 if self.__online:
483 483 sleep(3)
484 484 self.__reload()
485 485 else:
486 486 return False
487 487
488 488 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
489 489 return False
490 490 self.__thisUnixSample -= self.__samples_to_read
491 491
492 492 indexChannel = 0
493 493
494 494 dataOk = False
495 495
496 496 for thisChannelName in self.__channelNameList: # TODO VARIOS CHANNELS?
497 497 for indexSubchannel in range(self.__num_subchannels):
498 498 try:
499 t0 = time()
499 t0 = time()
500 500 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
501 501 self.__samples_to_read,
502 502 thisChannelName, sub_channel=indexSubchannel)
503 self.executionTime = time() - t0
503 self.executionTime = time() - t0
504 504 if self.oldAverage is None:
505 505 self.oldAverage = self.executionTime
506 self.oldAverage = (
506 self.oldAverage = (
507 507 self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
508 508 self.count = self.count + 1.0
509 509
510 510 except IOError as e:
511 511 # read next profile
512 512 self.__flagDiscontinuousBlock = True
513 513 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
514 514 break
515 515
516 516 if result.shape[0] != self.__samples_to_read:
517 517 self.__flagDiscontinuousBlock = True
518 518 print("[Reading] %s: Too few samples were found, just %d/%d samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
519 519 result.shape[0],
520 520 self.__samples_to_read))
521 521 break
522 522
523 523 self.__data_buffer[indexSubchannel, :] = result * volt_scale
524 indexChannel+=1
524 indexChannel += 1
525 525
526 dataOk = True
526 dataOk = True
527 527
528 self.__utctime = self.__thisUnixSample / self.__sample_rate
528 self.__utctime = self.__thisUnixSample / self.__sample_rate
529 529
530 530 if not dataOk:
531 531 return False
532 532
533 533 print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
534 534 self.__samples_to_read,
535 535 self.__timeInterval))
536 536
537 self.__bufferIndex = 0
537 self.__bufferIndex = 0
538 538
539 539 return True
540 540
541 541 def __isBufferEmpty(self):
542 542 return self.__bufferIndex > self.__samples_to_read - self.__nSamples # 40960 - 40
543 543
544 544 def getData(self, seconds=30, nTries=5):
545 545 '''
546 546 This method gets the data from files and put the data into the dataOut object
547 547
548 548 In addition, increase el the buffer counter in one.
549 549
550 550 Return:
551 551 data : retorna un perfil de voltages (alturas * canales) copiados desde el
552 552 buffer. Si no hay mas archivos a leer retorna None.
553 553
554 554 Affected:
555 555 self.dataOut
556 556 self.profileIndex
557 557 self.flagDiscontinuousBlock
558 558 self.flagIsNewBlock
559 559 '''
560 #print("getdata")
560 # print("getdata")
561 561 err_counter = 0
562 562 self.dataOut.flagNoData = True
563 563
564 564 if self.__isBufferEmpty():
565 #print("hi")
565 # print("hi")
566 566 self.__flagDiscontinuousBlock = False
567 567
568 568 while True:
569 #print ("q ha pasado")
569 # print ("q ha pasado")
570 570 if self.__readNextBlock():
571 571 break
572 572 if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
573 573 raise schainpy.admin.SchainError('Error')
574 574 return
575 575
576 576 if self.__flagDiscontinuousBlock:
577 577 raise schainpy.admin.SchainError('discontinuous block found')
578 578 return
579 579
580 580 if not self.__online:
581 581 raise schainpy.admin.SchainError('Online?')
582 582 return
583 583
584 584 err_counter += 1
585 585 if err_counter > nTries:
586 586 raise schainpy.admin.SchainError('Max retrys reach')
587 587 return
588 588
589 589 print('[Reading] waiting %d seconds to read a new block' % seconds)
590 590 time.sleep(seconds)
591 591
592 self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
593 self.dataOut.utctime = ( self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
594 self.dataOut.flagNoData = False
592 self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
593 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
594 self.dataOut.flagNoData = False
595 595 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
596 self.dataOut.profileIndex = self.profileIndex
596 self.dataOut.profileIndex = self.profileIndex
597 597
598 598 self.__bufferIndex += self.__nSamples
599 self.profileIndex += 1
599 self.profileIndex += 1
600 600
601 601 if self.profileIndex == self.dataOut.nProfiles:
602 602 self.profileIndex = 0
603 603
604 604 return True
605 605
606 606 def printInfo(self):
607 607 '''
608 608 '''
609 609 if self.__printInfo == False:
610 610 return
611 611
612 612 # self.systemHeaderObj.printInfo()
613 613 # self.radarControllerHeaderObj.printInfo()
614 614
615 615 self.__printInfo = False
616 616
617 617 def printNumberOfBlock(self):
618 618 '''
619 619 '''
620 620 return
621 621 # print self.profileIndex
622 622
623 623 def run(self, **kwargs):
624 624 '''
625 625 This method will be called many times so here you should put all your code
626 626 '''
627 627
628 628 if not self.isConfig:
629 629 self.setup(**kwargs)
630 #self.i = self.i+1
630 # self.i = self.i+1
631 631 self.getData(seconds=self.__delay)
632 632
633 633 return
634 634
635 635 @MPDecorator
636 636 class DigitalRFWriter(Operation):
637 637 '''
638 638 classdocs
639 639 '''
640 640
641 641 def __init__(self, **kwargs):
642 642 '''
643 643 Constructor
644 644 '''
645 645 Operation.__init__(self, **kwargs)
646 646 self.metadata_dict = {}
647 self.dataOut = None
648 self.dtype = None
649 self.oldAverage = 0
647 self.dataOut = None
648 self.dtype = None
649 self.oldAverage = 0
650 650
651 651 def setHeader(self):
652 652
653 self.metadata_dict['frequency'] = self.dataOut.frequency
654 self.metadata_dict['timezone'] = self.dataOut.timeZone
655 self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype)
656 self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
657 self.metadata_dict['heightList'] = self.dataOut.heightList
658 self.metadata_dict['channelList'] = self.dataOut.channelList
653 self.metadata_dict['frequency'] = self.dataOut.frequency
654 self.metadata_dict['timezone'] = self.dataOut.timeZone
655 self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype)
656 self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
657 self.metadata_dict['heightList'] = self.dataOut.heightList
658 self.metadata_dict['channelList'] = self.dataOut.channelList
659 659 self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
660 660 self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
661 self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
662 self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
663 self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
664 self.metadata_dict['type'] = self.dataOut.type
665 self.metadata_dict['flagDataAsBlock']= getattr(
661 self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
662 self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
663 self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
664 self.metadata_dict['type'] = self.dataOut.type
665 self.metadata_dict['flagDataAsBlock'] = getattr(
666 666 self.dataOut, 'flagDataAsBlock', None) # chequear
667 667
668 668 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
669 669 '''
670 670 In this method we should set all initial parameters.
671 671 Input:
672 672 dataOut: Input data will also be outputa data
673 673 '''
674 674 self.setHeader()
675 self.__ippSeconds = dataOut.ippSeconds
676 self.__deltaH = dataOut.getDeltaH()
675 self.__ippSeconds = dataOut.ippSeconds
676 self.__deltaH = dataOut.getDeltaH()
677 677 self.__sample_rate = 1e6 * 0.15 / self.__deltaH
678 self.__dtype = dataOut.dtype
678 self.__dtype = dataOut.dtype
679 679 if len(dataOut.dtype) == 2:
680 680 self.__dtype = dataOut.dtype[0]
681 self.__nSamples = dataOut.systemHeaderObj.nSamples
681 self.__nSamples = dataOut.systemHeaderObj.nSamples
682 682 self.__nProfiles = dataOut.nProfiles
683 683
684 684 if self.dataOut.type != 'Voltage':
685 685 raise 'Digital RF cannot be used with this data type'
686 686 self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
687 687 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
688 688 else:
689 689 self.arr_data = numpy.ones((self.__nSamples, len(
690 690 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
691 691
692 file_cadence_millisecs = 1000
692 file_cadence_millisecs = 1000
693 693
694 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
695 sample_rate_numerator = int(sample_rate_fraction.numerator)
694 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
695 sample_rate_numerator = int(sample_rate_fraction.numerator)
696 696 sample_rate_denominator = int(sample_rate_fraction.denominator)
697 start_global_index = dataOut.utctime * self.__sample_rate
697 start_global_index = dataOut.utctime * self.__sample_rate
698 698
699 uuid = 'prueba'
699 uuid = 'prueba'
700 700 compression_level = 0
701 checksum = False
702 is_complex = True
703 num_subchannels = len(dataOut.channelList)
704 is_continuous = True
705 marching_periods = False
701 checksum = False
702 is_complex = True
703 num_subchannels = len(dataOut.channelList)
704 is_continuous = True
705 marching_periods = False
706 706
707 707 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
708 708 fileCadence, start_global_index,
709 709 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
710 710 is_complex, num_subchannels, is_continuous, marching_periods)
711 metadata_dir = os.path.join(path, 'metadata')
711 metadata_dir = os.path.join(path, 'metadata')
712 712 os.system('mkdir %s' % (metadata_dir))
713 713 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, # 236, file_cadence_millisecs / 1000
714 714 sample_rate_numerator, sample_rate_denominator,
715 715 metadataFile)
716 self.isConfig = True
716 self.isConfig = True
717 717 self.currentSample = 0
718 self.oldAverage = 0
719 self.count = 0
718 self.oldAverage = 0
719 self.count = 0
720 720 return
721 721
722 722 def writeMetadata(self):
723 start_idx = self.__sample_rate * self.dataOut.utctime
723 start_idx = self.__sample_rate * self.dataOut.utctime
724 724
725 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict(
725 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict(
726 726 )
727 727 self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
728 728 )
729 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict(
729 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict(
730 730 )
731 731 self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
732 732 return
733 733
734 734 def timeit(self, toExecute):
735 735 t0 = time()
736 736 toExecute()
737 self.executionTime = time() - t0
737 self.executionTime = time() - t0
738 738 if self.oldAverage is None:
739 739 self.oldAverage = self.executionTime
740 self.oldAverage = (self.executionTime + self.count *
740 self.oldAverage = (self.executionTime + self.count *
741 741 self.oldAverage) / (self.count + 1.0)
742 self.count = self.count + 1.0
742 self.count = self.count + 1.0
743 743 return
744 744
745 745 def writeData(self):
746 746 if self.dataOut.type != 'Voltage':
747 747 raise 'Digital RF cannot be used with this data type'
748 748 for channel in self.dataOut.channelList:
749 749 for i in range(self.dataOut.nFFTPoints):
750 self.arr_data[1][channel * self.dataOut.nFFTPoints +
750 self.arr_data[1][channel * self.dataOut.nFFTPoints +
751 751 i]['r'] = self.dataOut.data[channel][i].real
752 self.arr_data[1][channel * self.dataOut.nFFTPoints +
752 self.arr_data[1][channel * self.dataOut.nFFTPoints +
753 753 i]['i'] = self.dataOut.data[channel][i].imag
754 754 else:
755 755 for i in range(self.dataOut.systemHeaderObj.nSamples):
756 756 for channel in self.dataOut.channelList:
757 757 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
758 758 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
759 759
760 760 def f(): return self.digitalWriteObj.rf_write(self.arr_data)
761 761 self.timeit(f)
762 762
763 763 return
764 764
765 765 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
766 766 '''
767 767 This method will be called many times so here you should put all your code
768 768 Inputs:
769 769 dataOut: object with the data
770 770 '''
771 771 # print dataOut.__dict__
772 772 self.dataOut = dataOut
773 773 if not self.isConfig:
774 774 self.setup(dataOut, path, frequency, fileCadence,
775 775 dirCadence, metadataCadence, **kwargs)
776 776 self.writeMetadata()
777 777
778 778 self.writeData()
779 779
780 ## self.currentSample += 1
780 # # self.currentSample += 1
781 781 # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
782 782 # self.writeMetadata()
783 ## if self.currentSample == self.__nProfiles: self.currentSample = 0
783 # # if self.currentSample == self.__nProfiles: self.currentSample = 0
784 784
785 return dataOut# en la version 2.7 no aparece este return
785 return dataOut # en la version 2.7 no aparece este return
786 786
787 787 def close(self):
788 788 print('[Writing] - Closing files ')
789 789 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
790 790 try:
791 791 self.digitalWriteObj.close()
792 792 except:
793 793 pass
Show file before | Show file after | Hide comments
@@ -1,850 +1,850
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import os, sys
8 8 import time, datetime
9 9 import numpy
10 10 import fnmatch
11 11 import glob
12 12 from time import sleep
13 13
14 14 try:
15 15 import pyfits
16 16 except ImportError as e:
17 17 pass
18 18
19 19 from xml.etree.ElementTree import ElementTree
20 20
21 21 from .jroIO_base import isRadarFolder, isNumber
22 22 from schainpy.model.data.jrodata import Fits
23 23 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecorator
24 24 from schainpy.utils import log
25 25
26 26
27 27 class PyFits(object):
28 name=None
29 format=None
30 array =None
31 data =None
32 thdulist=None
33 prihdr=None
34 hdu=None
28 name = None
29 format = None
30 array = None
31 data = None
32 thdulist = None
33 prihdr = None
34 hdu = None
35 35
36 36 def __init__(self):
37 37
38 38 pass
39 39
40 def setColF(self,name,format,array):
41 self.name=name
42 self.format=format
43 self.array=array
44 a1=numpy.array([self.array],dtype=numpy.float32)
40 def setColF(self, name, format, array):
41 self.name = name
42 self.format = format
43 self.array = array
44 a1 = numpy.array([self.array], dtype=numpy.float32)
45 45 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
46 46 return self.col1
47 47
48 48 # def setColP(self,name,format,data):
49 49 # self.name=name
50 50 # self.format=format
51 51 # self.data=data
52 52 # a2=numpy.array([self.data],dtype=numpy.float32)
53 53 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
54 54 # return self.col2
55 55
56 56
57 def writeData(self,name,format,data):
58 self.name=name
59 self.format=format
60 self.data=data
61 a2=numpy.array([self.data],dtype=numpy.float32)
57 def writeData(self, name, format, data):
58 self.name = name
59 self.format = format
60 self.data = data
61 a2 = numpy.array([self.data], dtype=numpy.float32)
62 62 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
63 63 return self.col2
64 64
65 def cFImage(self,idblock,year,month,day,hour,minute,second):
66 self.hdu= pyfits.PrimaryHDU(idblock)
67 self.hdu.header.set("Year",year)
68 self.hdu.header.set("Month",month)
69 self.hdu.header.set("Day",day)
70 self.hdu.header.set("Hour",hour)
71 self.hdu.header.set("Minute",minute)
72 self.hdu.header.set("Second",second)
65 def cFImage(self, idblock, year, month, day, hour, minute, second):
66 self.hdu = pyfits.PrimaryHDU(idblock)
67 self.hdu.header.set("Year", year)
68 self.hdu.header.set("Month", month)
69 self.hdu.header.set("Day", day)
70 self.hdu.header.set("Hour", hour)
71 self.hdu.header.set("Minute", minute)
72 self.hdu.header.set("Second", second)
73 73 return self.hdu
74 74
75 75
76 def Ctable(self,colList):
77 self.cols=pyfits.ColDefs(colList)
76 def Ctable(self, colList):
77 self.cols = pyfits.ColDefs(colList)
78 78 self.tbhdu = pyfits.new_table(self.cols)
79 79 return self.tbhdu
80 80
81 81
82 def CFile(self,hdu,tbhdu):
83 self.thdulist=pyfits.HDUList([hdu,tbhdu])
82 def CFile(self, hdu, tbhdu):
83 self.thdulist = pyfits.HDUList([hdu, tbhdu])
84 84
85 def wFile(self,filename):
85 def wFile(self, filename):
86 86 if os.path.isfile(filename):
87 87 os.remove(filename)
88 88 self.thdulist.writeto(filename)
89 89
90 90
91 91 class ParameterConf:
92 92 ELEMENTNAME = 'Parameter'
93 93 def __init__(self):
94 94 self.name = ''
95 95 self.value = ''
96 96
97 97 def readXml(self, parmElement):
98 98 self.name = parmElement.get('name')
99 99 self.value = parmElement.get('value')
100 100
101 101 def getElementName(self):
102 102 return self.ELEMENTNAME
103 103
104 104 class Metadata(object):
105 105
106 106 def __init__(self, filename):
107 107 self.parmConfObjList = []
108 108 self.readXml(filename)
109 109
110 110 def readXml(self, filename):
111 111 self.projectElement = None
112 112 self.procUnitConfObjDict = {}
113 113 self.projectElement = ElementTree().parse(filename)
114 114 self.project = self.projectElement.tag
115 115
116 116 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
117 117
118 118 for parmElement in parmElementList:
119 119 parmConfObj = ParameterConf()
120 120 parmConfObj.readXml(parmElement)
121 121 self.parmConfObjList.append(parmConfObj)
122 122
123 123 @MPDecorator
124 124 class FitsWriter(Operation):
125 125 def __init__(self, **kwargs):
126 126 Operation.__init__(self, **kwargs)
127 127 self.isConfig = False
128 128 self.dataBlocksPerFile = None
129 129 self.blockIndex = 0
130 130 self.flagIsNewFile = 1
131 131 self.fitsObj = None
132 132 self.optchar = 'P'
133 133 self.ext = '.fits'
134 134 self.setFile = 0
135 135
136 136 def setFitsHeader(self, dataOut, metadatafile=None):
137 137
138 138 header_data = pyfits.PrimaryHDU()
139 139
140 140 header_data.header['EXPNAME'] = "RADAR DATA"
141 141 header_data.header['DATATYPE'] = "SPECTRA"
142 142 header_data.header['COMMENT'] = ""
143 143
144 144 if metadatafile:
145 145
146 146 metadata4fits = Metadata(metadatafile)
147 147
148 148 for parameter in metadata4fits.parmConfObjList:
149 149 parm_name = parameter.name
150 150 parm_value = parameter.value
151 151
152 152 header_data.header[parm_name] = parm_value
153 153
154 154 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
155 155 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
156 156 header_data.header['NCHANNELS'] = dataOut.nChannels
157 #header_data.header['HEIGHTS'] = dataOut.heightList
157 # header_data.header['HEIGHTS'] = dataOut.heightList
158 158 header_data.header['NHEIGHTS'] = dataOut.nHeights
159 159
160 160 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
161 161 header_data.header['NCOHINT'] = dataOut.nCohInt
162 162 header_data.header['NINCOHINT'] = dataOut.nIncohInt
163 163 header_data.header['TIMEZONE'] = dataOut.timeZone
164 164 header_data.header['NBLOCK'] = self.blockIndex
165 165
166 166 header_data.writeto(self.filename)
167 167
168 self.addExtension(dataOut.heightList,'HEIGHTLIST')
168 self.addExtension(dataOut.heightList, 'HEIGHTLIST')
169 169
170 170
171 171 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
172 172
173 173 self.path = path
174 174 self.dataOut = dataOut
175 175 self.metadatafile = metadatafile
176 176 self.dataBlocksPerFile = dataBlocksPerFile
177 177
178 178 def open(self):
179 179 self.fitsObj = pyfits.open(self.filename, mode='update')
180 180
181 181
182 182 def addExtension(self, data, tagname):
183 183 self.open()
184 184 extension = pyfits.ImageHDU(data=data, name=tagname)
185 #extension.header['TAG'] = tagname
185 # extension.header['TAG'] = tagname
186 186 self.fitsObj.append(extension)
187 187 self.write()
188 188
189 189 def addData(self, data):
190 190 self.open()
191 191 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
192 192 extension.header['UTCTIME'] = self.dataOut.utctime
193 193 self.fitsObj.append(extension)
194 194 self.blockIndex += 1
195 195 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
196 196
197 197 self.write()
198 198
199 199 def write(self):
200 200
201 201 self.fitsObj.flush(verbose=True)
202 202 self.fitsObj.close()
203 203
204 204
205 205 def setNextFile(self):
206 206
207 207 ext = self.ext
208 208 path = self.path
209 209
210 timeTuple = time.localtime( self.dataOut.utctime)
211 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
210 timeTuple = time.localtime(self.dataOut.utctime)
211 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
212 212
213 fullpath = os.path.join( path, subfolder )
214 if not( os.path.exists(fullpath) ):
213 fullpath = os.path.join(path, subfolder)
214 if not(os.path.exists(fullpath)):
215 215 os.mkdir(fullpath)
216 self.setFile = -1 #inicializo mi contador de seteo
216 self.setFile = -1 # inicializo mi contador de seteo
217 217 else:
218 filesList = os.listdir( fullpath )
219 if len( filesList ) > 0:
220 filesList = sorted( filesList, key=str.lower )
218 filesList = os.listdir(fullpath)
219 if len(filesList) > 0:
220 filesList = sorted(filesList, key=str.lower)
221 221 filen = filesList[-1]
222 222
223 if isNumber( filen[8:11] ):
224 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
223 if isNumber(filen[8:11]):
224 self.setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
225 225 else:
226 226 self.setFile = -1
227 227 else:
228 self.setFile = -1 #inicializo mi contador de seteo
228 self.setFile = -1 # inicializo mi contador de seteo
229 229
230 230 setFile = self.setFile
231 231 setFile += 1
232 232
233 233 thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
234 234 timeTuple.tm_year,
235 235 timeTuple.tm_yday,
236 236 setFile,
237 ext )
237 ext)
238 238
239 filename = os.path.join( path, subfolder, thisFile )
239 filename = os.path.join(path, subfolder, thisFile)
240 240
241 241 self.blockIndex = 0
242 242 self.filename = filename
243 243 self.setFile = setFile
244 244 self.flagIsNewFile = 1
245 245
246 print('Writing the file: %s'%self.filename)
246 print('Writing the file: %s' % self.filename)
247 247
248 248 self.setFitsHeader(self.dataOut, self.metadatafile)
249 249
250 250 return 1
251 251
252 252 def writeBlock(self):
253 253 self.addData(self.dataOut.data_spc)
254 254 self.flagIsNewFile = 0
255 255
256 256
257 257 def __setNewBlock(self):
258 258
259 259 if self.flagIsNewFile:
260 260 return 1
261 261
262 262 if self.blockIndex < self.dataBlocksPerFile:
263 263 return 1
264 264
265 if not( self.setNextFile() ):
265 if not(self.setNextFile()):
266 266 return 0
267 267
268 268 return 1
269 269
270 270 def writeNextBlock(self):
271 if not( self.__setNewBlock() ):
271 if not(self.__setNewBlock()):
272 272 return 0
273 273 self.writeBlock()
274 274 return 1
275 275
276 276 def putData(self):
277 277 if self.flagIsNewFile:
278 278 self.setNextFile()
279 279 self.writeNextBlock()
280 280
281 281 def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs):
282 282 if not(self.isConfig):
283 283 self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs)
284 284 self.isConfig = True
285 285 self.putData()
286 286
287 287
288 288 class FitsReader(ProcessingUnit):
289 289
290 290 # __TIMEZONE = time.timezone
291 291
292 292 expName = None
293 293 datetimestr = None
294 294 utc = None
295 295 nChannels = None
296 296 nSamples = None
297 297 dataBlocksPerFile = None
298 298 comments = None
299 299 lastUTTime = None
300 300 header_dict = None
301 301 data = None
302 302 data_header_dict = None
303 303
304 def __init__(self):#, **kwargs):
305 ProcessingUnit.__init__(self)#, **kwargs)
304 def __init__(self): # , **kwargs):
305 ProcessingUnit.__init__(self) # , **kwargs)
306 306 self.isConfig = False
307 307 self.ext = '.fits'
308 308 self.setFile = 0
309 309 self.flagNoMoreFiles = 0
310 310 self.flagIsNewFile = 1
311 311 self.flagDiscontinuousBlock = None
312 312 self.fileIndex = None
313 313 self.filename = None
314 314 self.fileSize = None
315 315 self.fitsObj = None
316 316 self.timeZone = None
317 317 self.nReadBlocks = 0
318 318 self.nTotalBlocks = 0
319 319 self.dataOut = self.createObjByDefault()
320 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
320 self.maxTimeStep = 10 # deberia ser definido por el usuario usando el metodo setup()
321 321 self.blockIndex = 1
322 322
323 323 def createObjByDefault(self):
324 324
325 325 dataObj = Fits()
326 326
327 327 return dataObj
328 328
329 329 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
330 330 try:
331 fitsObj = pyfits.open(filename,'readonly')
331 fitsObj = pyfits.open(filename, 'readonly')
332 332 except:
333 print("File %s can't be opened" %(filename))
333 print("File %s can't be opened" % (filename))
334 334 return None
335 335
336 336 header = fitsObj[0].header
337 337 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
338 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
338 utc = time.mktime(struct_time) - time.timezone # TIMEZONE debe ser un parametro del header FITS
339 339
340 340 ltc = utc
341 341 if useLocalTime:
342 342 ltc -= time.timezone
343 343 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
344 344 thisTime = thisDatetime.time()
345 345
346 346 if not ((startTime <= thisTime) and (endTime > thisTime)):
347 347 return None
348 348
349 349 return thisDatetime
350 350
351 351 def __setNextFileOnline(self):
352 352 raise NotImplementedError
353 353
354 354 def __setNextFileOffline(self):
355 355 idFile = self.fileIndex
356 356
357 357 while (True):
358 358 idFile += 1
359 359 if not(idFile < len(self.filenameList)):
360 360 self.flagNoMoreFiles = 1
361 361 print("No more Files")
362 362 return 0
363 363
364 364 filename = self.filenameList[idFile]
365 365
366 366 # if not(self.__verifyFile(filename)):
367 367 # continue
368 368
369 369 fileSize = os.path.getsize(filename)
370 fitsObj = pyfits.open(filename,'readonly')
370 fitsObj = pyfits.open(filename, 'readonly')
371 371 break
372 372
373 373 self.flagIsNewFile = 1
374 374 self.fileIndex = idFile
375 375 self.filename = filename
376 376 self.fileSize = fileSize
377 377 self.fitsObj = fitsObj
378 378 self.blockIndex = 0
379 print("Setting the file: %s"%self.filename)
379 print("Setting the file: %s" % self.filename)
380 380
381 381 return 1
382 382
383 383 def __setValuesFromHeader(self):
384 384
385 385 self.dataOut.header = self.header_dict
386 386 self.dataOut.expName = self.expName
387 387
388 388 self.dataOut.timeZone = self.timeZone
389 389 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
390 390 self.dataOut.comments = self.comments
391 391 # self.dataOut.timeInterval = self.timeInterval
392 392 self.dataOut.channelList = self.channelList
393 393 self.dataOut.heightList = self.heightList
394 394
395 395 self.dataOut.nCohInt = self.nCohInt
396 396 self.dataOut.nIncohInt = self.nIncohInt
397 397 self.dataOut.ipp_sec = self.ippSeconds
398 398
399 399 def readHeader(self):
400 400 headerObj = self.fitsObj[0]
401 401
402 402 self.header_dict = headerObj.header
403 403 if 'EXPNAME' in list(headerObj.header.keys()):
404 404 self.expName = headerObj.header['EXPNAME']
405 405
406 406 if 'DATATYPE' in list(headerObj.header.keys()):
407 407 self.dataType = headerObj.header['DATATYPE']
408 408
409 409 self.datetimestr = headerObj.header['DATETIME']
410 410 channelList = headerObj.header['CHANNELLIST']
411 411 channelList = channelList.split('[')
412 412 channelList = channelList[1].split(']')
413 413 channelList = channelList[0].split(',')
414 414 channelList = [int(ch) for ch in channelList]
415 415 self.channelList = channelList
416 416 self.nChannels = headerObj.header['NCHANNELS']
417 417 self.nHeights = headerObj.header['NHEIGHTS']
418 418 self.ippSeconds = headerObj.header['IPPSECONDS']
419 419 self.nCohInt = headerObj.header['NCOHINT']
420 420 self.nIncohInt = headerObj.header['NINCOHINT']
421 421 self.dataBlocksPerFile = headerObj.header['NBLOCK']
422 422 self.timeZone = headerObj.header['TIMEZONE']
423 423
424 424 # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
425 425
426 426 if 'COMMENT' in list(headerObj.header.keys()):
427 427 self.comments = headerObj.header['COMMENT']
428 428
429 429 self.readHeightList()
430 430
431 431 def readHeightList(self):
432 432 self.blockIndex = self.blockIndex + 1
433 433 obj = self.fitsObj[self.blockIndex]
434 434 self.heightList = obj.data
435 435 self.blockIndex = self.blockIndex + 1
436 436
437 437 def readExtension(self):
438 438 obj = self.fitsObj[self.blockIndex]
439 439 self.heightList = obj.data
440 440 self.blockIndex = self.blockIndex + 1
441 441
442 442 def setNextFile(self):
443 443
444 444 if self.online:
445 445 newFile = self.__setNextFileOnline()
446 446 else:
447 447 newFile = self.__setNextFileOffline()
448 448
449 449 if not(newFile):
450 450 return 0
451 451
452 452 self.readHeader()
453 453 self.__setValuesFromHeader()
454 454 self.nReadBlocks = 0
455 455 # self.blockIndex = 1
456 456 return 1
457 457
458 458 def searchFilesOffLine(self,
459 459 path,
460 460 startDate,
461 461 endDate,
462 startTime=datetime.time(0,0,0),
463 endTime=datetime.time(23,59,59),
462 startTime=datetime.time(0, 0, 0),
463 endTime=datetime.time(23, 59, 59),
464 464 set=None,
465 465 expLabel='',
466 466 ext='.fits',
467 467 walk=True):
468 468
469 469 pathList = []
470 470
471 471 if not walk:
472 472 pathList.append(path)
473 473
474 474 else:
475 475 dirList = []
476 476 for thisPath in os.listdir(path):
477 if not os.path.isdir(os.path.join(path,thisPath)):
477 if not os.path.isdir(os.path.join(path, thisPath)):
478 478 continue
479 479 if not isRadarFolder(thisPath):
480 480 continue
481 481
482 482 dirList.append(thisPath)
483 483
484 484 if not(dirList):
485 485 return None, None
486 486
487 487 thisDate = startDate
488 488
489 489 while(thisDate <= endDate):
490 490 year = thisDate.timetuple().tm_year
491 491 doy = thisDate.timetuple().tm_yday
492 492
493 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
493 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year, doy) + '*')
494 494 if len(matchlist) == 0:
495 495 thisDate += datetime.timedelta(1)
496 496 continue
497 497 for match in matchlist:
498 pathList.append(os.path.join(path,match,expLabel))
498 pathList.append(os.path.join(path, match, expLabel))
499 499
500 500 thisDate += datetime.timedelta(1)
501 501
502 502 if pathList == []:
503 print("Any folder was found for the date range: %s-%s" %(startDate, endDate))
503 print("Any folder was found for the date range: %s-%s" % (startDate, endDate))
504 504 return None, None
505 505
506 print("%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate))
506 print("%d folder(s) was(were) found for the date range: %s - %s" % (len(pathList), startDate, endDate))
507 507
508 508 filenameList = []
509 509 datetimeList = []
510 510
511 511 for i in range(len(pathList)):
512 512
513 513 thisPath = pathList[i]
514 514
515 fileList = glob.glob1(thisPath, "*%s" %ext)
515 fileList = glob.glob1(thisPath, "*%s" % ext)
516 516 fileList.sort()
517 517
518 518 for thisFile in fileList:
519 519
520 filename = os.path.join(thisPath,thisFile)
520 filename = os.path.join(thisPath, thisFile)
521 521 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
522 522
523 523 if not(thisDatetime):
524 524 continue
525 525
526 526 filenameList.append(filename)
527 527 datetimeList.append(thisDatetime)
528 528
529 529 if not(filenameList):
530 print("Any file was found for the time range %s - %s" %(startTime, endTime))
530 print("Any file was found for the time range %s - %s" % (startTime, endTime))
531 531 return None, None
532 532
533 print("%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime))
533 print("%d file(s) was(were) found for the time range: %s - %s" % (len(filenameList), startTime, endTime))
534 534 print()
535 535
536 536 for i in range(len(filenameList)):
537 print("%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()))
537 print("%s -> [%s]" % (filenameList[i], datetimeList[i].ctime()))
538 538
539 539 self.filenameList = filenameList
540 540 self.datetimeList = datetimeList
541 541
542 542 return pathList, filenameList
543 543
544 544 def setup(self, path=None,
545 545 startDate=None,
546 546 endDate=None,
547 startTime=datetime.time(0,0,0),
548 endTime=datetime.time(23,59,59),
547 startTime=datetime.time(0, 0, 0),
548 endTime=datetime.time(23, 59, 59),
549 549 set=0,
550 expLabel = "",
551 ext = None,
552 online = False,
553 delay = 60,
554 walk = True):
550 expLabel="",
551 ext=None,
552 online=False,
553 delay=60,
554 walk=True):
555 555
556 556 if path == None:
557 557 raise ValueError("The path is not valid")
558 558
559 559 if ext == None:
560 560 ext = self.ext
561 561
562 562 if not(online):
563 563 print("Searching files in offline mode ...")
564 564 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
565 565 startTime=startTime, endTime=endTime,
566 566 set=set, expLabel=expLabel, ext=ext,
567 567 walk=walk)
568 568
569 569 if not(pathList):
570 print("No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
571 datetime.datetime.combine(startDate,startTime).ctime(),
572 datetime.datetime.combine(endDate,endTime).ctime()))
570 print("No *%s files into the folder %s \nfor the range: %s - %s" % (ext, path,
571 datetime.datetime.combine(startDate, startTime).ctime(),
572 datetime.datetime.combine(endDate, endTime).ctime()))
573 573
574 574 sys.exit(-1)
575 575
576 576 self.fileIndex = -1
577 577 self.pathList = pathList
578 578 self.filenameList = filenameList
579 579
580 580 self.online = online
581 581 self.delay = delay
582 582 ext = ext.lower()
583 583 self.ext = ext
584 584
585 585 if not(self.setNextFile()):
586 if (startDate!=None) and (endDate!=None):
587 print("No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
586 if (startDate != None) and (endDate != None):
587 print("No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
588 588 elif startDate != None:
589 print("No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()))
589 print("No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
590 590 else:
591 591 print("No files")
592 592
593 593 sys.exit(-1)
594 594
595 595
596 596
597 597 def readBlock(self):
598 598 dataObj = self.fitsObj[self.blockIndex]
599 599
600 600 self.data = dataObj.data
601 601 self.data_header_dict = dataObj.header
602 602 self.utc = self.data_header_dict['UTCTIME']
603 603
604 604 self.flagIsNewFile = 0
605 605 self.blockIndex += 1
606 606 self.nTotalBlocks += 1
607 607 self.nReadBlocks += 1
608 608
609 609 return 1
610 610
611 611 def __jumpToLastBlock(self):
612 612 raise NotImplementedError
613 613
614 614 def __waitNewBlock(self):
615 615 """
616 616 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
617 617
618 618 Si el modo de lectura es OffLine siempre retorn 0
619 619 """
620 620 if not self.online:
621 621 return 0
622 622
623 623 if (self.nReadBlocks >= self.dataBlocksPerFile):
624 624 return 0
625 625
626 626 currentPointer = self.fp.tell()
627 627
628 628 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
629 629
630 for nTries in range( self.nTries ):
630 for nTries in range(self.nTries):
631 631
632 632 self.fp.close()
633 self.fp = open( self.filename, 'rb' )
634 self.fp.seek( currentPointer )
633 self.fp = open(self.filename, 'rb')
634 self.fp.seek(currentPointer)
635 635
636 self.fileSize = os.path.getsize( self.filename )
636 self.fileSize = os.path.getsize(self.filename)
637 637 currentSize = self.fileSize - currentPointer
638 638
639 if ( currentSize >= neededSize ):
639 if (currentSize >= neededSize):
640 640 self.__rdBasicHeader()
641 641 return 1
642 642
643 print("\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1))
644 sleep( self.delay )
643 print("\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
644 sleep(self.delay)
645 645
646 646
647 647 return 0
648 648
649 649 def __setNewBlock(self):
650 650
651 651 if self.online:
652 652 self.__jumpToLastBlock()
653 653
654 654 if self.flagIsNewFile:
655 655 return 1
656 656
657 657 self.lastUTTime = self.utc
658 658
659 659 if self.online:
660 660 if self.__waitNewBlock():
661 661 return 1
662 662
663 663 if self.nReadBlocks < self.dataBlocksPerFile:
664 664 return 1
665 665
666 666 if not(self.setNextFile()):
667 667 return 0
668 668
669 669 deltaTime = self.utc - self.lastUTTime
670 670
671 671 self.flagDiscontinuousBlock = 0
672 672
673 673 if deltaTime > self.maxTimeStep:
674 674 self.flagDiscontinuousBlock = 1
675 675
676 676 return 1
677 677
678 678
679 679 def readNextBlock(self):
680 680 if not(self.__setNewBlock()):
681 681 return 0
682 682
683 683 if not(self.readBlock()):
684 684 return 0
685 685
686 686 return 1
687 687
688 688 def printInfo(self):
689 689
690 690 pass
691 691
692 692 def getData(self):
693 693
694 694 if self.flagNoMoreFiles:
695 695 self.dataOut.flagNoData = True
696 696 return (0, 'No more files')
697 697
698 698 self.flagDiscontinuousBlock = 0
699 699 self.flagIsNewBlock = 0
700 700
701 701 if not(self.readNextBlock()):
702 702 return (1, 'Error reading data')
703 703
704 704 if self.data is None:
705 705 self.dataOut.flagNoData = True
706 706 return (0, 'No more data')
707 707
708 708 self.dataOut.data = self.data
709 709 self.dataOut.data_header = self.data_header_dict
710 710 self.dataOut.utctime = self.utc
711 711
712 712 # self.dataOut.header = self.header_dict
713 713 # self.dataOut.expName = self.expName
714 714 # self.dataOut.nChannels = self.nChannels
715 715 # self.dataOut.timeZone = self.timeZone
716 716 # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
717 717 # self.dataOut.comments = self.comments
718 718 # # self.dataOut.timeInterval = self.timeInterval
719 719 # self.dataOut.channelList = self.channelList
720 720 # self.dataOut.heightList = self.heightList
721 721 self.dataOut.flagNoData = False
722 722 # return self.dataOut.data
723 723
724 724 def run(self, **kwargs):
725 725
726 726 if not(self.isConfig):
727 727 self.setup(**kwargs)
728 728 self.isConfig = True
729 729
730 730 self.getData()
731 731
732 732 @MPDecorator
733 733 class SpectraHeisWriter(Operation):
734 734 # set = None
735 735 setFile = None
736 736 idblock = None
737 737 doypath = None
738 738 subfolder = None
739 739
740 def __init__(self):#, **kwargs):
741 Operation.__init__(self)#, **kwargs)
740 def __init__(self): # , **kwargs):
741 Operation.__init__(self) # , **kwargs)
742 742 self.wrObj = PyFits()
743 743 # self.dataOut = dataOut
744 self.nTotalBlocks=0
744 self.nTotalBlocks = 0
745 745 # self.set = None
746 746 self.setFile = None
747 747 self.idblock = 0
748 748 self.wrpath = None
749 749 self.doypath = None
750 750 self.subfolder = None
751 751 self.isConfig = False
752 752
753 753 def isNumber(str):
754 754 """
755 755 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
756 756
757 757 Excepciones:
758 758 Si un determinado string no puede ser convertido a numero
759 759 Input:
760 760 str, string al cual se le analiza para determinar si convertible a un numero o no
761 761
762 762 Return:
763 763 True : si el string es uno numerico
764 764 False : no es un string numerico
765 765 """
766 766 try:
767 float( str )
767 float(str)
768 768 return True
769 769 except:
770 770 return False
771 771
772 772 def setup(self, dataOut, wrpath):
773 773
774 774 if not(os.path.exists(wrpath)):
775 775 os.mkdir(wrpath)
776 776
777 777 self.wrpath = wrpath
778 778 # self.setFile = 0
779 779 self.dataOut = dataOut
780 780
781 781 def putData(self):
782 name= time.localtime( self.dataOut.utctime)
783 ext=".fits"
782 name = time.localtime(self.dataOut.utctime)
783 ext = ".fits"
784 784
785 785 if self.doypath == None:
786 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
787 self.doypath = os.path.join( self.wrpath, self.subfolder )
786 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year, name.tm_yday, time.mktime(datetime.datetime.now().timetuple()))
787 self.doypath = os.path.join(self.wrpath, self.subfolder)
788 788 os.mkdir(self.doypath)
789 789
790 790 if self.setFile == None:
791 791 # self.set = self.dataOut.set
792 792 self.setFile = 0
793 793 # if self.set != self.dataOut.set:
794 ## self.set = self.dataOut.set
794 # # self.set = self.dataOut.set
795 795 # self.setFile = 0
796 796
797 #make the filename
798 thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
797 # make the filename
798 thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year, name.tm_yday, self.setFile, ext)
799 799
800 filename = os.path.join(self.wrpath,self.subfolder, thisFile)
800 filename = os.path.join(self.wrpath, self.subfolder, thisFile)
801 801
802 idblock = numpy.array([self.idblock],dtype="int64")
803 header=self.wrObj.cFImage(idblock=idblock,
802 idblock = numpy.array([self.idblock], dtype="int64")
803 header = self.wrObj.cFImage(idblock=idblock,
804 804 year=time.gmtime(self.dataOut.utctime).tm_year,
805 805 month=time.gmtime(self.dataOut.utctime).tm_mon,
806 806 day=time.gmtime(self.dataOut.utctime).tm_mday,
807 807 hour=time.gmtime(self.dataOut.utctime).tm_hour,
808 808 minute=time.gmtime(self.dataOut.utctime).tm_min,
809 809 second=time.gmtime(self.dataOut.utctime).tm_sec)
810 810
811 c=3E8
811 c = 3E8
812 812 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
813 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
813 freq = numpy.arange(-1 * self.dataOut.nHeights / 2., self.dataOut.nHeights / 2.) * (c / (2 * deltaHeight * 1000))
814 814
815 815 colList = []
816 816
817 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
817 colFreq = self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints) + 'E', array=freq)
818 818
819 819 colList.append(colFreq)
820 820
821 nchannel=self.dataOut.nChannels
821 nchannel = self.dataOut.nChannels
822 822
823 823 for i in range(nchannel):
824 col = self.wrObj.writeData(name="PCh"+str(i+1),
825 format=str(self.dataOut.nFFTPoints)+'E',
826 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
824 col = self.wrObj.writeData(name="PCh" + str(i + 1),
825 format=str(self.dataOut.nFFTPoints) + 'E',
826 data=10 * numpy.log10(self.dataOut.data_spc[i, :]))
827 827
828 828 colList.append(col)
829 829
830 data=self.wrObj.Ctable(colList=colList)
830 data = self.wrObj.Ctable(colList=colList)
831 831
832 self.wrObj.CFile(header,data)
832 self.wrObj.CFile(header, data)
833 833
834 834 self.wrObj.wFile(filename)
835 835
836 #update the setFile
836 # update the setFile
837 837 self.setFile += 1
838 838 self.idblock += 1
839 839
840 840 return 1
841 841
842 842 def run(self, dataOut, **kwargs):
843 843
844 844 if not(self.isConfig):
845 845
846 846 self.setup(dataOut, **kwargs)
847 847 self.isConfig = True
848 848
849 849 self.putData()
850 return dataOut No newline at end of file
850 return dataOut
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (506 lines changed) Show them Hide them
@@ -1,862 +1,862
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-com0419
5 5 '''
6 6
7 import os,sys
8 import time,datetime
7 import os, sys
8 import time, datetime
9 9 import h5py
10 10 import numpy
11 11 import fnmatch
12 12 import re
13 13
14 14 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 15 from schainpy.model.data.jrodata import Voltage
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
17 17
18 18
19 19 def isNumber(str):
20 20 """
21 21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22 22
23 23 Excepciones:
24 24 Si un determinado string no puede ser convertido a numero
25 25 Input:
26 26 str, string al cual se le analiza para determinar si convertible a un numero o no
27 27
28 28 Return:
29 29 True : si el string es uno numerico
30 30 False : no es un string numerico
31 31 """
32 32 try:
33 float( str )
33 float(str)
34 34 return True
35 35 except:
36 36 return False
37 37
38 38 def getFileFromSet(path, ext, set=None):
39 39 validFilelist = []
40 40 fileList = os.listdir(path)
41 41
42 42
43 43 if len(fileList) < 1:
44 44 return None
45 45
46 46 # 0 1234 567 89A BCDE
47 47 # H YYYY DDD SSS .ext
48 48
49 49 for thisFile in fileList:
50 50 try:
51 number= int(thisFile[6:16])
51 number = int(thisFile[6:16])
52 52
53 53 # year = int(thisFile[1:5])
54 54 # doy = int(thisFile[5:8])
55 55 except:
56 56 continue
57 57
58 58 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
59 59 continue
60 60
61 61 validFilelist.append(thisFile)
62 62
63 63 if len(validFilelist) < 1:
64 64 return None
65 65
66 validFilelist = sorted( validFilelist, key=str.lower )
66 validFilelist = sorted(validFilelist, key=str.lower)
67 67
68 68 if set == None:
69 69 return validFilelist[-1]
70 70
71 print("set =" ,set)
71 print("set =" , set)
72 72 for thisFile in validFilelist:
73 73 if set <= int(thisFile[6:16]):
74 print(thisFile,int(thisFile[6:16]))
74 print(thisFile, int(thisFile[6:16]))
75 75 return thisFile
76 76
77 77 return validFilelist[-1]
78 78
79 myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
80 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
79 myfile = fnmatch.filter(validFilelist, '*%10d*' % (set))
80 # myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
81 81
82 if len(myfile)!= 0:
82 if len(myfile) != 0:
83 83 return myfile[0]
84 84 else:
85 filename = '*%10.10d%s'%(set,ext.lower())
86 print('the filename %s does not exist'%filename)
85 filename = '*%10.10d%s' % (set, ext.lower())
86 print('the filename %s does not exist' % filename)
87 87 print('...going to the last file: ')
88 88
89 89 if validFilelist:
90 validFilelist = sorted( validFilelist, key=str.lower )
90 validFilelist = sorted(validFilelist, key=str.lower)
91 91 return validFilelist[-1]
92 92
93 93 return None
94 94
95 95 def getlastFileFromPath(path, ext):
96 96 """
97 97 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
98 98 al final de la depuracion devuelve el ultimo file de la lista que quedo.
99 99
100 100 Input:
101 101 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
102 102 ext : extension de los files contenidos en una carpeta
103 103
104 104 Return:
105 105 El ultimo file de una determinada carpeta, no se considera el path.
106 106 """
107 107 validFilelist = []
108 108 fileList = os.listdir(path)
109 109
110 110 # 0 1234 567 89A BCDE
111 111 # H YYYY DDD SSS .ext
112 112
113 113 for thisFile in fileList:
114 114
115 115 try:
116 number= int(thisFile[6:16])
116 number = int(thisFile[6:16])
117 117 except:
118 118 print("There is a file or folder with different format")
119 119 if not isNumber(number):
120 120 continue
121 121
122 122 # year = thisFile[1:5]
123 123 # if not isNumber(year):
124 124 # continue
125 125
126 126 # doy = thisFile[5:8]
127 127 # if not isNumber(doy):
128 128 # continue
129 129
130 number= int(number)
130 number = int(number)
131 131 # year = int(year)
132 132 # doy = int(doy)
133 133
134 134 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
135 135 continue
136 136
137 137
138 138 validFilelist.append(thisFile)
139 139
140 140
141 141 if validFilelist:
142 validFilelist = sorted( validFilelist, key=str.lower )
142 validFilelist = sorted(validFilelist, key=str.lower)
143 143 return validFilelist[-1]
144 144
145 145 return None
146 146
147 147
148 148
149 149 class HFReader(ProcessingUnit):
150 150 '''
151 151 classdocs
152 152 '''
153 path = None
154 startDate= None
155 endDate = None
156 startTime= None
157 endTime = None
158 walk = None
153 path = None
154 startDate = None
155 endDate = None
156 startTime = None
157 endTime = None
158 walk = None
159 159 isConfig = False
160 dataOut=None
160 dataOut = None
161 161 nTries = 3
162 ext = ".hdf5"
162 ext = ".hdf5"
163 163
164 164 def __init__(self, **kwargs):
165 165 '''
166 166 Constructor
167 167 '''
168 168 ProcessingUnit.__init__(self, **kwargs)
169 169
170 self.isConfig =False
170 self.isConfig = False
171 171
172 172 self.datablock = None
173 173
174 self.filename_current=None
174 self.filename_current = None
175 175
176 176 self.utc = 0
177 177
178 self.ext='.hdf5'
178 self.ext = '.hdf5'
179 179
180 180 self.flagIsNewFile = 1
181 181
182 182 #-------------------------------------------------
183 self.fileIndex=None
183 self.fileIndex = None
184 184
185 self.profileIndex_offset=None
185 self.profileIndex_offset = None
186 186
187 self.filenameList=[]
187 self.filenameList = []
188 188
189 self.hfFilePointer= None
189 self.hfFilePointer = None
190 190
191 191 self.filename_online = None
192 192
193 self.status=True
193 self.status = True
194 194
195 self.flagNoMoreFiles= False
195 self.flagNoMoreFiles = False
196 196
197 197 self.__waitForNewFile = 20
198 198
199 199
200 200 #--------------------------------------------------
201 201
202 202 self.dataOut = self.createObjByDefault()
203 203
204 204
205 205 def createObjByDefault(self):
206 206
207 207 dataObj = Voltage()
208 208
209 209 return dataObj
210 210
211 211 def setObjProperties(self):
212 212
213 213 pass
214 214
215 215 def getBlockDimension(self):
216 216 """
217 217 Obtiene la cantidad de puntos a leer por cada bloque de datos
218 218
219 219 Affected:
220 220 self.blocksize
221 221
222 222 Return:
223 223 None
224 224 """
225 pts2read =self.nChannels*self.nHeights*self.nProfiles
225 pts2read = self.nChannels * self.nHeights * self.nProfiles
226 226 self.blocksize = pts2read
227 227
228 228 def __readHeader(self):
229 229
230 230 self.nProfiles = 100
231 231 self.nHeights = 1000
232 232 self.nChannels = 2
233 self.__firstHeigth=0
234 self.__nSamples=1000
235 self.__deltaHeigth=1.5
236 self.__sample_rate=1e5
237 #self.__frequency=2.72e6
238 #self.__frequency=3.64e6
239 self.__frequency=None
233 self.__firstHeigth = 0
234 self.__nSamples = 1000
235 self.__deltaHeigth = 1.5
236 self.__sample_rate = 1e5
237 # self.__frequency=2.72e6
238 # self.__frequency=3.64e6
239 self.__frequency = None
240 240 self.__online = False
241 self.filename_next_set=None
241 self.filename_next_set = None
242 242
243 #print "Frequency of Operation:", self.__frequency
243 # print "Frequency of Operation:", self.__frequency
244 244
245 245
246 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
246 def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''):
247 247 self.path = path
248 248 self.startDate = startDate
249 249 self.endDate = endDate
250 250 self.startTime = startTime
251 251 self.endTime = endTime
252 252 self.walk = walk
253 253
254 254 def __checkPath(self):
255 255 if os.path.exists(self.path):
256 self.status=1
256 self.status = 1
257 257 else:
258 self.status=0
259 print('Path %s does not exits'%self.path)
258 self.status = 0
259 print('Path %s does not exits' % self.path)
260 260 return
261 261 return
262 262
263 263 def __selDates(self, hf_dirname_format):
264 264 try:
265 dir_hf_filename= self.path+"/"+hf_dirname_format
266 fp= h5py.File(dir_hf_filename,'r')
267 hipoc=fp['t'].value
265 dir_hf_filename = self.path + "/" + hf_dirname_format
266 fp = h5py.File(dir_hf_filename, 'r')
267 hipoc = fp['t'].value
268 268 fp.close()
269 date_time=datetime.datetime.utcfromtimestamp(hipoc)
270 year =int(date_time[0:4])
271 month=int(date_time[5:7])
272 dom =int(date_time[8:10])
273 thisDate= datetime.date(year,month,dom)
274 if (thisDate>=self.startDate and thisDate <= self.endDate):
269 date_time = datetime.datetime.utcfromtimestamp(hipoc)
270 year = int(date_time[0:4])
271 month = int(date_time[5:7])
272 dom = int(date_time[8:10])
273 thisDate = datetime.date(year, month, dom)
274 if (thisDate >= self.startDate and thisDate <= self.endDate):
275 275 return hf_dirname_format
276 276 except:
277 277 return None
278 278
279 def __findDataForDates(self,online=False):
279 def __findDataForDates(self, online=False):
280 280 if not(self.status):
281 281 return None
282 282
283 283 pat = '\d+.\d+'
284 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
285 dirnameList = [x for x in dirnameList if x!=None]
284 dirnameList = [re.search(pat, x) for x in os.listdir(self.path)]
285 dirnameList = [x for x in dirnameList if x != None]
286 286 dirnameList = [x.string for x in dirnameList]
287 287 if not(online):
288 288
289 289 dirnameList = [self.__selDates(x) for x in dirnameList]
290 dirnameList = [x for x in dirnameList if x!=None]
290 dirnameList = [x for x in dirnameList if x != None]
291 291
292 if len(dirnameList)>0:
292 if len(dirnameList) > 0:
293 293 self.status = 1
294 294 self.dirnameList = dirnameList
295 295 self.dirnameList.sort()
296 296
297 297 else:
298 298 self.status = 0
299 299 return None
300 300
301 301 def __getTimeFromData(self):
302 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
303 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
304 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
302 startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime)
303 endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime)
304 print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader))
305 305 print('........................................')
306 filter_filenameList=[]
306 filter_filenameList = []
307 307 self.filenameList.sort()
308 for i in range(len(self.filenameList)-1):
309 filename=self.filenameList[i]
310 dir_hf_filename= filename
311 fp= h5py.File(dir_hf_filename,'r')
312 hipoc=fp['t'].value
313 hipoc=hipoc+self.timezone
314 date_time=datetime.datetime.utcfromtimestamp(hipoc)
308 for i in range(len(self.filenameList) - 1):
309 filename = self.filenameList[i]
310 dir_hf_filename = filename
311 fp = h5py.File(dir_hf_filename, 'r')
312 hipoc = fp['t'].value
313 hipoc = hipoc + self.timezone
314 date_time = datetime.datetime.utcfromtimestamp(hipoc)
315 315 fp.close()
316 year =int(date_time[0:4])
317 month=int(date_time[5:7])
318 dom =int(date_time[8:10])
319 hour =int(date_time[11:13])
320 min =int(date_time[14:16])
321 sec =int(date_time[17:19])
322 this_time=datetime.datetime(year,month,dom,hour,min,sec)
323 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
316 year = int(date_time[0:4])
317 month = int(date_time[5:7])
318 dom = int(date_time[8:10])
319 hour = int(date_time[11:13])
320 min = int(date_time[14:16])
321 sec = int(date_time[17:19])
322 this_time = datetime.datetime(year, month, dom, hour, min, sec)
323 if (this_time >= startDateTime_Reader and this_time <= endDateTime_Reader):
324 324 filter_filenameList.append(filename)
325 325 filter_filenameList.sort()
326 326 self.filenameList = filter_filenameList
327 327 return 1
328 328
329 329 def __getFilenameList(self):
330 #print "hola"
331 #print self.dirnameList
332 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
333 self.filenameList= dirList
334 #print self.filenameList
335 #print "pase",len(self.filenameList)
330 # print "hola"
331 # print self.dirnameList
332 dirList = [os.path.join(self.path, x) for x in self.dirnameList]
333 self.filenameList = dirList
334 # print self.filenameList
335 # print "pase",len(self.filenameList)
336 336
337 337 def __selectDataForTimes(self, online=False):
338 338
339 339 if not(self.status):
340 340 return None
341 341 #----------------
342 342 self.__getFilenameList()
343 343 #----------------
344 344 if not(online):
345 345 if not(self.all):
346 346 self.__getTimeFromData()
347 if len(self.filenameList)>0:
348 self.status=1
347 if len(self.filenameList) > 0:
348 self.status = 1
349 349 self.filenameList.sort()
350 350 else:
351 self.status=0
351 self.status = 0
352 352 return None
353 353 else:
354 354 if self.set != None:
355 355
356 filename=getFileFromSet(self.path,self.ext,self.set)
356 filename = getFileFromSet(self.path, self.ext, self.set)
357 357
358 if self.flag_nextfile==True:
359 self.dirnameList=[filename]
360 fullfilename=self.path+"/"+filename
361 self.filenameList=[fullfilename]
362 self.filename_next_set=int(filename[6:16])+10
358 if self.flag_nextfile == True:
359 self.dirnameList = [filename]
360 fullfilename = self.path + "/" + filename
361 self.filenameList = [fullfilename]
362 self.filename_next_set = int(filename[6:16]) + 10
363 363
364 self.flag_nextfile=False
364 self.flag_nextfile = False
365 365 else:
366 366 print(filename)
367 367 print("PRIMERA CONDICION")
368 #if self.filename_next_set== int(filename[6:16]):
368 # if self.filename_next_set== int(filename[6:16]):
369 369 print("TODO BIEN")
370 370
371 371 if filename == None:
372 372 raise ValueError("corregir")
373 373
374 self.dirnameList=[filename]
375 fullfilename=self.path+"/"+filename
376 self.filenameList=[fullfilename]
377 self.filename_next_set=int(filename[6:16])+10
378 print("Setting next file",self.filename_next_set)
379 self.set=int(filename[6:16])
374 self.dirnameList = [filename]
375 fullfilename = self.path + "/" + filename
376 self.filenameList = [fullfilename]
377 self.filename_next_set = int(filename[6:16]) + 10
378 print("Setting next file", self.filename_next_set)
379 self.set = int(filename[6:16])
380 380 if True:
381 381 pass
382 382 else:
383 383 print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
384 384
385 385 else:
386 filename =getlastFileFromPath(self.path,self.ext)
386 filename = getlastFileFromPath(self.path, self.ext)
387 387
388 if self.flag_nextfile==True:
389 self.dirnameList=[filename]
390 fullfilename=self.path+"/"+filename
391 self.filenameList=[self.filenameList[-1]]
392 self.filename_next_set=int(filename[6:16])+10
388 if self.flag_nextfile == True:
389 self.dirnameList = [filename]
390 fullfilename = self.path + "/" + filename
391 self.filenameList = [self.filenameList[-1]]
392 self.filename_next_set = int(filename[6:16]) + 10
393 393
394 self.flag_nextfile=False
394 self.flag_nextfile = False
395 395 else:
396 filename=getFileFromSet(self.path,self.ext,self.set)
396 filename = getFileFromSet(self.path, self.ext, self.set)
397 397 print(filename)
398 398 print("PRIMERA CONDICION")
399 #if self.filename_next_set== int(filename[6:16]):
399 # if self.filename_next_set== int(filename[6:16]):
400 400 print("TODO BIEN")
401 401
402 402 if filename == None:
403 403 raise ValueError("corregir")
404 404
405 self.dirnameList=[filename]
406 fullfilename=self.path+"/"+filename
407 self.filenameList=[fullfilename]
408 self.filename_next_set=int(filename[6:16])+10
409 print("Setting next file",self.filename_next_set)
410 self.set=int(filename[6:16])
405 self.dirnameList = [filename]
406 fullfilename = self.path + "/" + filename
407 self.filenameList = [fullfilename]
408 self.filename_next_set = int(filename[6:16]) + 10
409 print("Setting next file", self.filename_next_set)
410 self.set = int(filename[6:16])
411 411 if True:
412 412 pass
413 413 else:
414 414 print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
415 415
416 416
417 417
418 418 def searchFilesOffLine(self,
419 419 path,
420 420 startDate,
421 421 endDate,
422 422 ext,
423 startTime=datetime.time(0,0,0),
424 endTime=datetime.time(23,59,59),
423 startTime=datetime.time(0, 0, 0),
424 endTime=datetime.time(23, 59, 59),
425 425 walk=True):
426 426
427 427 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
428 428
429 429 self.__checkPath()
430 430
431 431 self.__findDataForDates()
432 #print self.dirnameList
432 # print self.dirnameList
433 433
434 434 self.__selectDataForTimes()
435 435
436 436 for i in range(len(self.filenameList)):
437 print("%s"% (self.filenameList[i]))
437 print("%s" % (self.filenameList[i]))
438 438
439 439 return
440 440
441 441 def searchFilesOnLine(self,
442 442 path,
443 expLabel= "",
443 expLabel="",
444 444 ext=None,
445 445 startDate=None,
446 446 endDate=None,
447 447 walk=True,
448 448 set=None):
449 449
450 450
451 451 startDate = datetime.datetime.utcnow().date()
452 452 endDate = datetime.datetime.utcnow().date()
453 453
454 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
454 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
455 455
456 456 self.__checkPath()
457 457
458 fullpath=path
459 print("%s folder was found: " %(fullpath ))
458 fullpath = path
459 print("%s folder was found: " % (fullpath))
460 460
461 461 if set == None:
462 self.set=None
463 filename =getlastFileFromPath(fullpath,ext)
464 startDate= datetime.datetime.utcnow().date
465 endDate= datetime.datetime.utcnow().date()
462 self.set = None
463 filename = getlastFileFromPath(fullpath, ext)
464 startDate = datetime.datetime.utcnow().date
465 endDate = datetime.datetime.utcnow().date()
466 466 #
467 467 else:
468 filename= getFileFromSet(fullpath,ext,set)
469 startDate=None
470 endDate=None
468 filename = getFileFromSet(fullpath, ext, set)
469 startDate = None
470 endDate = None
471 471 #
472 472 if not (filename):
473 return None,None,None,None,None
474 #print "%s file was found" %(filename)
473 return None, None, None, None, None
474 # print "%s file was found" %(filename)
475 475
476 476 #
477 477 # dir_hf_filename= self.path+"/"+filename
478 478 # fp= h5py.File(dir_hf_filename,'r')
479 479 # hipoc=fp['t'].value
480 480 # fp.close()
481 481 # date_time=datetime.datetime.utcfromtimestamp(hipoc)
482 482 #
483 483 # year =int(date_time[0:4])
484 484 # month=int(date_time[5:7])
485 485 # dom =int(date_time[8:10])
486 486 # set= int(filename[4:10])
487 487 # self.set=set-1
488 #self.dirnameList=[filename]
489 filenameList= fullpath+"/"+filename
490 self.dirnameList=[filename]
491 self.filenameList=[filenameList]
492 self.flag_nextfile=True
493
494 #self.__findDataForDates(online=True)
495 #self.dirnameList=[self.dirnameList[-1]]
496 #print self.dirnameList
497 #self.__selectDataForTimes(online=True)
498 #return fullpath,filename,year,month,dom,set
488 # self.dirnameList=[filename]
489 filenameList = fullpath + "/" + filename
490 self.dirnameList = [filename]
491 self.filenameList = [filenameList]
492 self.flag_nextfile = True
493
494 # self.__findDataForDates(online=True)
495 # self.dirnameList=[self.dirnameList[-1]]
496 # print self.dirnameList
497 # self.__selectDataForTimes(online=True)
498 # return fullpath,filename,year,month,dom,set
499 499 return
500 500
501 def __setNextFile(self,online=False):
501 def __setNextFile(self, online=False):
502 502 """
503 503 """
504 504 if not(online):
505 505 newFile = self.__setNextFileOffline()
506 506 else:
507 507 newFile = self.__setNextFileOnline()
508 508
509 509 if not(newFile):
510 510 return 0
511 511 return 1
512 512
513 513 def __setNextFileOffline(self):
514 514 """
515 515 """
516 idFile= self.fileIndex
516 idFile = self.fileIndex
517 517 while(True):
518 518 idFile += 1
519 519 if not (idFile < len(self.filenameList)):
520 520 self.flagNoMoreFiles = 1
521 521 print("No more Files")
522 522 return 0
523 523 filename = self.filenameList[idFile]
524 hfFilePointer =h5py.File(filename,'r')
524 hfFilePointer = h5py.File(filename, 'r')
525 525
526 epoc=hfFilePointer['t'].value
527 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
526 epoc = hfFilePointer['t'].value
527 # this_time=datetime.datetime(year,month,dom,hour,min,sec)
528 528 break
529 529
530 530 self.flagIsNewFile = 1
531 531 self.fileIndex = idFile
532 532 self.filename = filename
533 533
534 534 self.hfFilePointer = hfFilePointer
535 535 hfFilePointer.close()
536 self.__t0=epoc
537 print("Setting the file: %s"%self.filename)
536 self.__t0 = epoc
537 print("Setting the file: %s" % self.filename)
538 538
539 539 return 1
540 540
541 541 def __setNextFileOnline(self):
542 542 """
543 543 """
544 print("SOY NONE",self.set)
545 if self.set==None:
544 print("SOY NONE", self.set)
545 if self.set == None:
546 546 pass
547 547 else:
548 self.set +=10
548 self.set += 10
549 549
550 filename = self.filenameList[0]#fullfilename
550 filename = self.filenameList[0] # fullfilename
551 551 if self.filename_online != None:
552 552 self.__selectDataForTimes(online=True)
553 553 filename = self.filenameList[0]
554 554 while self.filename_online == filename:
555 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
555 print('waiting %d seconds to get a new file...' % (self.__waitForNewFile))
556 556 time.sleep(self.__waitForNewFile)
557 #self.__findDataForDates(online=True)
558 self.set=self.filename_next_set
557 # self.__findDataForDates(online=True)
558 self.set = self.filename_next_set
559 559 self.__selectDataForTimes(online=True)
560 560 filename = self.filenameList[0]
561 sizeoffile=os.path.getsize(filename)
562
563 #print filename
564 sizeoffile=os.path.getsize(filename)
565 if sizeoffile<1670240:
566 print("%s is not the rigth size"%filename)
567 delay=50
568 print('waiting %d seconds for delay...'%(delay))
561 sizeoffile = os.path.getsize(filename)
562
563 # print filename
564 sizeoffile = os.path.getsize(filename)
565 if sizeoffile < 1670240:
566 print("%s is not the rigth size" % filename)
567 delay = 50
568 print('waiting %d seconds for delay...' % (delay))
569 569 time.sleep(delay)
570 sizeoffile=os.path.getsize(filename)
571 if sizeoffile<1670240:
572 delay=50
573 print('waiting %d more seconds for delay...'%(delay))
570 sizeoffile = os.path.getsize(filename)
571 if sizeoffile < 1670240:
572 delay = 50
573 print('waiting %d more seconds for delay...' % (delay))
574 574 time.sleep(delay)
575 575
576 sizeoffile=os.path.getsize(filename)
577 if sizeoffile<1670240:
578 delay=50
579 print('waiting %d more seconds for delay...'%(delay))
576 sizeoffile = os.path.getsize(filename)
577 if sizeoffile < 1670240:
578 delay = 50
579 print('waiting %d more seconds for delay...' % (delay))
580 580 time.sleep(delay)
581 581
582 582 try:
583 hfFilePointer=h5py.File(filename,'r')
583 hfFilePointer = h5py.File(filename, 'r')
584 584
585 585 except:
586 print("Error reading file %s"%filename)
586 print("Error reading file %s" % filename)
587 587
588 self.filename_online=filename
589 epoc=hfFilePointer['t'].value
588 self.filename_online = filename
589 epoc = hfFilePointer['t'].value
590 590
591 self.hfFilePointer=hfFilePointer
591 self.hfFilePointer = hfFilePointer
592 592 hfFilePointer.close()
593 self.__t0=epoc
593 self.__t0 = epoc
594 594
595 595
596 596 self.flagIsNewFile = 1
597 597 self.filename = filename
598 598
599 print("Setting the file: %s"%self.filename)
599 print("Setting the file: %s" % self.filename)
600 600 return 1
601 601
602 602 def __getExpParameters(self):
603 603 if not(self.status):
604 604 return None
605 605
606 606 def setup(self,
607 path = None,
608 startDate = None,
609 endDate = None,
610 startTime = datetime.time(0,0,0),
611 endTime = datetime.time(23,59,59),
612 set = None,
613 expLabel = "",
614 ext = None,
607 path=None,
608 startDate=None,
609 endDate=None,
610 startTime=datetime.time(0, 0, 0),
611 endTime=datetime.time(23, 59, 59),
612 set=None,
613 expLabel="",
614 ext=None,
615 615 all=0,
616 616 timezone=0,
617 online = False,
618 delay = 60,
619 walk = True):
617 online=False,
618 delay=60,
619 walk=True):
620 620 '''
621 621 In this method we should set all initial parameters.
622 622
623 623 '''
624 if path==None:
624 if path == None:
625 625 raise ValueError("The path is not valid")
626 626
627 if ext==None:
627 if ext == None:
628 628 ext = self.ext
629 629
630 self.timezone= timezone
631 self.online= online
632 self.all=all
633 #if set==None:
630 self.timezone = timezone
631 self.online = online
632 self.all = all
633 # if set==None:
634 634
635 #print set
635 # print set
636 636 if not(online):
637 637 print("Searching files in offline mode...")
638 638
639 639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 640 else:
641 641 print("Searching files in online mode...")
642 self.searchFilesOnLine(path, walk,ext,set=set)
643 if set==None:
642 self.searchFilesOnLine(path, walk, ext, set=set)
643 if set == None:
644 644 pass
645 645 else:
646 self.set=set-10
646 self.set = set - 10
647 647
648 648 # for nTries in range(self.nTries):
649 649 #
650 650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 651 #
652 652 # if fullpath:
653 653 # break
654 654 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
655 655 # time.sleep(self.delay)
656 656 # if not(fullpath):
657 657 # print "There ins't valid files in %s" % path
658 658 # return None
659 659
660 660
661 661 if not(self.filenameList):
662 print("There is no files into the folder: %s"%(path))
662 print("There is no files into the folder: %s" % (path))
663 663 sys.exit(-1)
664 664
665 665 self.__getExpParameters()
666 666
667 667
668 668 self.fileIndex = -1
669 669
670 670 self.__setNextFile(online)
671 671
672 672 self.__readMetadata()
673 673
674 674 self.__setLocalVariables()
675 675
676 676 self.__setHeaderDO()
677 #self.profileIndex_offset= 0
677 # self.profileIndex_offset= 0
678 678
679 #self.profileIndex = self.profileIndex_offset
679 # self.profileIndex = self.profileIndex_offset
680 680
681 681 self.isConfig = True
682 682
683 683 def __readMetadata(self):
684 684 self.__readHeader()
685 685
686 686
687 687 def __setLocalVariables(self):
688 688
689 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
689 self.datablock = numpy.zeros((self.nChannels, self.nHeights, self.nProfiles), dtype=numpy.complex)
690 690 #
691 691
692 692
693 693
694 694 self.profileIndex = 9999
695 695
696 696
697 697 def __setHeaderDO(self):
698 698
699 699
700 700 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
701 701
702 702 self.dataOut.systemHeaderObj = SystemHeader()
703 703
704 704
705 705 #---------------------------------------------------------
706 self.dataOut.systemHeaderObj.nProfiles=100
707 self.dataOut.systemHeaderObj.nSamples=1000
706 self.dataOut.systemHeaderObj.nProfiles = 100
707 self.dataOut.systemHeaderObj.nSamples = 1000
708 708
709 709
710 SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
711 self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
712 self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
713 self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
714 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
715 self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
710 SAMPLING_STRUCTURE = [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
711 self.dataOut.radarControllerHeaderObj.samplingWindow = numpy.zeros((1,), SAMPLING_STRUCTURE)
712 self.dataOut.radarControllerHeaderObj.samplingWindow['h0'] = 0
713 self.dataOut.radarControllerHeaderObj.samplingWindow['dh'] = 1.5
714 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'] = 1000
715 self.dataOut.radarControllerHeaderObj.nHeights = int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
716 716 self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
717 717 self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
718 718 self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
719 719
720 self.dataOut.radarControllerHeaderObj.nWindows=1
721 self.dataOut.radarControllerHeaderObj.codetype=0
722 self.dataOut.radarControllerHeaderObj.numTaus=0
723 #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
720 self.dataOut.radarControllerHeaderObj.nWindows = 1
721 self.dataOut.radarControllerHeaderObj.codetype = 0
722 self.dataOut.radarControllerHeaderObj.numTaus = 0
723 # self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
724 724
725 725
726 #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
727 #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
728 #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
726 # self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
727 # self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
728 # self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
729 729
730 self.dataOut.radarControllerHeaderObj.code_size=0
731 self.dataOut.nBaud=0
732 self.dataOut.nCode=0
733 self.dataOut.nPairs=0
730 self.dataOut.radarControllerHeaderObj.code_size = 0
731 self.dataOut.nBaud = 0
732 self.dataOut.nCode = 0
733 self.dataOut.nPairs = 0
734 734
735 735
736 736 #---------------------------------------------------------
737 737
738 738 self.dataOut.type = "Voltage"
739 739
740 740 self.dataOut.data = None
741 741
742 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
742 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
743 743
744 744 self.dataOut.nProfiles = 1
745 745
746 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
746 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float) * self.__deltaHeigth
747 747
748 748 self.dataOut.channelList = list(range(self.nChannels))
749 749
750 #self.dataOut.channelIndexList = None
750 # self.dataOut.channelIndexList = None
751 751
752 752 self.dataOut.flagNoData = True
753 753
754 #Set to TRUE if the data is discontinuous
754 # Set to TRUE if the data is discontinuous
755 755 self.dataOut.flagDiscontinuousBlock = False
756 756
757 757 self.dataOut.utctime = None
758 758
759 759 self.dataOut.timeZone = self.timezone
760 760
761 761 self.dataOut.dstFlag = 0
762 762
763 763 self.dataOut.errorCount = 0
764 764
765 765 self.dataOut.nCohInt = 1
766 766
767 767 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
768 768
769 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
769 self.dataOut.flagDecodeData = False # asumo que la data esta decodificada
770 770
771 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
771 self.dataOut.flagDeflipData = False # asumo que la data esta sin flip
772 772
773 773 self.dataOut.flagShiftFFT = False
774 774
775 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
775 self.dataOut.ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
776 776
777 #Time interval between profiles
778 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
777 # Time interval between profiles
778 # self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
779 779
780 780
781 781 self.dataOut.frequency = self.__frequency
782 782
783 783 self.dataOut.realtime = self.__online
784 784
785 785 def __hasNotDataInBuffer(self):
786 786
787 787 if self.profileIndex >= self.nProfiles:
788 788 return 1
789 789
790 790 return 0
791 791
792 792 def readNextBlock(self):
793 793 if not(self.__setNewBlock()):
794 794 return 0
795 795
796 796 if not(self.readBlock()):
797 797 return 0
798 798
799 799 return 1
800 800
801 801 def __setNewBlock(self):
802 802
803 if self.hfFilePointer==None:
803 if self.hfFilePointer == None:
804 804 return 0
805 805
806 806 if self.flagIsNewFile:
807 807 return 1
808 808
809 809 if self.profileIndex < self.nProfiles:
810 810 return 1
811 811
812 812 self.__setNextFile(self.online)
813 813
814 814 return 1
815 815
816 816
817 817
818 818 def readBlock(self):
819 fp=h5py.File(self.filename,'r')
820 #Puntero que apunta al archivo hdf5
821 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
822 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
819 fp = h5py.File(self.filename, 'r')
820 # Puntero que apunta al archivo hdf5
821 ch0 = (fp['ch0']).value # Primer canal (100,1000)--(perfiles,alturas)
822 ch1 = (fp['ch1']).value # Segundo canal (100,1000)--(perfiles,alturas)
823 823 fp.close()
824 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
825 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
826 self.datablock = numpy.array([ch0,ch1])
827 self.flagIsNewFile=0
824 ch0 = ch0.swapaxes(0, 1) # Primer canal (100,1000)--(alturas,perfiles)
825 ch1 = ch1.swapaxes(0, 1) # Segundo canal (100,1000)--(alturas,perfiles)
826 self.datablock = numpy.array([ch0, ch1])
827 self.flagIsNewFile = 0
828 828
829 self.profileIndex=0
829 self.profileIndex = 0
830 830
831 831 return 1
832 832
833 833 def getData(self):
834 834 if self.flagNoMoreFiles:
835 835 self.dataOut.flagNoData = True
836 836 return 0
837 837
838 838 if self.__hasNotDataInBuffer():
839 839 if not(self.readNextBlock()):
840 self.dataOut.flagNodata=True
840 self.dataOut.flagNodata = True
841 841 return 0
842 842
843 843 ##############################
844 844 ##############################
845 self.dataOut.data = self.datablock[:,:,self.profileIndex]
846 self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
847 self.dataOut.profileIndex= self.profileIndex
848 self.dataOut.flagNoData=False
849 self.profileIndex +=1
845 self.dataOut.data = self.datablock[:, :, self.profileIndex]
846 self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds * self.profileIndex
847 self.dataOut.profileIndex = self.profileIndex
848 self.dataOut.flagNoData = False
849 self.profileIndex += 1
850 850
851 851 return self.dataOut.data
852 852
853 853
854 854 def run(self, **kwargs):
855 855 '''
856 856 This method will be called many times so here you should put all your code
857 857 '''
858 858
859 859 if not self.isConfig:
860 860 self.setup(**kwargs)
861 861 self.isConfig = True
862 self.getData() No newline at end of file
862 self.getData()
Show file before | Show file after | Hide comments
@@ -1,629 +1,629
1 1 '''
2 2 Created on Set 9, 2015
3 3
4 4 @author: roj-idl71 Karim Kuyeng
5 5 '''
6 6
7 7 import os
8 8 import sys
9 9 import glob
10 10 import fnmatch
11 11 import datetime
12 12 import time
13 13 import re
14 14 import h5py
15 15 import numpy
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
23 23 from schainpy.model.data.jrodata import Voltage
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25 from numpy import imag
26 26
27 27 class AMISRReader(ProcessingUnit):
28 28 '''
29 29 classdocs
30 30 '''
31 31
32 32 def __init__(self):
33 33 '''
34 34 Constructor
35 35 '''
36 36
37 37 ProcessingUnit.__init__(self)
38 38
39 39 self.set = None
40 40 self.subset = None
41 41 self.extension_file = '.h5'
42 42 self.dtc_str = 'dtc'
43 43 self.dtc_id = 0
44 44 self.status = True
45 45 self.isConfig = False
46 46 self.dirnameList = []
47 47 self.filenameList = []
48 48 self.fileIndex = None
49 49 self.flagNoMoreFiles = False
50 50 self.flagIsNewFile = 0
51 51 self.filename = ''
52 52 self.amisrFilePointer = None
53 53
54 54
55 55 self.dataset = None
56 56
57 57
58 58
59 59
60 60 self.profileIndex = 0
61 61
62 62
63 63 self.beamCodeByFrame = None
64 64 self.radacTimeByFrame = None
65 65
66 66 self.dataset = None
67 67
68 68
69 69
70 70
71 71 self.__firstFile = True
72 72
73 73 self.buffer = None
74 74
75 75
76 76 self.timezone = 'ut'
77 77
78 78 self.__waitForNewFile = 20
79 79 self.__filename_online = None
80 #Is really necessary create the output object in the initializer
80 # Is really necessary create the output object in the initializer
81 81 self.dataOut = Voltage()
82 82
83 def setup(self,path=None,
84 startDate=None,
85 endDate=None,
86 startTime=None,
83 def setup(self, path=None,
84 startDate=None,
85 endDate=None,
86 startTime=None,
87 87 endTime=None,
88 88 walk=True,
89 89 timezone='ut',
90 90 all=0,
91 code = None,
92 nCode = 0,
93 nBaud = 0,
91 code=None,
92 nCode=0,
93 nBaud=0,
94 94 online=False):
95 95
96 96 self.timezone = timezone
97 97 self.all = all
98 98 self.online = online
99 99
100 100 self.code = code
101 101 self.nCode = int(nCode)
102 102 self.nBaud = int(nBaud)
103 103
104 104
105 105
106 #self.findFiles()
106 # self.findFiles()
107 107 if not(online):
108 #Busqueda de archivos offline
108 # Busqueda de archivos offline
109 109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 110 else:
111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
111 self.searchFilesOnLine(path, startDate, endDate, startTime, endTime, walk)
112 112
113 113 if not(self.filenameList):
114 print("There is no files into the folder: %s"%(path))
114 print("There is no files into the folder: %s" % (path))
115 115
116 116 sys.exit(-1)
117 117
118 118 self.fileIndex = -1
119 119
120 120 self.readNextFile(online)
121 121
122 122 '''
123 123 Add code
124 124 '''
125 125 self.isConfig = True
126 126
127 127 pass
128 128
129 129
130 def readAMISRHeader(self,fp):
130 def readAMISRHeader(self, fp):
131 131 header = 'Raw11/Data/RadacHeader'
132 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
133 self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
134 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
135 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
136 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
137 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
138 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
139 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
140 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
141 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
132 self.beamCodeByPulse = fp.get(header + '/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
133 self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
134 # self.code = fp.get(header+'/Code') # NOT USE FOR THIS
135 self.frameCount = fp.get(header + '/FrameCount') # NOT USE FOR THIS
136 self.modeGroup = fp.get(header + '/ModeGroup') # NOT USE FOR THIS
137 self.nsamplesPulse = fp.get(header + '/NSamplesPulse') # TO GET NSA OR USING DATA FOR THAT
138 self.pulseCount = fp.get(header + '/PulseCount') # NOT USE FOR THIS
139 self.radacTime = fp.get(header + '/RadacTime') # 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
140 self.timeCount = fp.get(header + '/TimeCount') # NOT USE FOR THIS
141 self.timeStatus = fp.get(header + '/TimeStatus') # NOT USE FOR THIS
142 142 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
143 self.frequency = fp.get('Rx/Frequency')
143 self.frequency = fp.get('Rx/Frequency')
144 144 txAus = fp.get('Raw11/Data/Pulsewidth')
145 145
146 146
147 self.nblocks = self.pulseCount.shape[0] #nblocks
147 self.nblocks = self.pulseCount.shape[0] # nblocks
148 148
149 self.nprofiles = self.pulseCount.shape[1] #nprofile
150 self.nsa = self.nsamplesPulse[0,0] #ngates
149 self.nprofiles = self.pulseCount.shape[1] # nprofile
150 self.nsa = self.nsamplesPulse[0, 0] # ngates
151 151 self.nchannels = self.beamCode.shape[1]
152 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
153 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
154 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
152 self.ippSeconds = (self.radacTime[0][1] - self.radacTime[0][0]) # Ipp in seconds
153 # self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
154 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
155 155
156 #filling radar controller header parameters
157 self.__ippKm = self.ippSeconds *.15*1e6 # in km
158 self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
156 # filling radar controller header parameters
157 self.__ippKm = self.ippSeconds * .15 * 1e6 # in km
158 self.__txA = (txAus.value) * .15 # (ipp[us]*.15km/1us) in km
159 159 self.__txB = 0
160 nWindows=1
160 nWindows = 1
161 161 self.__nSamples = self.nsa
162 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
163 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
162 self.__firstHeight = self.rangeFromFile[0][0] / 1000 # in km
163 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0]) / 1000
164 164
165 #for now until understand why the code saved is different (code included even though code not in tuf file)
166 #self.__codeType = 0
165 # for now until understand why the code saved is different (code included even though code not in tuf file)
166 # self.__codeType = 0
167 167 # self.__nCode = None
168 168 # self.__nBaud = None
169 169 self.__code = self.code
170 170 self.__codeType = 0
171 171 if self.code != None:
172 172 self.__codeType = 1
173 173 self.__nCode = self.nCode
174 174 self.__nBaud = self.nBaud
175 #self.__code = 0
175 # self.__code = 0
176 176
177 #filling system header parameters
177 # filling system header parameters
178 178 self.__nSamples = self.nsa
179 self.newProfiles = self.nprofiles/self.nchannels
179 self.newProfiles = self.nprofiles / self.nchannels
180 180 self.__channelList = list(range(self.nchannels))
181 181
182 182 self.__frequency = self.frequency[0][0]
183 183
184 184
185 185
186 186 def createBuffers(self):
187 187
188 188 pass
189 189
190 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
190 def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''):
191 191 self.path = path
192 192 self.startDate = startDate
193 193 self.endDate = endDate
194 194 self.startTime = startTime
195 195 self.endTime = endTime
196 196 self.walk = walk
197 197
198 198 def __checkPath(self):
199 199 if os.path.exists(self.path):
200 200 self.status = 1
201 201 else:
202 202 self.status = 0
203 print('Path:%s does not exists'%self.path)
203 print('Path:%s does not exists' % self.path)
204 204
205 205 return
206 206
207 207
208 208 def __selDates(self, amisr_dirname_format):
209 209 try:
210 210 year = int(amisr_dirname_format[0:4])
211 211 month = int(amisr_dirname_format[4:6])
212 212 dom = int(amisr_dirname_format[6:8])
213 thisDate = datetime.date(year,month,dom)
213 thisDate = datetime.date(year, month, dom)
214 214
215 if (thisDate>=self.startDate and thisDate <= self.endDate):
215 if (thisDate >= self.startDate and thisDate <= self.endDate):
216 216 return amisr_dirname_format
217 217 except:
218 218 return None
219 219
220 220
221 def __findDataForDates(self,online=False):
221 def __findDataForDates(self, online=False):
222 222
223 223 if not(self.status):
224 224 return None
225 225
226 226 pat = '\d+.\d+'
227 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
228 dirnameList = [x for x in dirnameList if x!=None]
227 dirnameList = [re.search(pat, x) for x in os.listdir(self.path)]
228 dirnameList = [x for x in dirnameList if x != None]
229 229 dirnameList = [x.string for x in dirnameList]
230 230 if not(online):
231 231 dirnameList = [self.__selDates(x) for x in dirnameList]
232 dirnameList = [x for x in dirnameList if x!=None]
233 if len(dirnameList)>0:
232 dirnameList = [x for x in dirnameList if x != None]
233 if len(dirnameList) > 0:
234 234 self.status = 1
235 235 self.dirnameList = dirnameList
236 236 self.dirnameList.sort()
237 237 else:
238 238 self.status = 0
239 239 return None
240 240
241 241 def __getTimeFromData(self):
242 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
243 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
242 startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime)
243 endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime)
244 244
245 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
245 print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader))
246 246 print('........................................')
247 247 filter_filenameList = []
248 248 self.filenameList.sort()
249 #for i in range(len(self.filenameList)-1):
249 # for i in range(len(self.filenameList)-1):
250 250 for i in range(len(self.filenameList)):
251 251 filename = self.filenameList[i]
252 fp = h5py.File(filename,'r')
252 fp = h5py.File(filename, 'r')
253 253 time_str = fp.get('Time/RadacTimeString')
254 254
255 255 startDateTimeStr_File = time_str[0][0].split('.')[0]
256 256 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
257 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
257 startDateTime_File = datetime.datetime(junk.tm_year, junk.tm_mon, junk.tm_mday, junk.tm_hour, junk.tm_min, junk.tm_sec)
258 258
259 259 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
260 260 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
261 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
261 endDateTime_File = datetime.datetime(junk.tm_year, junk.tm_mon, junk.tm_mday, junk.tm_hour, junk.tm_min, junk.tm_sec)
262 262
263 263 fp.close()
264 264
265 265 if self.timezone == 'lt':
266 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
267 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
266 startDateTime_File = startDateTime_File - datetime.timedelta(minutes=300)
267 endDateTime_File = endDateTime_File - datetime.timedelta(minutes=300)
268 268
269 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
270 #self.filenameList.remove(filename)
269 if (endDateTime_File >= startDateTime_Reader and endDateTime_File < endDateTime_Reader):
270 # self.filenameList.remove(filename)
271 271 filter_filenameList.append(filename)
272 272
273 if (endDateTime_File>=endDateTime_Reader):
273 if (endDateTime_File >= endDateTime_Reader):
274 274 break
275 275
276 276
277 277 filter_filenameList.sort()
278 278 self.filenameList = filter_filenameList
279 279 return 1
280 280
281 281 def __filterByGlob1(self, dirName):
282 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
282 filter_files = glob.glob1(dirName, '*.*%s' % self.extension_file)
283 283 filter_files.sort()
284 284 filterDict = {}
285 285 filterDict.setdefault(dirName)
286 286 filterDict[dirName] = filter_files
287 287 return filterDict
288 288
289 289 def __getFilenameList(self, fileListInKeys, dirList):
290 290 for value in fileListInKeys:
291 291 dirName = list(value.keys())[0]
292 292 for file in value[dirName]:
293 293 filename = os.path.join(dirName, file)
294 294 self.filenameList.append(filename)
295 295
296 296
297 297 def __selectDataForTimes(self, online=False):
298 #aun no esta implementado el filtro for tiempo
298 # aun no esta implementado el filtro for tiempo
299 299 if not(self.status):
300 300 return None
301 301
302 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
302 dirList = [os.path.join(self.path, x) for x in self.dirnameList]
303 303
304 304 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
305 305
306 306 self.__getFilenameList(fileListInKeys, dirList)
307 307 if not(online):
308 #filtro por tiempo
308 # filtro por tiempo
309 309 if not(self.all):
310 310 self.__getTimeFromData()
311 311
312 if len(self.filenameList)>0:
312 if len(self.filenameList) > 0:
313 313 self.status = 1
314 314 self.filenameList.sort()
315 315 else:
316 316 self.status = 0
317 317 return None
318 318
319 319 else:
320 #get the last file - 1
320 # get the last file - 1
321 321 self.filenameList = [self.filenameList[-2]]
322 322
323 323 new_dirnameList = []
324 324 for dirname in self.dirnameList:
325 325 junk = numpy.array([dirname in x for x in self.filenameList])
326 326 junk_sum = junk.sum()
327 327 if junk_sum > 0:
328 328 new_dirnameList.append(dirname)
329 329 self.dirnameList = new_dirnameList
330 330 return 1
331 331
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 endTime=datetime.time(23,59,59),walk=True):
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0, 0, 0),
333 endTime=datetime.time(23, 59, 59), walk=True):
334 334
335 if endDate ==None:
335 if endDate == None:
336 336 startDate = datetime.datetime.utcnow().date()
337 337 endDate = datetime.datetime.utcnow().date()
338 338
339 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
339 self.__setParameters(path=path, startDate=startDate, endDate=endDate, startTime=startTime, endTime=endTime, walk=walk)
340 340
341 341 self.__checkPath()
342 342
343 343 self.__findDataForDates(online=True)
344 344
345 345 self.dirnameList = [self.dirnameList[-1]]
346 346
347 347 self.__selectDataForTimes(online=True)
348 348
349 349 return
350 350
351 351
352 352 def searchFilesOffLine(self,
353 353 path,
354 354 startDate,
355 355 endDate,
356 startTime=datetime.time(0,0,0),
357 endTime=datetime.time(23,59,59),
356 startTime=datetime.time(0, 0, 0),
357 endTime=datetime.time(23, 59, 59),
358 358 walk=True):
359 359
360 360 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
361 361
362 362 self.__checkPath()
363 363
364 364 self.__findDataForDates()
365 365
366 366 self.__selectDataForTimes()
367 367
368 368 for i in range(len(self.filenameList)):
369 print("%s" %(self.filenameList[i]))
369 print("%s" % (self.filenameList[i]))
370 370
371 371 return
372 372
373 373 def __setNextFileOffline(self):
374 374 idFile = self.fileIndex
375 375
376 376 while (True):
377 377 idFile += 1
378 378 if not(idFile < len(self.filenameList)):
379 379 self.flagNoMoreFiles = 1
380 380 print("No more Files")
381 381 return 0
382 382
383 383 filename = self.filenameList[idFile]
384 384
385 amisrFilePointer = h5py.File(filename,'r')
385 amisrFilePointer = h5py.File(filename, 'r')
386 386
387 387 break
388 388
389 389 self.flagIsNewFile = 1
390 390 self.fileIndex = idFile
391 391 self.filename = filename
392 392
393 393 self.amisrFilePointer = amisrFilePointer
394 394
395 print("Setting the file: %s"%self.filename)
395 print("Setting the file: %s" % self.filename)
396 396
397 397 return 1
398 398
399 399
400 400 def __setNextFileOnline(self):
401 401 filename = self.filenameList[0]
402 402 if self.__filename_online != None:
403 403 self.__selectDataForTimes(online=True)
404 404 filename = self.filenameList[0]
405 405 wait = 0
406 406 while self.__filename_online == filename:
407 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
407 print('waiting %d seconds to get a new file...' % (self.__waitForNewFile))
408 408 if wait == 5:
409 409 return 0
410 410 sleep(self.__waitForNewFile)
411 411 self.__selectDataForTimes(online=True)
412 412 filename = self.filenameList[0]
413 413 wait += 1
414 414
415 415 self.__filename_online = filename
416 416
417 self.amisrFilePointer = h5py.File(filename,'r')
417 self.amisrFilePointer = h5py.File(filename, 'r')
418 418 self.flagIsNewFile = 1
419 419 self.filename = filename
420 print("Setting the file: %s"%self.filename)
420 print("Setting the file: %s" % self.filename)
421 421 return 1
422 422
423 423
424 424 def readData(self):
425 425 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
426 re = buffer[:,:,:,0]
427 im = buffer[:,:,:,1]
428 dataset = re + im*1j
426 re = buffer[:, :, :, 0]
427 im = buffer[:, :, :, 1]
428 dataset = re + im * 1j
429 429 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
430 timeset = self.radacTime[:,0]
431 return dataset,timeset
430 timeset = self.radacTime[:, 0]
431 return dataset, timeset
432 432
433 433 def reshapeData(self):
434 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
435 channels = self.beamCodeByPulse[0,:]
434 # self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
435 channels = self.beamCodeByPulse[0, :]
436 436 nchan = self.nchannels
437 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
437 # self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
438 438 nblocks = self.nblocks
439 439 nsamples = self.nsa
440 440
441 #Dimensions : nChannels, nProfiles, nSamples
441 # Dimensions : nChannels, nProfiles, nSamples
442 442 new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
443 443 ############################################
444 444
445 445 for thisChannel in range(nchan):
446 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
446 new_block[:, thisChannel, :, :] = self.dataset[:, numpy.where(channels == self.beamCode[0][thisChannel])[0], :]
447 447
448 448
449 new_block = numpy.transpose(new_block, (1,0,2,3))
450 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
449 new_block = numpy.transpose(new_block, (1, 0, 2, 3))
450 new_block = numpy.reshape(new_block, (nchan, -1, nsamples))
451 451
452 452 return new_block
453 453
454 454 def updateIndexes(self):
455 455
456 456 pass
457 457
458 458 def fillJROHeader(self):
459 459
460 #fill radar controller header
460 # fill radar controller header
461 461 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
462 462 txA=self.__txA,
463 463 txB=0,
464 464 nWindows=1,
465 465 nHeights=self.__nSamples,
466 466 firstHeight=self.__firstHeight,
467 467 deltaHeight=self.__deltaHeight,
468 468 codeType=self.__codeType,
469 469 nCode=self.__nCode, nBaud=self.__nBaud,
470 code = self.__code,
470 code=self.__code,
471 471 fClock=1)
472 472
473 473
474 474
475 #fill system header
475 # fill system header
476 476 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
477 477 nProfiles=self.newProfiles,
478 478 nChannels=len(self.__channelList),
479 479 adcResolution=14,
480 480 pciDioBusWith=32)
481 481
482 482 self.dataOut.type = "Voltage"
483 483
484 484 self.dataOut.data = None
485 485
486 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
486 self.dataOut.dtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
487 487
488 488 # self.dataOut.nChannels = 0
489 489
490 490 # self.dataOut.nHeights = 0
491 491
492 self.dataOut.nProfiles = self.newProfiles*self.nblocks
492 self.dataOut.nProfiles = self.newProfiles * self.nblocks
493 493
494 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
495 ranges = numpy.reshape(self.rangeFromFile.value,(-1))
496 self.dataOut.heightList = ranges/1000.0 #km
494 # self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
495 ranges = numpy.reshape(self.rangeFromFile.value, (-1))
496 self.dataOut.heightList = ranges / 1000.0 # km
497 497
498 498
499 499 self.dataOut.channelList = self.__channelList
500 500
501 501 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
502 502
503 503 # self.dataOut.channelIndexList = None
504 504
505 505 self.dataOut.flagNoData = True
506 506
507 #Set to TRUE if the data is discontinuous
507 # Set to TRUE if the data is discontinuous
508 508 self.dataOut.flagDiscontinuousBlock = False
509 509
510 510 self.dataOut.utctime = None
511 511
512 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
512 # self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
513 513 if self.timezone == 'lt':
514 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
514 self.dataOut.timeZone = time.timezone / 60. # get the timezone in minutes
515 515 else:
516 self.dataOut.timeZone = 0 #by default time is UTC
516 self.dataOut.timeZone = 0 # by default time is UTC
517 517
518 518 self.dataOut.dstFlag = 0
519 519
520 520 self.dataOut.errorCount = 0
521 521
522 522 self.dataOut.nCohInt = 1
523 523
524 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
524 self.dataOut.flagDecodeData = False # asumo que la data esta decodificada
525 525
526 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
526 self.dataOut.flagDeflipData = False # asumo que la data esta sin flip
527 527
528 528 self.dataOut.flagShiftFFT = False
529 529
530 530 self.dataOut.ippSeconds = self.ippSeconds
531 531
532 #Time interval between profiles
533 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
532 # Time interval between profiles
533 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
534 534
535 535 self.dataOut.frequency = self.__frequency
536 536
537 537 self.dataOut.realtime = self.online
538 538 pass
539 539
540 def readNextFile(self,online=False):
540 def readNextFile(self, online=False):
541 541
542 542 if not(online):
543 543 newFile = self.__setNextFileOffline()
544 544 else:
545 545 newFile = self.__setNextFileOnline()
546 546
547 547 if not(newFile):
548 548 return 0
549 549
550 #if self.__firstFile:
550 # if self.__firstFile:
551 551 self.readAMISRHeader(self.amisrFilePointer)
552 552 self.createBuffers()
553 553 self.fillJROHeader()
554 #self.__firstFile = False
554 # self.__firstFile = False
555 555
556 556
557 557
558 self.dataset,self.timeset = self.readData()
558 self.dataset, self.timeset = self.readData()
559 559
560 if self.endDate!=None:
561 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
560 if self.endDate != None:
561 endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime)
562 562 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
563 563 startDateTimeStr_File = time_str[0][0].split('.')[0]
564 564 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
565 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
565 startDateTime_File = datetime.datetime(junk.tm_year, junk.tm_mon, junk.tm_mday, junk.tm_hour, junk.tm_min, junk.tm_sec)
566 566 if self.timezone == 'lt':
567 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
568 if (startDateTime_File>endDateTime_Reader):
567 startDateTime_File = startDateTime_File - datetime.timedelta(minutes=300)
568 if (startDateTime_File > endDateTime_Reader):
569 569 return 0
570 570
571 571 self.jrodataset = self.reshapeData()
572 572 #----self.updateIndexes()
573 573 self.profileIndex = 0
574 574
575 575 return 1
576 576
577 577
578 578 def __hasNotDataInBuffer(self):
579 if self.profileIndex >= (self.newProfiles*self.nblocks):
579 if self.profileIndex >= (self.newProfiles * self.nblocks):
580 580 return 1
581 581 return 0
582 582
583 583
584 584 def getData(self):
585 585
586 586 if self.flagNoMoreFiles:
587 587 self.dataOut.flagNoData = True
588 588 return 0
589 589
590 590 if self.__hasNotDataInBuffer():
591 591 if not (self.readNextFile(self.online)):
592 592 return 0
593 593
594 594
595 if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
595 if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
596 596 self.dataOut.flagNoData = True
597 597 return 0
598 598
599 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
599 # self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
600 600
601 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
601 self.dataOut.data = self.jrodataset[:, self.profileIndex, :]
602 602
603 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
604 #verificar basic header de jro data y ver si es compatible con este valor
605 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
603 # self.dataOut.utctime = self.jrotimeset[self.profileIndex]
604 # verificar basic header de jro data y ver si es compatible con este valor
605 # self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
606 606 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
607 indexblock = self.profileIndex/self.newProfiles
608 #print indexblock, indexprof
607 indexblock = self.profileIndex / self.newProfiles
608 # print indexblock, indexprof
609 609 self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
610 610 self.dataOut.profileIndex = self.profileIndex
611 611 self.dataOut.flagNoData = False
612 612 # if indexprof == 0:
613 613 # print self.dataOut.utctime
614 614
615 615 self.profileIndex += 1
616 616
617 617 return self.dataOut.data
618 618
619 619
620 620 def run(self, **kwargs):
621 621 '''
622 622 This method will be called many times so here you should put all your code
623 623 '''
624 624
625 625 if not self.isConfig:
626 626 self.setup(**kwargs)
627 627 self.isConfig = True
628 628
629 629 self.getData()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now