##// END OF EJS Templates
schain
RSS
Errors handling and gracefully terminate main process
jespinoza -
r1241:c3044f867269
parent child
Show More
Side by Side Unified
Context file:
r1241:c3044f867269 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,506 +1,505
1 1 """
2 2 The admin module contains all administrative classes relating to the schain python api.
3 3
4 4 The main role of this module is to send some reports. It contains a
5 5 notification class and a standard error handing class.
6 6
7 7 $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
8 8 """
9 9 import os
10 10 import sys
11 11 import time
12 12 import traceback
13 13 import smtplib
14 14 if sys.version[0] == '3':
15 15 from configparser import ConfigParser
16 16 else:
17 17 from ConfigParser import ConfigParser
18 18 import io
19 19 from threading import Thread
20 20 from multiprocessing import Process
21 21 from email.mime.text import MIMEText
22 22 from email.mime.application import MIMEApplication
23 23 from email.mime.multipart import MIMEMultipart
24 24
25 25 import schainpy
26 26 from schainpy.utils import log
27 27 from schainpy.model.graphics.jroplot_base import popup
28 28
29 29 def get_path():
30 30 '''
31 31 Return schainpy path
32 32 '''
33 33
34 34 try:
35 35 root = __file__
36 36 if os.path.islink(root):
37 37 root = os.path.realpath(root)
38 38
39 39 return os.path.dirname(os.path.abspath(root))
40 40 except:
41 41 log.error('I am sorry, but something is wrong... __file__ not found')
42 42
43 43 class Alarm(Process):
44 44 '''
45 45 modes:
46 46 0 - All
47 47 1 - Send email
48 48 2 - Popup message
49 49 3 - Sound alarm
50 50 4 - Send to alarm system TODO
51 51 '''
52 52
53 53 def __init__(self, modes=[], **kwargs):
54 54 Process.__init__(self)
55 55 self.modes = modes
56 56 self.kwargs = kwargs
57 57
58 58 @staticmethod
59 59 def play_sound():
60 60 sound = os.path.join(get_path(), 'alarm1.oga')
61 61 if os.path.exists(sound):
62 62 for __ in range(2):
63 63 os.system('paplay {}'.format(sound))
64 64 time.sleep(0.5)
65 65 else:
66 66 log.warning('Unable to play alarm, sound file not found', 'ADMIN')
67 67
68 68 @staticmethod
69 69 def send_email(**kwargs):
70 70 notifier = SchainNotify()
71 print(kwargs)
72 71 notifier.notify(**kwargs)
73 72
74 73 @staticmethod
75 74 def show_popup(message):
76 75 if isinstance(message, list):
77 76 message = message[-1]
78 77 popup(message)
79 78
80 79 @staticmethod
81 80 def send_alarm():
82 81 pass
83 82
84 83 @staticmethod
85 84 def get_kwargs(kwargs, keys):
86 85 ret = {}
87 86 for key in keys:
88 87 ret[key] = kwargs[key]
89 88 return ret
90 89
91 90 def run(self):
92 91 tasks = {
93 92 1 : self.send_email,
94 93 2 : self.show_popup,
95 94 3 : self.play_sound,
96 95 4 : self.send_alarm,
97 96 }
98 97
99 98 tasks_args = {
100 99 1: ['email', 'message', 'subject', 'subtitle', 'filename'],
101 100 2: ['message'],
102 101 3: [],
103 102 4: [],
104 103 }
105 104 procs = []
106 105 for mode in self.modes:
107 106 if 0 in self.modes:
108 107 for x in tasks:
109 108 t = Thread(target=tasks[x], kwargs=self.get_kwargs(self.kwargs, tasks_args[x]))
110 109 t.start()
111 110 procs.append(t)
112 111 break
113 112 else:
114 113 t = Thread(target=tasks[mode], kwargs=self.get_kwargs(self.kwargs, tasks_args[mode]))
115 114 t.start()
116 115 procs.append(t)
117 116 for t in procs:
118 117 t.join()
119 118
120 119
121 120 class SchainConfigure():
122 121
123 122 __DEFAULT_ADMINISTRATOR_EMAIL = "juan.espinoza@jro.igp.gob.pe"
124 123 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
125 124 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
126 125 __DEFAULT_SENDER_PASS = ""
127 126
128 127 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
129 128 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
130 129 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
131 130 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
132 131
133 132 def __init__(self, initFile = None):
134 133
135 134 # Set configuration file
136 135 if (initFile == None):
137 136 self.__confFilePath = "/etc/schain.conf"
138 137 else:
139 138 self.__confFilePath = initFile
140 139
141 140 # open configuration file
142 141 try:
143 142 self.__confFile = open(self.__confFilePath, "r")
144 143 except IOError:
145 144 # can't read from file - use all hard-coded values
146 145 self.__initFromHardCode()
147 146 return
148 147
149 148 # create Parser using standard module ConfigParser
150 149 self.__parser = ConfigParser()
151 150
152 151 # read conf file into a StringIO with "[madrigal]\n" section heading prepended
153 152 strConfFile = io.StringIO("[schain]\n" + self.__confFile.read())
154 153
155 154 # parse StringIO configuration file
156 155 self.__parser.readfp(strConfFile)
157 156
158 157 # read information from configuration file
159 158 self.__readConfFile()
160 159
161 160 # close conf file
162 161 self.__confFile.close()
163 162
164 163
165 164 def __initFromHardCode(self):
166 165
167 166 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
168 167 self.__sender_pass = self.__DEFAULT_SENDER_PASS
169 168 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
170 169 self.__email_server = self.__DEFAULT_EMAIL_SERVER
171 170
172 171 def __readConfFile(self):
173 172 """__readConfFile is a private helper function that reads information from the parsed config file.
174 173
175 174 Inputs: None
176 175
177 176 Returns: Void.
178 177
179 178 Affects: Initializes class member variables that are found in the config file.
180 179
181 180 Exceptions: MadrigalError thrown if any key not found.
182 181 """
183 182
184 183 # get the sender email
185 184 try:
186 185 self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
187 186 except:
188 187 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
189 188
190 189 # get the sender password
191 190 try:
192 191 self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
193 192 except:
194 193 self.__sender_pass = self.__DEFAULT_SENDER_PASS
195 194
196 195 # get the administrator email
197 196 try:
198 197 self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
199 198 except:
200 199 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
201 200
202 201 # get the server email
203 202 try:
204 203 self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
205 204 except:
206 205 self.__email_server = self.__DEFAULT_EMAIL_SERVER
207 206
208 207 def getEmailServer(self):
209 208
210 209 return self.__email_server
211 210
212 211 def getSenderEmail(self):
213 212
214 213 return self.__sender_email
215 214
216 215 def getSenderPass(self):
217 216
218 217 return self.__sender_pass
219 218
220 219 def getAdminEmail(self):
221 220
222 221 return self.__admin_email
223 222
224 223 class SchainNotify:
225 224 """SchainNotify is an object used to send messages to an administrator about a Schain software.
226 225
227 226 This object provides functions needed to send messages to an administrator about a Schain , for now
228 227 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
229 228
230 229 Usage example:
231 230
232 231 import schainpy.admin
233 232
234 233 try:
235 234
236 235 adminObj = schainpy.admin.SchainNotify()
237 236 adminObj.sendAlert('This is important!', 'Important Message')
238 237
239 238 except schainpy.admin.SchainError, e:
240 239
241 240 print e.getExceptionStr()
242 241
243 242
244 243 Non-standard Python modules used:
245 244 None
246 245
247 246 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
248 247 throwing exceptions, since this is the class that will generally be called when there is a problem.
249 248
250 249 Change history:
251 250
252 251 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Dec. 1, 2015
253 252 """
254 253
255 254 #constants
256 255
257 256 def __init__(self):
258 257 """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
259 258
260 259 Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
261 260 this is the class that will generally be called when there is a problem.
262 261
263 262 Inputs: Existing SchainDB object, by default = None.
264 263
265 264 Returns: void
266 265
267 266 Affects: Initializes self.__binDir.
268 267
269 268 Exceptions: None.
270 269 """
271 270
272 271 # note that the main configuration file is unavailable
273 272 # the best that can be done is send an email to root using localhost mailserver
274 273 confObj = SchainConfigure()
275 274
276 275 self.__emailFromAddress = confObj.getSenderEmail()
277 276 self.__emailPass = confObj.getSenderPass()
278 277 self.__emailToAddress = confObj.getAdminEmail()
279 278 self.__emailServer = confObj.getEmailServer()
280 279
281 280 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
282 281
283 282 if not email_to:
284 283 return 0
285 284
286 285 if not self.__emailServer:
287 286 return 0
288 287
289 288 log.success('Sending email to {}...'.format(email_to), 'System')
290 289
291 290 msg = MIMEMultipart()
292 291 msg['Subject'] = subject
293 msg['From'] = "(Python SChain API): " + email_from
292 msg['From'] = "SChain API (v{}) <{}>".format(schainpy.__version__, email_from)
294 293 msg['Reply-to'] = email_from
295 294 msg['To'] = email_to
296 295
297 296 # That is what u see if dont have an email reader:
298 297 msg.preamble = 'SChainPy'
299 298
300 299 if html_format:
301 300 message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
302 301 message = "<html>\n" + message + '</html>'
303 302
304 303 # This is the textual part:
305 304 part = MIMEText(message, "html")
306 305 else:
307 306 message = subject + "\n" + subtitle + "\n" + message
308 307 part = MIMEText(message)
309 308
310 309 msg.attach(part)
311 310
312 311 if filename and os.path.isfile(filename):
313 312 # This is the binary part(The Attachment):
314 313 part = MIMEApplication(open(filename,"rb").read())
315 314 part.add_header('Content-Disposition',
316 315 'attachment',
317 316 filename=os.path.basename(filename))
318 317 msg.attach(part)
319 318
320 319 # Create an instance in SMTP server
321 320 try:
322 321 smtp = smtplib.SMTP(self.__emailServer)
323 322 except:
324 323 log.error('Could not connect to server {}'.format(self.__emailServer), 'System')
325 324 return 0
326 325
327 326 # Start the server:
328 327 # smtp.ehlo()
329 328 if self.__emailPass:
330 329 smtp.login(self.__emailFromAddress, self.__emailPass)
331 330
332 331 # Send the email
333 try:
332 try:
334 333 smtp.sendmail(msg['From'], msg['To'], msg.as_string())
335 334 except:
336 335 log.error('Could not send the email to {}'.format(msg['To']), 'System')
337 336 smtp.quit()
338 337 return 0
339 338
340 339 smtp.quit()
341 340
342 341 log.success('Email sent ', 'System')
343 342
344 343 return 1
345 344
346 345 def sendAlert(self, message, subject = "", subtitle="", filename=""):
347 346 """sendAlert sends an email with the given message and optional title.
348 347
349 348 Inputs: message (string), and optional title (string)
350 349
351 350 Returns: void
352 351
353 352 Affects: none
354 353
355 354 Exceptions: None.
356 355 """
357 356
358 357 if not self.__emailToAddress:
359 358 return 0
360 359
361 360 print("***** Sending alert to %s *****" %self.__emailToAddress)
362 361 # set up message
363 362
364 363 sent=self.sendEmail(email_from=self.__emailFromAddress,
365 364 email_to=self.__emailToAddress,
366 365 subject=subject,
367 366 message=message,
368 367 subtitle=subtitle,
369 368 filename=filename)
370 369
371 370 if not sent:
372 371 return 0
373 372
374 373 return 1
375 374
376 375 def notify(self, email, message, subject = "", subtitle="", filename=""):
377 376 """notify sends an email with the given message and title to email.
378 377
379 378 Inputs: email (string), message (string), and subject (string)
380 379
381 380 Returns: void
382 381
383 382 Affects: none
384 383
385 384 Exceptions: None.
386 385 """
387 386
388 387 if email is None:
389 388 email = self.__emailToAddress
390 389
391 390 self.sendEmail(
392 391 email_from=self.__emailFromAddress,
393 392 email_to=email,
394 393 subject=subject,
395 394 message=message,
396 395 subtitle=subtitle,
397 396 filename=filename
398 397 )
399 398
400 399
401 400 class SchainError(Exception):
402 401 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
403 402
404 403 Usage example:
405 404
406 405 import sys, traceback
407 406 import schainpy.admin
408 407
409 408 try:
410 409
411 410 test = open('ImportantFile.txt', 'r')
412 411
413 412 except:
414 413
415 414 raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
416 415 traceback.format_exception(sys.exc_info()[0],
417 416 sys.exc_info()[1],
418 417 sys.exc_info()[2]))
419 418 """
420 419
421 420
422 421 def __init__(self, strInterpretation, exceptionList=None):
423 422 """ __init__ gathers the interpretation string along with all information from sys.exc_info().
424 423
425 424 Inputs:
426 425 strIntepretation - A string representing the programmer's interpretation of
427 426 why the exception occurred
428 427
429 428 exceptionList - a list of strings completely describing the exception.
430 429 Generated by traceback.format_exception(sys.exc_info()[0],
431 430 sys.exc_info()[1],
432 431 sys.exc_info()[2])
433 432
434 433 Returns: Void.
435 434
436 435 Affects: Initializes class member variables _strInterp, _strExcList.
437 436
438 437 Exceptions: None.
439 438 """
440 439
441 440 if not exceptionList:
442 441 exceptionList = traceback.format_exception(sys.exc_info()[0],
443 442 sys.exc_info()[1],
444 443 sys.exc_info()[2])
445 444
446 445 self._strInterp = strInterpretation
447 446 self._strExcList = exceptionList
448 447
449 448
450 449 def getExceptionStr(self):
451 450 """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
452 451
453 452 Inputs: None
454 453
455 454 Returns: A formatted string ready for printing completely describing the exception.
456 455
457 456 Affects: None
458 457
459 458 Exceptions: None.
460 459 """
461 460 excStr = ''
462 461 excStr = excStr + self._strInterp + '\n\n'
463 462
464 463 if self._strExcList != None:
465 464 for item in self._strExcList:
466 465 excStr = excStr + str(item) + '\n'
467 466
468 467 return excStr
469 468
470 469 def __str__(self):
471 470
472 471 return(self.getExceptionStr())
473 472
474 473
475 474 def getExceptionHtml(self):
476 475 """ getExceptionHtml returns an Html formatted string completely describing the exception.
477 476
478 477 Inputs: None
479 478
480 479 Returns: A formatted string ready for printing completely describing the exception.
481 480
482 481 Affects: None
483 482
484 483 Exceptions: None.
485 484 """
486 485
487 486 excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
488 487 excStr = excStr + self._strInterp + '\n<BR>\n'
489 488
490 489 if self._strExcList != None:
491 490 for item in self._strExcList:
492 491 excStr = excStr + str(item) + '\n<BR>'
493 492
494 493 return excStr
495 494
496 495 class SchainWarning(Exception):
497 496 pass
498 497
499 498
500 499 if __name__ == '__main__':
501 500
502 501 test = SchainNotify()
503 502
504 503 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
505 504
506 505 print('Hopefully message sent - check.') No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1256 +1,1284
1 1 '''
2 2 Updated on January , 2018, for multiprocessing purposes
3 3 Author: Sergio Cortez
4 4 Created on September , 2012
5 5 '''
6 6 from platform import python_version
7 7 import sys
8 8 import ast
9 9 import datetime
10 10 import traceback
11 11 import math
12 12 import time
13 13 import zmq
14 from multiprocessing import Process, cpu_count
14 from multiprocessing import Process, Queue, cpu_count
15 15 from threading import Thread
16 16 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
17 17 from xml.dom import minidom
18 18
19 19
20 20 from schainpy.admin import Alarm, SchainWarning
21 21 from schainpy.model import *
22 22 from schainpy.utils import log
23 23
24 24
25 25 DTYPES = {
26 26 'Voltage': '.r',
27 27 'Spectra': '.pdata'
28 28 }
29 29
30 30
31 31 def MPProject(project, n=cpu_count()):
32 32 '''
33 33 Project wrapper to run schain in n processes
34 34 '''
35 35
36 36 rconf = project.getReadUnitObj()
37 37 op = rconf.getOperationObj('run')
38 38 dt1 = op.getParameterValue('startDate')
39 39 dt2 = op.getParameterValue('endDate')
40 40 tm1 = op.getParameterValue('startTime')
41 41 tm2 = op.getParameterValue('endTime')
42 42 days = (dt2 - dt1).days
43 43
44 44 for day in range(days + 1):
45 45 skip = 0
46 46 cursor = 0
47 47 processes = []
48 48 dt = dt1 + datetime.timedelta(day)
49 49 dt_str = dt.strftime('%Y/%m/%d')
50 50 reader = JRODataReader()
51 51 paths, files = reader.searchFilesOffLine(path=rconf.path,
52 52 startDate=dt,
53 53 endDate=dt,
54 54 startTime=tm1,
55 55 endTime=tm2,
56 56 ext=DTYPES[rconf.datatype])
57 57 nFiles = len(files)
58 58 if nFiles == 0:
59 59 continue
60 60 skip = int(math.ceil(nFiles / n))
61 61 while nFiles > cursor * skip:
62 62 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
63 63 skip=skip)
64 64 p = project.clone()
65 65 p.start()
66 66 processes.append(p)
67 67 cursor += 1
68 68
69 69 def beforeExit(exctype, value, trace):
70 70 for process in processes:
71 71 process.terminate()
72 72 process.join()
73 73 print(traceback.print_tb(trace))
74 74
75 75 sys.excepthook = beforeExit
76 76
77 77 for process in processes:
78 78 process.join()
79 79 process.terminate()
80 80
81 81 time.sleep(3)
82 82
83 83 def wait(context):
84 84
85 85 time.sleep(1)
86 86 c = zmq.Context()
87 87 receiver = c.socket(zmq.SUB)
88 88 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
89 89 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
90 90 msg = receiver.recv_multipart()[1]
91 91 context.terminate()
92 92
93 93 class ParameterConf():
94 94
95 95 id = None
96 96 name = None
97 97 value = None
98 98 format = None
99 99
100 100 __formated_value = None
101 101
102 102 ELEMENTNAME = 'Parameter'
103 103
104 104 def __init__(self):
105 105
106 106 self.format = 'str'
107 107
108 108 def getElementName(self):
109 109
110 110 return self.ELEMENTNAME
111 111
112 112 def getValue(self):
113 113
114 114 value = self.value
115 115 format = self.format
116 116
117 117 if self.__formated_value != None:
118 118
119 119 return self.__formated_value
120 120
121 121 if format == 'obj':
122 122 return value
123 123
124 124 if format == 'str':
125 125 self.__formated_value = str(value)
126 126 return self.__formated_value
127 127
128 128 if value == '':
129 129 raise ValueError('%s: This parameter value is empty' % self.name)
130 130
131 131 if format == 'list':
132 132 strList = [s.strip() for s in value.split(',')]
133 133 self.__formated_value = strList
134 134
135 135 return self.__formated_value
136 136
137 137 if format == 'intlist':
138 138 '''
139 139 Example:
140 140 value = (0,1,2)
141 141 '''
142 142
143 143 new_value = ast.literal_eval(value)
144 144
145 145 if type(new_value) not in (tuple, list):
146 146 new_value = [int(new_value)]
147 147
148 148 self.__formated_value = new_value
149 149
150 150 return self.__formated_value
151 151
152 152 if format == 'floatlist':
153 153 '''
154 154 Example:
155 155 value = (0.5, 1.4, 2.7)
156 156 '''
157 157
158 158 new_value = ast.literal_eval(value)
159 159
160 160 if type(new_value) not in (tuple, list):
161 161 new_value = [float(new_value)]
162 162
163 163 self.__formated_value = new_value
164 164
165 165 return self.__formated_value
166 166
167 167 if format == 'date':
168 168 strList = value.split('/')
169 169 intList = [int(x) for x in strList]
170 170 date = datetime.date(intList[0], intList[1], intList[2])
171 171
172 172 self.__formated_value = date
173 173
174 174 return self.__formated_value
175 175
176 176 if format == 'time':
177 177 strList = value.split(':')
178 178 intList = [int(x) for x in strList]
179 179 time = datetime.time(intList[0], intList[1], intList[2])
180 180
181 181 self.__formated_value = time
182 182
183 183 return self.__formated_value
184 184
185 185 if format == 'pairslist':
186 186 '''
187 187 Example:
188 188 value = (0,1),(1,2)
189 189 '''
190 190
191 191 new_value = ast.literal_eval(value)
192 192
193 193 if type(new_value) not in (tuple, list):
194 194 raise ValueError('%s has to be a tuple or list of pairs' % value)
195 195
196 196 if type(new_value[0]) not in (tuple, list):
197 197 if len(new_value) != 2:
198 198 raise ValueError('%s has to be a tuple or list of pairs' % value)
199 199 new_value = [new_value]
200 200
201 201 for thisPair in new_value:
202 202 if len(thisPair) != 2:
203 203 raise ValueError('%s has to be a tuple or list of pairs' % value)
204 204
205 205 self.__formated_value = new_value
206 206
207 207 return self.__formated_value
208 208
209 209 if format == 'multilist':
210 210 '''
211 211 Example:
212 212 value = (0,1,2),(3,4,5)
213 213 '''
214 214 multiList = ast.literal_eval(value)
215 215
216 216 if type(multiList[0]) == int:
217 217 multiList = ast.literal_eval('(' + value + ')')
218 218
219 219 self.__formated_value = multiList
220 220
221 221 return self.__formated_value
222 222
223 223 if format == 'bool':
224 224 value = int(value)
225 225
226 226 if format == 'int':
227 227 value = float(value)
228 228
229 229 format_func = eval(format)
230 230
231 231 self.__formated_value = format_func(value)
232 232
233 233 return self.__formated_value
234 234
235 235 def updateId(self, new_id):
236 236
237 237 self.id = str(new_id)
238 238
239 239 def setup(self, id, name, value, format='str'):
240 240 self.id = str(id)
241 241 self.name = name
242 242 if format == 'obj':
243 243 self.value = value
244 244 else:
245 245 self.value = str(value)
246 246 self.format = str.lower(format)
247 247
248 248 self.getValue()
249 249
250 250 return 1
251 251
252 252 def update(self, name, value, format='str'):
253 253
254 254 self.name = name
255 255 self.value = str(value)
256 256 self.format = format
257 257
258 258 def makeXml(self, opElement):
259 259 if self.name not in ('queue',):
260 260 parmElement = SubElement(opElement, self.ELEMENTNAME)
261 261 parmElement.set('id', str(self.id))
262 262 parmElement.set('name', self.name)
263 263 parmElement.set('value', self.value)
264 264 parmElement.set('format', self.format)
265 265
266 266 def readXml(self, parmElement):
267 267
268 268 self.id = parmElement.get('id')
269 269 self.name = parmElement.get('name')
270 270 self.value = parmElement.get('value')
271 271 self.format = str.lower(parmElement.get('format'))
272 272
273 273 # Compatible with old signal chain version
274 274 if self.format == 'int' and self.name == 'idfigure':
275 275 self.name = 'id'
276 276
277 277 def printattr(self):
278 278
279 279 print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
280 280
281 281 class OperationConf():
282 282
283 283 ELEMENTNAME = 'Operation'
284 284
285 285 def __init__(self):
286 286
287 287 self.id = '0'
288 288 self.name = None
289 289 self.priority = None
290 290 self.topic = None
291 291
292 292 def __getNewId(self):
293 293
294 294 return int(self.id) * 10 + len(self.parmConfObjList) + 1
295 295
296 296 def getId(self):
297 297 return self.id
298 298
299 299 def updateId(self, new_id):
300 300
301 301 self.id = str(new_id)
302 302
303 303 n = 1
304 304 for parmObj in self.parmConfObjList:
305 305
306 306 idParm = str(int(new_id) * 10 + n)
307 307 parmObj.updateId(idParm)
308 308
309 309 n += 1
310 310
311 311 def getElementName(self):
312 312
313 313 return self.ELEMENTNAME
314 314
315 315 def getParameterObjList(self):
316 316
317 317 return self.parmConfObjList
318 318
319 319 def getParameterObj(self, parameterName):
320 320
321 321 for parmConfObj in self.parmConfObjList:
322 322
323 323 if parmConfObj.name != parameterName:
324 324 continue
325 325
326 326 return parmConfObj
327 327
328 328 return None
329 329
330 330 def getParameterObjfromValue(self, parameterValue):
331 331
332 332 for parmConfObj in self.parmConfObjList:
333 333
334 334 if parmConfObj.getValue() != parameterValue:
335 335 continue
336 336
337 337 return parmConfObj.getValue()
338 338
339 339 return None
340 340
341 341 def getParameterValue(self, parameterName):
342 342
343 343 parameterObj = self.getParameterObj(parameterName)
344 344
345 345 # if not parameterObj:
346 346 # return None
347 347
348 348 value = parameterObj.getValue()
349 349
350 350 return value
351 351
352 352 def getKwargs(self):
353 353
354 354 kwargs = {}
355 355
356 356 for parmConfObj in self.parmConfObjList:
357 357 if self.name == 'run' and parmConfObj.name == 'datatype':
358 358 continue
359 359
360 360 kwargs[parmConfObj.name] = parmConfObj.getValue()
361 361
362 362 return kwargs
363 363
364 def setup(self, id, name, priority, type, project_id):
364 def setup(self, id, name, priority, type, project_id, err_queue):
365 365
366 366 self.id = str(id)
367 367 self.project_id = project_id
368 368 self.name = name
369 369 self.type = type
370 370 self.priority = priority
371 self.err_queue = err_queue
371 372 self.parmConfObjList = []
372 373
373 374 def removeParameters(self):
374 375
375 376 for obj in self.parmConfObjList:
376 377 del obj
377 378
378 379 self.parmConfObjList = []
379 380
380 381 def addParameter(self, name, value, format='str'):
381 382
382 383 if value is None:
383 384 return None
384 385 id = self.__getNewId()
385 386
386 387 parmConfObj = ParameterConf()
387 388 if not parmConfObj.setup(id, name, value, format):
388 389 return None
389 390
390 391 self.parmConfObjList.append(parmConfObj)
391 392
392 393 return parmConfObj
393 394
394 395 def changeParameter(self, name, value, format='str'):
395 396
396 397 parmConfObj = self.getParameterObj(name)
397 398 parmConfObj.update(name, value, format)
398 399
399 400 return parmConfObj
400 401
401 402 def makeXml(self, procUnitElement):
402 403
403 404 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
404 405 opElement.set('id', str(self.id))
405 406 opElement.set('name', self.name)
406 407 opElement.set('type', self.type)
407 408 opElement.set('priority', str(self.priority))
408 409
409 410 for parmConfObj in self.parmConfObjList:
410 411 parmConfObj.makeXml(opElement)
411 412
412 413 def readXml(self, opElement, project_id):
413 414
414 415 self.id = opElement.get('id')
415 416 self.name = opElement.get('name')
416 417 self.type = opElement.get('type')
417 418 self.priority = opElement.get('priority')
418 419 self.project_id = str(project_id)
419 420
420 421 # Compatible with old signal chain version
421 422 # Use of 'run' method instead 'init'
422 423 if self.type == 'self' and self.name == 'init':
423 424 self.name = 'run'
424 425
425 426 self.parmConfObjList = []
426 427
427 428 parmElementList = opElement.iter(ParameterConf().getElementName())
428 429
429 430 for parmElement in parmElementList:
430 431 parmConfObj = ParameterConf()
431 432 parmConfObj.readXml(parmElement)
432 433
433 434 # Compatible with old signal chain version
434 435 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
435 436 if self.type != 'self' and self.name == 'Plot':
436 437 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
437 438 self.name = parmConfObj.value
438 439 continue
439 440
440 441 self.parmConfObjList.append(parmConfObj)
441 442
442 443 def printattr(self):
443 444
444 445 print('%s[%s]: name = %s, type = %s, priority = %s, project_id = %s' % (self.ELEMENTNAME,
445 446 self.id,
446 447 self.name,
447 448 self.type,
448 449 self.priority,
449 450 self.project_id))
450 451
451 452 for parmConfObj in self.parmConfObjList:
452 453 parmConfObj.printattr()
453 454
454 455 def createObject(self):
455 456
456 457 className = eval(self.name)
457 458
458 459 if self.type == 'other':
459 460 opObj = className()
460 461 elif self.type == 'external':
461 462 kwargs = self.getKwargs()
462 opObj = className(self.id, self.project_id, **kwargs)
463 opObj = className(self.id, self.id, self.project_id, self.err_queue, 'Operation', **kwargs)
463 464 opObj.start()
465 self.opObj = opObj
464 466
465 467 return opObj
466 468
467 469 class ProcUnitConf():
468 470
469 471 ELEMENTNAME = 'ProcUnit'
470 472
471 473 def __init__(self):
472 474
473 475 self.id = None
474 476 self.datatype = None
475 477 self.name = None
476 478 self.inputId = None
477 479 self.opConfObjList = []
478 480 self.procUnitObj = None
479 481 self.opObjDict = {}
480 482
481 483 def __getPriority(self):
482 484
483 485 return len(self.opConfObjList) + 1
484 486
485 487 def __getNewId(self):
486 488
487 489 return int(self.id) * 10 + len(self.opConfObjList) + 1
488 490
489 491 def getElementName(self):
490 492
491 493 return self.ELEMENTNAME
492 494
493 495 def getId(self):
494 496
495 497 return self.id
496 498
497 499 def updateId(self, new_id):
498 500 '''
499 501 new_id = int(parentId) * 10 + (int(self.id) % 10)
500 502 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
501 503
502 504 # If this proc unit has not inputs
503 505 #if self.inputId == '0':
504 506 #new_inputId = 0
505 507
506 508 n = 1
507 509 for opConfObj in self.opConfObjList:
508 510
509 511 idOp = str(int(new_id) * 10 + n)
510 512 opConfObj.updateId(idOp)
511 513
512 514 n += 1
513 515
514 516 self.parentId = str(parentId)
515 517 self.id = str(new_id)
516 518 #self.inputId = str(new_inputId)
517 519 '''
518 520 n = 1
519 521
520 522 def getInputId(self):
521 523
522 524 return self.inputId
523 525
524 526 def getOperationObjList(self):
525 527
526 528 return self.opConfObjList
527 529
528 530 def getOperationObj(self, name=None):
529 531
530 532 for opConfObj in self.opConfObjList:
531 533
532 534 if opConfObj.name != name:
533 535 continue
534 536
535 537 return opConfObj
536 538
537 539 return None
538 540
539 541 def getOpObjfromParamValue(self, value=None):
540 542
541 543 for opConfObj in self.opConfObjList:
542 544 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
543 545 continue
544 546 return opConfObj
545 547 return None
546 548
547 549 def getProcUnitObj(self):
548 550
549 551 return self.procUnitObj
550 552
551 def setup(self, project_id, id, name, datatype, inputId):
553 def setup(self, project_id, id, name, datatype, inputId, err_queue):
552 554 '''
553 555 id sera el topico a publicar
554 556 inputId sera el topico a subscribirse
555 557 '''
556 558
557 559 # Compatible with old signal chain version
558 560 if datatype == None and name == None:
559 561 raise ValueError('datatype or name should be defined')
560 562
561 563 #Definir una condicion para inputId cuando sea 0
562 564
563 565 if name == None:
564 566 if 'Proc' in datatype:
565 567 name = datatype
566 568 else:
567 569 name = '%sProc' % (datatype)
568 570
569 571 if datatype == None:
570 572 datatype = name.replace('Proc', '')
571 573
572 574 self.id = str(id)
573 575 self.project_id = project_id
574 576 self.name = name
575 577 self.datatype = datatype
576 self.inputId = inputId
578 self.inputId = inputId
579 self.err_queue = err_queue
577 580 self.opConfObjList = []
578 581
579 582 self.addOperation(name='run', optype='self')
580 583
581 584 def removeOperations(self):
582 585
583 586 for obj in self.opConfObjList:
584 587 del obj
585 588
586 589 self.opConfObjList = []
587 590 self.addOperation(name='run')
588 591
589 592 def addParameter(self, **kwargs):
590 593 '''
591 594 Add parameters to 'run' operation
592 595 '''
593 596 opObj = self.opConfObjList[0]
594 597
595 598 opObj.addParameter(**kwargs)
596 599
597 600 return opObj
598 601
599 602 def addOperation(self, name, optype='self'):
600 603 '''
601 604 Actualizacion - > proceso comunicacion
602 605 En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
603 606 definir el tipoc de socket o comunicacion ipc++
604 607
605 608 '''
606 609
607 610 id = self.__getNewId()
608 611 priority = self.__getPriority() # Sin mucho sentido, pero puede usarse
609 612 opConfObj = OperationConf()
610 opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id)
613 opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id, err_queue=self.err_queue)
611 614 self.opConfObjList.append(opConfObj)
612 615
613 616 return opConfObj
614 617
615 618 def makeXml(self, projectElement):
616 619
617 620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
618 621 procUnitElement.set('id', str(self.id))
619 622 procUnitElement.set('name', self.name)
620 623 procUnitElement.set('datatype', self.datatype)
621 624 procUnitElement.set('inputId', str(self.inputId))
622 625
623 626 for opConfObj in self.opConfObjList:
624 627 opConfObj.makeXml(procUnitElement)
625 628
626 629 def readXml(self, upElement, project_id):
627 630
628 631 self.id = upElement.get('id')
629 632 self.name = upElement.get('name')
630 633 self.datatype = upElement.get('datatype')
631 634 self.inputId = upElement.get('inputId')
632 635 self.project_id = str(project_id)
633 636
634 637 if self.ELEMENTNAME == 'ReadUnit':
635 638 self.datatype = self.datatype.replace('Reader', '')
636 639
637 640 if self.ELEMENTNAME == 'ProcUnit':
638 641 self.datatype = self.datatype.replace('Proc', '')
639 642
640 643 if self.inputId == 'None':
641 644 self.inputId = '0'
642 645
643 646 self.opConfObjList = []
644 647
645 648 opElementList = upElement.iter(OperationConf().getElementName())
646 649
647 650 for opElement in opElementList:
648 651 opConfObj = OperationConf()
649 652 opConfObj.readXml(opElement, project_id)
650 653 self.opConfObjList.append(opConfObj)
651 654
652 655 def printattr(self):
653 656
654 657 print('%s[%s]: name = %s, datatype = %s, inputId = %s, project_id = %s' % (self.ELEMENTNAME,
655 658 self.id,
656 659 self.name,
657 660 self.datatype,
658 661 self.inputId,
659 662 self.project_id))
660 663
661 664 for opConfObj in self.opConfObjList:
662 665 opConfObj.printattr()
663 666
664 667 def getKwargs(self):
665 668
666 669 opObj = self.opConfObjList[0]
667 670 kwargs = opObj.getKwargs()
668 671
669 672 return kwargs
670 673
671 674 def createObjects(self):
672 675 '''
673 676 Instancia de unidades de procesamiento.
674 677 '''
675 678
676 679 className = eval(self.name)
677 680 kwargs = self.getKwargs()
678 procUnitObj = className(self.id, self.inputId, self.project_id, **kwargs) # necesitan saber su id y su entrada por fines de ipc
681 procUnitObj = className(self.id, self.inputId, self.project_id, self.err_queue, 'ProcUnit', **kwargs)
679 682 log.success('creating process...', self.name)
680 683
681 684 for opConfObj in self.opConfObjList:
682 685
683 686 if opConfObj.type == 'self' and opConfObj.name == 'run':
684 687 continue
685 688 elif opConfObj.type == 'self':
686 689 opObj = getattr(procUnitObj, opConfObj.name)
687 690 else:
688 691 opObj = opConfObj.createObject()
689 692
690 log.success('creating operation: {}, type:{}'.format(
693 log.success('adding operation: {}, type:{}'.format(
691 694 opConfObj.name,
692 695 opConfObj.type), self.name)
693 696
694 697 procUnitObj.addOperation(opConfObj, opObj)
695 698
696 699 procUnitObj.start()
697 700 self.procUnitObj = procUnitObj
698 701
699 702 def close(self):
700 703
701 704 for opConfObj in self.opConfObjList:
702 705 if opConfObj.type == 'self':
703 706 continue
704 707
705 708 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
706 709 opObj.close()
707 710
708 711 self.procUnitObj.close()
709 712
710 713 return
711 714
712 715
713 716 class ReadUnitConf(ProcUnitConf):
714 717
715 718 ELEMENTNAME = 'ReadUnit'
716 719
717 720 def __init__(self):
718 721
719 722 self.id = None
720 723 self.datatype = None
721 724 self.name = None
722 725 self.inputId = None
723 726 self.opConfObjList = []
724 727
725 728 def getElementName(self):
726 729
727 730 return self.ELEMENTNAME
728 731
729 def setup(self, project_id, id, name, datatype, path='', startDate='', endDate='',
732 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
730 733 startTime='', endTime='', server=None, **kwargs):
731 734
732 735
733 736 '''
734 737 *****el id del proceso sera el Topico
735 738
736 739 Adicion de {topic}, si no esta presente -> error
737 740 kwargs deben ser trasmitidos en la instanciacion
738 741
739 742 '''
740 743
741 744 # Compatible with old signal chain version
742 745 if datatype == None and name == None:
743 746 raise ValueError('datatype or name should be defined')
744 747 if name == None:
745 748 if 'Reader' in datatype:
746 749 name = datatype
747 750 datatype = name.replace('Reader','')
748 751 else:
749 752 name = '{}Reader'.format(datatype)
750 753 if datatype == None:
751 754 if 'Reader' in name:
752 755 datatype = name.replace('Reader','')
753 756 else:
754 757 datatype = name
755 758 name = '{}Reader'.format(name)
756 759
757 760 self.id = id
758 761 self.project_id = project_id
759 762 self.name = name
760 763 self.datatype = datatype
761 764 if path != '':
762 765 self.path = os.path.abspath(path)
763 766 self.startDate = startDate
764 767 self.endDate = endDate
765 768 self.startTime = startTime
766 769 self.endTime = endTime
767 770 self.server = server
771 self.err_queue = err_queue
768 772 self.addRunOperation(**kwargs)
769 773
770 774 def update(self, **kwargs):
771 775
772 776 if 'datatype' in kwargs:
773 777 datatype = kwargs.pop('datatype')
774 778 if 'Reader' in datatype:
775 779 self.name = datatype
776 780 else:
777 781 self.name = '%sReader' % (datatype)
778 782 self.datatype = self.name.replace('Reader', '')
779 783
780 784 attrs = ('path', 'startDate', 'endDate',
781 785 'startTime', 'endTime')
782 786
783 787 for attr in attrs:
784 788 if attr in kwargs:
785 789 setattr(self, attr, kwargs.pop(attr))
786 790
787 791 self.updateRunOperation(**kwargs)
788 792
789 793 def removeOperations(self):
790 794
791 795 for obj in self.opConfObjList:
792 796 del obj
793 797
794 798 self.opConfObjList = []
795 799
796 800 def addRunOperation(self, **kwargs):
797 801
798 802 opObj = self.addOperation(name='run', optype='self')
799 803
800 804 if self.server is None:
801 805 opObj.addParameter(
802 806 name='datatype', value=self.datatype, format='str')
803 807 opObj.addParameter(name='path', value=self.path, format='str')
804 808 opObj.addParameter(
805 809 name='startDate', value=self.startDate, format='date')
806 810 opObj.addParameter(
807 811 name='endDate', value=self.endDate, format='date')
808 812 opObj.addParameter(
809 813 name='startTime', value=self.startTime, format='time')
810 814 opObj.addParameter(
811 815 name='endTime', value=self.endTime, format='time')
812 816
813 817 for key, value in list(kwargs.items()):
814 818 opObj.addParameter(name=key, value=value,
815 819 format=type(value).__name__)
816 820 else:
817 821 opObj.addParameter(name='server', value=self.server, format='str')
818 822
819 823 return opObj
820 824
821 825 def updateRunOperation(self, **kwargs):
822 826
823 827 opObj = self.getOperationObj(name='run')
824 828 opObj.removeParameters()
825 829
826 830 opObj.addParameter(name='datatype', value=self.datatype, format='str')
827 831 opObj.addParameter(name='path', value=self.path, format='str')
828 832 opObj.addParameter(
829 833 name='startDate', value=self.startDate, format='date')
830 834 opObj.addParameter(name='endDate', value=self.endDate, format='date')
831 835 opObj.addParameter(
832 836 name='startTime', value=self.startTime, format='time')
833 837 opObj.addParameter(name='endTime', value=self.endTime, format='time')
834 838
835 839 for key, value in list(kwargs.items()):
836 840 opObj.addParameter(name=key, value=value,
837 841 format=type(value).__name__)
838 842
839 843 return opObj
840 844
841 845 def readXml(self, upElement, project_id):
842 846
843 847 self.id = upElement.get('id')
844 848 self.name = upElement.get('name')
845 849 self.datatype = upElement.get('datatype')
846 850 self.project_id = str(project_id) #yong
847 851
848 852 if self.ELEMENTNAME == 'ReadUnit':
849 853 self.datatype = self.datatype.replace('Reader', '')
850 854
851 855 self.opConfObjList = []
852 856
853 857 opElementList = upElement.iter(OperationConf().getElementName())
854 858
855 859 for opElement in opElementList:
856 860 opConfObj = OperationConf()
857 861 opConfObj.readXml(opElement, project_id)
858 862 self.opConfObjList.append(opConfObj)
859 863
860 864 if opConfObj.name == 'run':
861 865 self.path = opConfObj.getParameterValue('path')
862 866 self.startDate = opConfObj.getParameterValue('startDate')
863 867 self.endDate = opConfObj.getParameterValue('endDate')
864 868 self.startTime = opConfObj.getParameterValue('startTime')
865 869 self.endTime = opConfObj.getParameterValue('endTime')
866 870
867 871
868 872 class Project(Process):
869 873
870 874 ELEMENTNAME = 'Project'
871 875
872 876 def __init__(self):
873 877
874 878 Process.__init__(self)
875 879 self.id = None
876 880 self.filename = None
877 881 self.description = None
878 882 self.email = None
879 883 self.alarm = None
880 884 self.procUnitConfObjDict = {}
885 self.err_queue = Queue()
881 886
882 887 def __getNewId(self):
883 888
884 889 idList = list(self.procUnitConfObjDict.keys())
885 890 id = int(self.id) * 10
886 891
887 892 while True:
888 893 id += 1
889 894
890 895 if str(id) in idList:
891 896 continue
892 897
893 898 break
894 899
895 900 return str(id)
896 901
897 902 def getElementName(self):
898 903
899 904 return self.ELEMENTNAME
900 905
901 906 def getId(self):
902 907
903 908 return self.id
904 909
905 910 def updateId(self, new_id):
906 911
907 912 self.id = str(new_id)
908 913
909 914 keyList = list(self.procUnitConfObjDict.keys())
910 915 keyList.sort()
911 916
912 917 n = 1
913 918 newProcUnitConfObjDict = {}
914 919
915 920 for procKey in keyList:
916 921
917 922 procUnitConfObj = self.procUnitConfObjDict[procKey]
918 923 idProcUnit = str(int(self.id) * 10 + n)
919 924 procUnitConfObj.updateId(idProcUnit)
920 925 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
921 926 n += 1
922 927
923 928 self.procUnitConfObjDict = newProcUnitConfObjDict
924 929
925 930 def setup(self, id=1, name='', description='', email=None, alarm=[]):
926 931
927 932 print(' ')
928 933 print('*' * 60)
929 934 print('* Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
930 935 print('*' * 60)
931 936 print("* Python " + python_version() + " *")
932 937 print('*' * 19)
933 938 print(' ')
934 939 self.id = str(id)
935 940 self.description = description
936 941 self.email = email
937 942 self.alarm = alarm
943 if name:
944 self.name = '{} ({})'.format(Process.__name__, name)
938 945
939 946 def update(self, **kwargs):
940 947
941 948 for key, value in list(kwargs.items()):
942 949 setattr(self, key, value)
943 950
944 951 def clone(self):
945 952
946 953 p = Project()
947 954 p.procUnitConfObjDict = self.procUnitConfObjDict
948 955 return p
949 956
950 957 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
951 958
952 959 '''
953 960 Actualizacion:
954 961 Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
955 962
956 963 * El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
957 964
958 965 '''
959 966
960 967 if id is None:
961 968 idReadUnit = self.__getNewId()
962 969 else:
963 970 idReadUnit = str(id)
964 971
965 972 readUnitConfObj = ReadUnitConf()
966 readUnitConfObj.setup(self.id, idReadUnit, name, datatype, **kwargs)
973 readUnitConfObj.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
967 974 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
968 975
969 976 return readUnitConfObj
970 977
971 978 def addProcUnit(self, inputId='0', datatype=None, name=None):
972 979
973 980 '''
974 981 Actualizacion:
975 982 Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
976 983 Deberia reemplazar a "inputId"
977 984
978 985 ** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
979 986 (proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
980 987
981 988 '''
982 989
983 idProcUnit = self.__getNewId() #Topico para subscripcion
990 idProcUnit = self.__getNewId()
984 991 procUnitConfObj = ProcUnitConf()
985 procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId) #topic_read, topic_write,
992 procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
986 993 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
987 994
988 995 return procUnitConfObj
989 996
990 997 def removeProcUnit(self, id):
991 998
992 999 if id in list(self.procUnitConfObjDict.keys()):
993 1000 self.procUnitConfObjDict.pop(id)
994 1001
995 1002 def getReadUnitId(self):
996 1003
997 1004 readUnitConfObj = self.getReadUnitObj()
998 1005
999 1006 return readUnitConfObj.id
1000 1007
1001 1008 def getReadUnitObj(self):
1002 1009
1003 1010 for obj in list(self.procUnitConfObjDict.values()):
1004 1011 if obj.getElementName() == 'ReadUnit':
1005 1012 return obj
1006 1013
1007 1014 return None
1008 1015
1009 1016 def getProcUnitObj(self, id=None, name=None):
1010 1017
1011 1018 if id != None:
1012 1019 return self.procUnitConfObjDict[id]
1013 1020
1014 1021 if name != None:
1015 1022 return self.getProcUnitObjByName(name)
1016 1023
1017 1024 return None
1018 1025
1019 1026 def getProcUnitObjByName(self, name):
1020 1027
1021 1028 for obj in list(self.procUnitConfObjDict.values()):
1022 1029 if obj.name == name:
1023 1030 return obj
1024 1031
1025 1032 return None
1026 1033
1027 1034 def procUnitItems(self):
1028 1035
1029 1036 return list(self.procUnitConfObjDict.items())
1030 1037
1031 1038 def makeXml(self):
1032 1039
1033 1040 projectElement = Element('Project')
1034 1041 projectElement.set('id', str(self.id))
1035 1042 projectElement.set('name', self.name)
1036 1043 projectElement.set('description', self.description)
1037 1044
1038 1045 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1039 1046 procUnitConfObj.makeXml(projectElement)
1040 1047
1041 1048 self.projectElement = projectElement
1042 1049
1043 1050 def writeXml(self, filename=None):
1044 1051
1045 1052 if filename == None:
1046 1053 if self.filename:
1047 1054 filename = self.filename
1048 1055 else:
1049 1056 filename = 'schain.xml'
1050 1057
1051 1058 if not filename:
1052 1059 print('filename has not been defined. Use setFilename(filename) for do it.')
1053 1060 return 0
1054 1061
1055 1062 abs_file = os.path.abspath(filename)
1056 1063
1057 1064 if not os.access(os.path.dirname(abs_file), os.W_OK):
1058 1065 print('No write permission on %s' % os.path.dirname(abs_file))
1059 1066 return 0
1060 1067
1061 1068 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1062 1069 print('File %s already exists and it could not be overwriten' % abs_file)
1063 1070 return 0
1064 1071
1065 1072 self.makeXml()
1066 1073
1067 1074 ElementTree(self.projectElement).write(abs_file, method='xml')
1068 1075
1069 1076 self.filename = abs_file
1070 1077
1071 1078 return 1
1072 1079
1073 1080 def readXml(self, filename=None):
1074 1081
1075 1082 if not filename:
1076 1083 print('filename is not defined')
1077 1084 return 0
1078 1085
1079 1086 abs_file = os.path.abspath(filename)
1080 1087
1081 1088 if not os.path.isfile(abs_file):
1082 1089 print('%s file does not exist' % abs_file)
1083 1090 return 0
1084 1091
1085 1092 self.projectElement = None
1086 1093 self.procUnitConfObjDict = {}
1087 1094
1088 1095 try:
1089 1096 self.projectElement = ElementTree().parse(abs_file)
1090 1097 except:
1091 1098 print('Error reading %s, verify file format' % filename)
1092 1099 return 0
1093 1100
1094 1101 self.project = self.projectElement.tag
1095 1102
1096 1103 self.id = self.projectElement.get('id')
1097 1104 self.name = self.projectElement.get('name')
1098 1105 self.description = self.projectElement.get('description')
1099 1106
1100 1107 readUnitElementList = self.projectElement.iter(
1101 1108 ReadUnitConf().getElementName())
1102 1109
1103 1110 for readUnitElement in readUnitElementList:
1104 1111 readUnitConfObj = ReadUnitConf()
1105 1112 readUnitConfObj.readXml(readUnitElement, self.id)
1106 1113 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1107 1114
1108 1115 procUnitElementList = self.projectElement.iter(
1109 1116 ProcUnitConf().getElementName())
1110 1117
1111 1118 for procUnitElement in procUnitElementList:
1112 1119 procUnitConfObj = ProcUnitConf()
1113 1120 procUnitConfObj.readXml(procUnitElement, self.id)
1114 1121 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1115 1122
1116 1123 self.filename = abs_file
1117 1124
1118 1125 return 1
1119 1126
1120 1127 def __str__(self):
1121 1128
1122 print('Project[%s]: name = %s, description = %s, project_id = %s' % (self.id,
1129 print('Project: name = %s, description = %s, id = %s' % (
1123 1130 self.name,
1124 1131 self.description,
1125 self.project_id))
1132 self.id))
1126 1133
1127 1134 for procUnitConfObj in self.procUnitConfObjDict.values():
1128 1135 print(procUnitConfObj)
1129 1136
1130 1137 def createObjects(self):
1131 1138
1132 1139
1133 1140 keys = list(self.procUnitConfObjDict.keys())
1134 1141 keys.sort()
1135 1142 for key in keys:
1136 1143 self.procUnitConfObjDict[key].createObjects()
1137 1144
1138 def __handleError(self, procUnitConfObj, modes=None, stdout=True):
1145 def monitor(self):
1139 1146
1140 import socket
1147 t = Thread(target=self.__monitor, args=(self.err_queue, self.ctx))
1148 t.start()
1149
1150 def __monitor(self, queue, ctx):
1141 1151
1142 if modes is None:
1143 modes = self.alarm
1152 import socket
1144 1153
1145 if not self.alarm:
1146 modes = []
1147
1148 err = traceback.format_exception(sys.exc_info()[0],
1149 sys.exc_info()[1],
1150 sys.exc_info()[2])
1154 procs = 0
1155 err_msg = ''
1156
1157 while True:
1158 msg = queue.get()
1159 if '#_start_#' in msg:
1160 procs += 1
1161 elif '#_end_#' in msg:
1162 procs -=1
1163 else:
1164 err_msg = msg
1165
1166 if procs == 0 or 'Traceback' in err_msg:
1167 break
1168 time.sleep(0.1)
1169
1170 if '|' in err_msg:
1171 name, err = err_msg.split('|')
1172 if 'SchainWarning' in err:
1173 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
1174 elif 'SchainError' in err:
1175 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
1176 else:
1177 log.error(err, name)
1178 else:
1179 name, err = self.name, err_msg
1180
1181 time.sleep(2)
1151 1182
1152 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1183 for conf in self.procUnitConfObjDict.values():
1184 for confop in conf.opConfObjList:
1185 if confop.type == 'external':
1186 confop.opObj.terminate()
1187 conf.procUnitObj.terminate()
1188
1189 ctx.term()
1153 1190
1154 1191 message = ''.join(err)
1155 1192
1156 if stdout:
1157 sys.stderr.write(message)
1158
1159 subject = 'SChain v%s: Error running %s\n' % (
1160 schainpy.__version__, procUnitConfObj.name)
1161
1162 subtitle = '%s: %s\n' % (
1163 procUnitConfObj.getElementName(), procUnitConfObj.name)
1164 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1165 socket.gethostname())
1166 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1167 subtitle += 'Configuration file: %s\n' % self.filename
1168 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1169
1170 readUnitConfObj = self.getReadUnitObj()
1171 if readUnitConfObj:
1172 subtitle += '\nInput parameters:\n'
1173 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1174 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1175 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1176 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1177 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1178 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1179
1180 a = Alarm(
1181 modes=modes,
1182 email=self.email,
1183 message=message,
1184 subject=subject,
1185 subtitle=subtitle,
1186 filename=self.filename
1187 )
1188
1189 return a
1193 if err_msg:
1194 subject = 'SChain v%s: Error running %s\n' % (
1195 schainpy.__version__, self.name)
1196
1197 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
1198 socket.gethostname())
1199 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1200 subtitle += 'Configuration file: %s\n' % self.filename
1201 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1202
1203 readUnitConfObj = self.getReadUnitObj()
1204 if readUnitConfObj:
1205 subtitle += '\nInput parameters:\n'
1206 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1207 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1208 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1209 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1210 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1211 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1212
1213 a = Alarm(
1214 modes=self.alarm,
1215 email=self.email,
1216 message=message,
1217 subject=subject,
1218 subtitle=subtitle,
1219 filename=self.filename
1220 )
1221
1222 a.start()
1190 1223
1191 1224 def isPaused(self):
1192 1225 return 0
1193 1226
1194 1227 def isStopped(self):
1195 1228 return 0
1196 1229
1197 1230 def runController(self):
1198 1231 '''
1199 1232 returns 0 when this process has been stopped, 1 otherwise
1200 1233 '''
1201 1234
1202 1235 if self.isPaused():
1203 1236 print('Process suspended')
1204 1237
1205 1238 while True:
1206 1239 time.sleep(0.1)
1207 1240
1208 1241 if not self.isPaused():
1209 1242 break
1210 1243
1211 1244 if self.isStopped():
1212 1245 break
1213 1246
1214 1247 print('Process reinitialized')
1215 1248
1216 1249 if self.isStopped():
1217 1250 print('Process stopped')
1218 1251 return 0
1219 1252
1220 1253 return 1
1221 1254
1222 1255 def setFilename(self, filename):
1223 1256
1224 1257 self.filename = filename
1225 1258
1226 def setProxyCom(self):
1259 def setProxy(self):
1227 1260
1228 1261 if not os.path.exists('/tmp/schain'):
1229 1262 os.mkdir('/tmp/schain')
1230 1263
1231 1264 self.ctx = zmq.Context()
1232 1265 xpub = self.ctx.socket(zmq.XPUB)
1233 1266 xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))
1234 1267 xsub = self.ctx.socket(zmq.XSUB)
1235 1268 xsub.bind('ipc:///tmp/schain/{}_sub'.format(self.id))
1236
1269 self.monitor()
1237 1270 try:
1238 1271 zmq.proxy(xpub, xsub)
1239 except: # zmq.ContextTerminated:
1272 except zmq.ContextTerminated:
1240 1273 xpub.close()
1241 1274 xsub.close()
1242 1275
1243 1276 def run(self):
1244 1277
1245 1278 log.success('Starting {}: {}'.format(self.name, self.id), tag='')
1246 self.start_time = time.time()
1247 self.createObjects()
1248 # t = Thread(target=wait, args=(self.ctx, ))
1249 # t.start()
1250 self.setProxyCom()
1251
1252 # Iniciar todos los procesos .start(), monitoreo de procesos. ELiminar lo de abajo
1253
1254 log.success('{} Done (time: {}s)'.format(
1279 self.start_time = time.time()
1280 self.createObjects()
1281 self.setProxy()
1282 log.success('{} Done (Time: {}s)'.format(
1255 1283 self.name,
1256 time.time()-self.start_time))
1284 time.time()-self.start_time), '')
Show file before | Show file after | Hide comments
@@ -1,407 +1,407
1 1 '''
2 2 Created on Nov 9, 2016
3 3
4 4 @author: roj- LouVD
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 import schainpy.admin
16 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
17 18 from schainpy.model.data.jrodata import Parameters
18 19 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19 20 from schainpy.utils import log
20 21
21 22 FILE_HEADER_STRUCTURE = numpy.dtype([
22 23 ('FMN', '<u4'),
23 24 ('nrec', '<u4'),
24 25 ('fr_offset', '<u4'),
25 26 ('id', '<u4'),
26 27 ('site', 'u1', (32,))
27 28 ])
28 29
29 30 REC_HEADER_STRUCTURE = numpy.dtype([
30 31 ('rmn', '<u4'),
31 32 ('rcounter', '<u4'),
32 33 ('nr_offset', '<u4'),
33 34 ('tr_offset', '<u4'),
34 35 ('time', '<u4'),
35 36 ('time_msec', '<u4'),
36 37 ('tag', 'u1', (32,)),
37 38 ('comments', 'u1', (32,)),
38 39 ('lat', '<f4'),
39 40 ('lon', '<f4'),
40 41 ('gps_status', '<u4'),
41 42 ('freq', '<u4'),
42 43 ('freq0', '<u4'),
43 44 ('nchan', '<u4'),
44 45 ('delta_r', '<u4'),
45 46 ('nranges', '<u4'),
46 47 ('r0', '<u4'),
47 48 ('prf', '<u4'),
48 49 ('ncoh', '<u4'),
49 50 ('npoints', '<u4'),
50 51 ('polarization', '<i4'),
51 52 ('rx_filter', '<u4'),
52 53 ('nmodes', '<u4'),
53 54 ('dmode_index', '<u4'),
54 55 ('dmode_rngcorr', '<u4'),
55 56 ('nrxs', '<u4'),
56 57 ('acf_length', '<u4'),
57 58 ('acf_lags', '<u4'),
58 59 ('sea_to_atmos', '<f4'),
59 60 ('sea_notch', '<u4'),
60 61 ('lh_sea', '<u4'),
61 62 ('hh_sea', '<u4'),
62 63 ('nbins_sea', '<u4'),
63 64 ('min_snr', '<f4'),
64 65 ('min_cc', '<f4'),
65 66 ('max_time_diff', '<f4')
66 67 ])
67 68
68 69 DATA_STRUCTURE = numpy.dtype([
69 70 ('range', '<u4'),
70 71 ('status', '<u4'),
71 72 ('zonal', '<f4'),
72 73 ('meridional', '<f4'),
73 74 ('vertical', '<f4'),
74 75 ('zonal_a', '<f4'),
75 76 ('meridional_a', '<f4'),
76 77 ('corrected_fading', '<f4'), # seconds
77 78 ('uncorrected_fading', '<f4'), # seconds
78 79 ('time_diff', '<f4'),
79 80 ('major_axis', '<f4'),
80 81 ('axial_ratio', '<f4'),
81 82 ('orientation', '<f4'),
82 83 ('sea_power', '<u4'),
83 84 ('sea_algorithm', '<u4')
84 85 ])
85 86
86 87 @MPDecorator
87 88 class BLTRParamReader(JRODataReader, ProcessingUnit):
88 89 '''
89 90 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
90 91 from *.sswma files
91 92 '''
92 93
93 94 ext = '.sswma'
94 95
95 96 def __init__(self):
96 97
97 98 ProcessingUnit.__init__(self)
98 99
99 100 self.dataOut = Parameters()
100 101 self.counter_records = 0
101 102 self.flagNoMoreFiles = 0
102 103 self.isConfig = False
103 104 self.filename = None
104 105
105 106 def setup(self,
106 107 path=None,
107 108 startDate=None,
108 109 endDate=None,
109 110 ext=None,
110 111 startTime=datetime.time(0, 0, 0),
111 112 endTime=datetime.time(23, 59, 59),
112 113 timezone=0,
113 114 status_value=0,
114 115 **kwargs):
115 116 self.path = path
116 117 self.startDate = startDate
117 118 self.endDate = endDate
118 119 self.startTime = startTime
119 120 self.endTime = endTime
120 121 self.status_value = status_value
121 122 self.datatime = datetime.datetime(1900,1,1)
122 123 self.delay = kwargs.get('delay', 10)
123 124 self.online = kwargs.get('online', False)
124 125 self.nTries = kwargs.get('nTries', 3)
125 126
126 127 if self.path is None:
127 128 raise ValueError("The path is not valid")
128 129
129 130 if ext is None:
130 131 ext = self.ext
131 132
132 133 self.fileList = self.search_files(self.path, startDate, endDate, ext)
133 134 self.timezone = timezone
134 135 self.fileIndex = 0
135 136
136 137 if not self.fileList:
137 138 raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
138 139 path))
139 140
140 141 self.setNextFile()
141 142
142 143 def search_last_file(self):
143 144 '''
144 145 Get last file and add it to the list
145 146 '''
146 147
147 148 for n in range(self.nTries+1):
148 149 if n>0:
149 150 log.warning(
150 151 "Waiting %0.2f seconds for the next file, try %03d ..." % (self.delay, n+1),
151 152 self.name
152 153 )
153 154 time.sleep(self.delay)
154 155 file_list = os.listdir(self.path)
155 156 file_list.sort()
156 157 if file_list:
157 158 if self.filename:
158 159 if file_list[-1] not in self.filename:
159 160 return file_list[-1]
160 161 else:
161 162 continue
162 163 return file_list[-1]
163 164 return 0
164 165
165 166 def search_files(self, path, startDate, endDate, ext):
166 167 '''
167 168 Searching for BLTR rawdata file in path
168 169 Creating a list of file to proces included in [startDate,endDate]
169 170
170 171 Input:
171 172 path - Path to find BLTR rawdata files
172 173 startDate - Select file from this date
173 174 enDate - Select file until this date
174 175 ext - Extension of the file to read
175 176 '''
176 177
177 178 log.success('Searching files in {} '.format(path), 'BLTRParamReader')
178 179 foldercounter = 0
179 180 fileList0 = glob.glob1(path, "*%s" % ext)
180 181 fileList0.sort()
181 182
182 183 for thisFile in fileList0:
183 184 year = thisFile[-14:-10]
184 185 if not isNumber(year):
185 186 continue
186 187
187 188 month = thisFile[-10:-8]
188 189 if not isNumber(month):
189 190 continue
190 191
191 192 day = thisFile[-8:-6]
192 193 if not isNumber(day):
193 194 continue
194 195
195 196 year, month, day = int(year), int(month), int(day)
196 197 dateFile = datetime.date(year, month, day)
197 198
198 199 if (startDate > dateFile) or (endDate < dateFile):
199 200 continue
200 201
201 202 yield thisFile
202 203
203 204 return
204 205
205 206 def setNextFile(self):
206 207
207 208 if self.online:
208 209 filename = self.search_last_file()
209 210 if not filename:
210 211 self.flagNoMoreFiles = 1
211 212 return 0
212 213 else:
213 214 try:
214 215 filename = next(self.fileList)
215 216 except StopIteration:
216 217 self.flagNoMoreFiles = 1
217 218 return 0
218 219
219 220 log.success('Opening {}'.format(filename), 'BLTRParamReader')
220 221
221 222 dirname, name = os.path.split(filename)
222 223 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
223 224 self.siteFile = filename.split('.')[0]
224 225 if self.filename is not None:
225 226 self.fp.close()
226 227 self.filename = os.path.join(self.path, filename)
227 228 self.fp = open(self.filename, 'rb')
228 229 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
229 230 self.nrecords = self.header_file['nrec'][0]
230 231 self.sizeOfFile = os.path.getsize(self.filename)
231 232 self.counter_records = 0
232 233 self.flagIsNewFile = 0
233 234 self.fileIndex += 1
234 235
235 236 return 1
236 237
237 238 def readNextBlock(self):
238 239
239 240 while True:
240 241 if not self.online and self.counter_records == self.nrecords:
241 242 self.flagIsNewFile = 1
242 243 if not self.setNextFile():
243 244 return 0
244 245
245 246 try:
246 247 pointer = self.fp.tell()
247 248 self.readBlock()
248 249 except:
249 250 if self.online and self.waitDataBlock(pointer, 38512) == 1:
250 251 continue
251 252 else:
252 253 if not self.setNextFile():
253 254 return 0
254 255
255 256 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
256 257 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
257 258 log.warning(
258 259 'Reading Record No. {}/{} -> {} [Skipping]'.format(
259 260 self.counter_records,
260 261 self.nrecords,
261 262 self.datatime.ctime()),
262 263 'BLTRParamReader')
263 264 continue
264 265 break
265 266
266 267 log.log('Reading Record No. {} -> {}'.format(
267 268 self.counter_records,
268 269 # self.nrecords,
269 270 self.datatime.ctime()), 'BLTRParamReader')
270 271
271 272 return 1
272 273
273 274 def readBlock(self):
274 275
275 276 pointer = self.fp.tell()
276 277 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
277 278 self.nchannels = int(header_rec['nchan'][0] / 2)
278 279 self.kchan = header_rec['nrxs'][0]
279 280 self.nmodes = header_rec['nmodes'][0]
280 281 self.nranges = header_rec['nranges'][0]
281 282 self.fp.seek(pointer)
282 283 self.height = numpy.empty((self.nmodes, self.nranges))
283 284 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
284 285 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
285 286 self.flagDiscontinuousBlock = 0
286 287
287 288 for mode in range(self.nmodes):
288 289 self.readHeader()
289 290 data = self.readData()
290 291 self.height[mode] = (data[0] - self.correction) / 1000.
291 292 self.buffer[mode] = data[1]
292 293 self.snr[mode] = data[2]
293 294
294 295 self.counter_records = self.counter_records + self.nmodes
295 296
296 297 return
297 298
298 299 def readHeader(self):
299 300 '''
300 301 RecordHeader of BLTR rawdata file
301 302 '''
302 303
303 304 header_structure = numpy.dtype(
304 305 REC_HEADER_STRUCTURE.descr + [
305 306 ('antenna_coord', 'f4', (2, int(self.nchannels))),
306 307 ('rx_gains', 'u4', (int(self.nchannels),)),
307 308 ('rx_analysis', 'u4', (int(self.nchannels),))
308 309 ]
309 310 )
310 311
311 312 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
312 313 self.lat = self.header_rec['lat'][0]
313 314 self.lon = self.header_rec['lon'][0]
314 315 self.delta = self.header_rec['delta_r'][0]
315 316 self.correction = self.header_rec['dmode_rngcorr'][0]
316 317 self.imode = self.header_rec['dmode_index'][0]
317 318 self.antenna = self.header_rec['antenna_coord']
318 319 self.rx_gains = self.header_rec['rx_gains']
319 320 self.time = self.header_rec['time'][0]
320 321 dt = datetime.datetime.utcfromtimestamp(self.time)
321 322 if dt.date()>self.datatime.date():
322 323 self.flagDiscontinuousBlock = 1
323 324 self.datatime = dt
324 325
325 326 def readData(self):
326 327 '''
327 328 Reading and filtering data block record of BLTR rawdata file,
328 329 filtering is according to status_value.
329 330
330 331 Input:
331 332 status_value - Array data is set to NAN for values that are not
332 333 equal to status_value
333 334
334 335 '''
335 336 self.nchannels = int(self.nchannels)
336 337
337 338 data_structure = numpy.dtype(
338 339 DATA_STRUCTURE.descr + [
339 340 ('rx_saturation', 'u4', (self.nchannels,)),
340 341 ('chan_offset', 'u4', (2 * self.nchannels,)),
341 342 ('rx_amp', 'u4', (self.nchannels,)),
342 343 ('rx_snr', 'f4', (self.nchannels,)),
343 344 ('cross_snr', 'f4', (self.kchan,)),
344 345 ('sea_power_relative', 'f4', (self.kchan,))]
345 346 )
346 347
347 348 data = numpy.fromfile(self.fp, data_structure, self.nranges)
348 349
349 350 height = data['range']
350 351 winds = numpy.array(
351 352 (data['zonal'], data['meridional'], data['vertical']))
352 353 snr = data['rx_snr'].T
353 354
354 355 winds[numpy.where(winds == -9999.)] = numpy.nan
355 356 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
356 357 snr[numpy.where(snr == -9999.)] = numpy.nan
357 358 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
358 359 snr = numpy.power(10, snr / 10)
359 360
360 361 return height, winds, snr
361 362
362 363 def set_output(self):
363 364 '''
364 365 Storing data from databuffer to dataOut object
365 366 '''
366 367
367 368 self.dataOut.data_SNR = self.snr
368 369 self.dataOut.height = self.height
369 370 self.dataOut.data = self.buffer
370 371 self.dataOut.utctimeInit = self.time
371 372 self.dataOut.utctime = self.dataOut.utctimeInit
372 373 self.dataOut.useLocalTime = False
373 374 self.dataOut.paramInterval = 157
374 375 self.dataOut.timezone = self.timezone
375 376 self.dataOut.site = self.siteFile
376 377 self.dataOut.nrecords = self.nrecords / self.nmodes
377 378 self.dataOut.sizeOfFile = self.sizeOfFile
378 379 self.dataOut.lat = self.lat
379 380 self.dataOut.lon = self.lon
380 381 self.dataOut.channelList = list(range(self.nchannels))
381 382 self.dataOut.kchan = self.kchan
382 383 self.dataOut.delta = self.delta
383 384 self.dataOut.correction = self.correction
384 385 self.dataOut.nmodes = self.nmodes
385 386 self.dataOut.imode = self.imode
386 387 self.dataOut.antenna = self.antenna
387 388 self.dataOut.rx_gains = self.rx_gains
388 389 self.dataOut.flagNoData = False
389 390 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
390 391
391 392 def getData(self):
392 393 '''
393 394 Storing data from databuffer to dataOut object
394 395 '''
395 396 if self.flagNoMoreFiles:
396 397 self.dataOut.flagNoData = True
397 self.dataOut.error = 'No More files to read'
398 return
398 raise schainpy.admin.SchainError('No More files to read')
399 399
400 400 if not self.readNextBlock():
401 401 self.dataOut.flagNoData = True
402 self.dataOut.error = 'Time for wait new file reach!!!'
402 raise schainpy.admin.SchainError('Time for wait new file reach!!!')
403 403
404 404 self.set_output()
405 405
406 406 return 1
407 407 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,466 +1,462
1 1 import os
2 2 import sys
3 3 import glob
4 4 import fnmatch
5 5 import datetime
6 6 import time
7 7 import re
8 8 import h5py
9 9 import numpy
10 10
11 11 import pylab as plb
12 12 from scipy.optimize import curve_fit
13 13 from scipy import asarray as ar, exp
14 14
15 15 SPEED_OF_LIGHT = 299792458
16 16 SPEED_OF_LIGHT = 3e8
17 17
18 try:
19 from gevent import sleep
20 except:
21 from time import sleep
22
23 18 from .utils import folder_in_range
24 19
20 import schainpy.admin
25 21 from schainpy.model.data.jrodata import Spectra
26 22 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 23 from schainpy.utils import log
28 24 from schainpy.model.io.jroIO_base import JRODataReader
29 25
30 26 def pol2cart(rho, phi):
31 27 x = rho * numpy.cos(phi)
32 28 y = rho * numpy.sin(phi)
33 29 return(x, y)
34 30
35 31 FILE_STRUCTURE = numpy.dtype([ # HEADER 48bytes
36 32 ('FileMgcNumber', '<u4'), # 0x23020100
37 33 ('nFDTdataRecors', '<u4'),
38 34 ('OffsetStartHeader', '<u4'),
39 35 ('RadarUnitId', '<u4'),
40 36 ('SiteName', 'S32'), # Null terminated
41 37 ])
42 38
43 39
44 40 class FileHeaderBLTR():
45 41
46 42 def __init__(self, fo):
47 43
48 44 self.fo = fo
49 45 self.size = 48
50 46 self.read()
51 47
52 48 def read(self):
53 49
54 50 header = numpy.fromfile(self.fo, FILE_STRUCTURE, 1)
55 51 self.FileMgcNumber = hex(header['FileMgcNumber'][0])
56 52 self.nFDTdataRecors = int(header['nFDTdataRecors'][0])
57 53 self.RadarUnitId = int(header['RadarUnitId'][0])
58 54 self.OffsetStartHeader = int(header['OffsetStartHeader'][0])
59 55 self.SiteName = header['SiteName'][0]
60 56
61 57 def write(self, fp):
62 58
63 59 headerTuple = (self.FileMgcNumber,
64 60 self.nFDTdataRecors,
65 61 self.RadarUnitId,
66 62 self.SiteName,
67 63 self.size)
68 64
69 65 header = numpy.array(headerTuple, FILE_STRUCTURE)
70 66 header.tofile(fp)
71 67 ''' ndarray.tofile(fid, sep, format) Write array to a file as text or binary (default).
72 68
73 69 fid : file or str
74 70 An open file object, or a string containing a filename.
75 71
76 72 sep : str
77 73 Separator between array items for text output. If "" (empty), a binary file is written,
78 74 equivalent to file.write(a.tobytes()).
79 75
80 76 format : str
81 77 Format string for text file output. Each entry in the array is formatted to text by
82 78 first converting it to the closest Python type, and then using "format" % item.
83 79
84 80 '''
85 81
86 82 return 1
87 83
88 84
89 85 RECORD_STRUCTURE = numpy.dtype([ # RECORD HEADER 180+20N bytes
90 86 ('RecMgcNumber', '<u4'), # 0x23030001
91 87 ('RecCounter', '<u4'), # Record counter(0,1, ...)
92 88 # Offset to start of next record form start of this record
93 89 ('Off2StartNxtRec', '<u4'),
94 90 # Offset to start of data from start of this record
95 91 ('Off2StartData', '<u4'),
96 92 # Epoch time stamp of start of acquisition (seconds)
97 93 ('nUtime', '<i4'),
98 94 # Millisecond component of time stamp (0,...,999)
99 95 ('nMilisec', '<u4'),
100 96 # Experiment tag name (null terminated)
101 97 ('ExpTagName', 'S32'),
102 98 # Experiment comment (null terminated)
103 99 ('ExpComment', 'S32'),
104 100 # Site latitude (from GPS) in degrees (positive implies North)
105 101 ('SiteLatDegrees', '<f4'),
106 102 # Site longitude (from GPS) in degrees (positive implies East)
107 103 ('SiteLongDegrees', '<f4'),
108 104 # RTC GPS engine status (0=SEEK, 1=LOCK, 2=NOT FITTED, 3=UNAVAILABLE)
109 105 ('RTCgpsStatus', '<u4'),
110 106 ('TransmitFrec', '<u4'), # Transmit frequency (Hz)
111 107 ('ReceiveFrec', '<u4'), # Receive frequency
112 108 # First local oscillator frequency (Hz)
113 109 ('FirstOsciFrec', '<u4'),
114 110 # (0="O", 1="E", 2="linear 1", 3="linear2")
115 111 ('Polarisation', '<u4'),
116 112 # Receiver filter settings (0,1,2,3)
117 113 ('ReceiverFiltSett', '<u4'),
118 114 # Number of modes in use (1 or 2)
119 115 ('nModesInUse', '<u4'),
120 116 # Dual Mode index number for these data (0 or 1)
121 117 ('DualModeIndex', '<u4'),
122 118 # Dual Mode range correction for these data (m)
123 119 ('DualModeRange', '<u4'),
124 120 # Number of digital channels acquired (2*N)
125 121 ('nDigChannels', '<u4'),
126 122 # Sampling resolution (meters)
127 123 ('SampResolution', '<u4'),
128 124 # Number of range gates sampled
129 125 ('nHeights', '<u4'),
130 126 # Start range of sampling (meters)
131 127 ('StartRangeSamp', '<u4'),
132 128 ('PRFhz', '<u4'), # PRF (Hz)
133 129 ('nCohInt', '<u4'), # Integrations
134 130 # Number of data points transformed
135 131 ('nProfiles', '<u4'),
136 132 # Number of receive beams stored in file (1 or N)
137 133 ('nChannels', '<u4'),
138 134 ('nIncohInt', '<u4'), # Number of spectral averages
139 135 # FFT windowing index (0 = no window)
140 136 ('FFTwindowingInd', '<u4'),
141 137 # Beam steer angle (azimuth) in degrees (clockwise from true North)
142 138 ('BeamAngleAzim', '<f4'),
143 139 # Beam steer angle (zenith) in degrees (0=> vertical)
144 140 ('BeamAngleZen', '<f4'),
145 141 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
146 142 ('AntennaCoord0', '<f4'),
147 143 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
148 144 ('AntennaAngl0', '<f4'),
149 145 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
150 146 ('AntennaCoord1', '<f4'),
151 147 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
152 148 ('AntennaAngl1', '<f4'),
153 149 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
154 150 ('AntennaCoord2', '<f4'),
155 151 # Antenna coordinates (Range(meters), Bearing(degrees)) - N pairs
156 152 ('AntennaAngl2', '<f4'),
157 153 # Receiver phase calibration (degrees) - N values
158 154 ('RecPhaseCalibr0', '<f4'),
159 155 # Receiver phase calibration (degrees) - N values
160 156 ('RecPhaseCalibr1', '<f4'),
161 157 # Receiver phase calibration (degrees) - N values
162 158 ('RecPhaseCalibr2', '<f4'),
163 159 # Receiver amplitude calibration (ratio relative to receiver one) - N values
164 160 ('RecAmpCalibr0', '<f4'),
165 161 # Receiver amplitude calibration (ratio relative to receiver one) - N values
166 162 ('RecAmpCalibr1', '<f4'),
167 163 # Receiver amplitude calibration (ratio relative to receiver one) - N values
168 164 ('RecAmpCalibr2', '<f4'),
169 165 # Receiver gains in dB - N values
170 166 ('ReceiverGaindB0', '<i4'),
171 167 # Receiver gains in dB - N values
172 168 ('ReceiverGaindB1', '<i4'),
173 169 # Receiver gains in dB - N values
174 170 ('ReceiverGaindB2', '<i4'),
175 171 ])
176 172
177 173
178 174 class RecordHeaderBLTR():
179 175
180 176 def __init__(self, fo):
181 177
182 178 self.fo = fo
183 179 self.OffsetStartHeader = 48
184 180 self.Off2StartNxtRec = 811248
185 181
186 182 def read(self, block):
187 183 OffRHeader = self.OffsetStartHeader + block * self.Off2StartNxtRec
188 184 self.fo.seek(OffRHeader, os.SEEK_SET)
189 185 header = numpy.fromfile(self.fo, RECORD_STRUCTURE, 1)
190 186 self.RecMgcNumber = hex(header['RecMgcNumber'][0]) # 0x23030001
191 187 self.RecCounter = int(header['RecCounter'][0])
192 188 self.Off2StartNxtRec = int(header['Off2StartNxtRec'][0])
193 189 self.Off2StartData = int(header['Off2StartData'][0])
194 190 self.nUtime = header['nUtime'][0]
195 191 self.nMilisec = header['nMilisec'][0]
196 192 self.ExpTagName = '' # str(header['ExpTagName'][0])
197 193 self.ExpComment = '' # str(header['ExpComment'][0])
198 194 self.SiteLatDegrees = header['SiteLatDegrees'][0]
199 195 self.SiteLongDegrees = header['SiteLongDegrees'][0]
200 196 self.RTCgpsStatus = header['RTCgpsStatus'][0]
201 197 self.TransmitFrec = header['TransmitFrec'][0]
202 198 self.ReceiveFrec = header['ReceiveFrec'][0]
203 199 self.FirstOsciFrec = header['FirstOsciFrec'][0]
204 200 self.Polarisation = header['Polarisation'][0]
205 201 self.ReceiverFiltSett = header['ReceiverFiltSett'][0]
206 202 self.nModesInUse = header['nModesInUse'][0]
207 203 self.DualModeIndex = header['DualModeIndex'][0]
208 204 self.DualModeRange = header['DualModeRange'][0]
209 205 self.nDigChannels = header['nDigChannels'][0]
210 206 self.SampResolution = header['SampResolution'][0]
211 207 self.nHeights = header['nHeights'][0]
212 208 self.StartRangeSamp = header['StartRangeSamp'][0]
213 209 self.PRFhz = header['PRFhz'][0]
214 210 self.nCohInt = header['nCohInt'][0]
215 211 self.nProfiles = header['nProfiles'][0]
216 212 self.nChannels = header['nChannels'][0]
217 213 self.nIncohInt = header['nIncohInt'][0]
218 214 self.FFTwindowingInd = header['FFTwindowingInd'][0]
219 215 self.BeamAngleAzim = header['BeamAngleAzim'][0]
220 216 self.BeamAngleZen = header['BeamAngleZen'][0]
221 217 self.AntennaCoord0 = header['AntennaCoord0'][0]
222 218 self.AntennaAngl0 = header['AntennaAngl0'][0]
223 219 self.AntennaCoord1 = header['AntennaCoord1'][0]
224 220 self.AntennaAngl1 = header['AntennaAngl1'][0]
225 221 self.AntennaCoord2 = header['AntennaCoord2'][0]
226 222 self.AntennaAngl2 = header['AntennaAngl2'][0]
227 223 self.RecPhaseCalibr0 = header['RecPhaseCalibr0'][0]
228 224 self.RecPhaseCalibr1 = header['RecPhaseCalibr1'][0]
229 225 self.RecPhaseCalibr2 = header['RecPhaseCalibr2'][0]
230 226 self.RecAmpCalibr0 = header['RecAmpCalibr0'][0]
231 227 self.RecAmpCalibr1 = header['RecAmpCalibr1'][0]
232 228 self.RecAmpCalibr2 = header['RecAmpCalibr2'][0]
233 229 self.ReceiverGaindB0 = header['ReceiverGaindB0'][0]
234 230 self.ReceiverGaindB1 = header['ReceiverGaindB1'][0]
235 231 self.ReceiverGaindB2 = header['ReceiverGaindB2'][0]
236 232 self.ipp = 0.5 * (SPEED_OF_LIGHT / self.PRFhz)
237 233 self.RHsize = 180 + 20 * self.nChannels
238 234 self.Datasize = self.nProfiles * self.nChannels * self.nHeights * 2 * 4
239 235 endFp = self.OffsetStartHeader + self.RecCounter * self.Off2StartNxtRec
240 236
241 237
242 238 if OffRHeader > endFp:
243 239 sys.stderr.write(
244 240 "Warning %s: Size value read from System Header is lower than it has to be\n" % fp)
245 241 return 0
246 242
247 243 if OffRHeader < endFp:
248 244 sys.stderr.write(
249 245 "Warning %s: Size value read from System Header size is greater than it has to be\n" % fp)
250 246 return 0
251 247
252 248 return 1
253 249
254 250 @MPDecorator
255 251 class BLTRSpectraReader (ProcessingUnit):
256 252
257 253 def __init__(self):
258 254
259 255 ProcessingUnit.__init__(self)
260 256
261 257 self.ext = ".fdt"
262 258 self.optchar = "P"
263 259 self.fpFile = None
264 260 self.fp = None
265 261 self.BlockCounter = 0
266 262 self.fileSizeByHeader = None
267 263 self.filenameList = []
268 264 self.fileSelector = 0
269 265 self.Off2StartNxtRec = 0
270 266 self.RecCounter = 0
271 267 self.flagNoMoreFiles = 0
272 268 self.data_spc = None
273 269 self.data_cspc = None
274 270 self.path = None
275 271 self.OffsetStartHeader = 0
276 272 self.Off2StartData = 0
277 273 self.ipp = 0
278 274 self.nFDTdataRecors = 0
279 275 self.blocksize = 0
280 276 self.dataOut = Spectra()
281 277 self.dataOut.flagNoData = False
282 278
283 279 def search_files(self):
284 280 '''
285 281 Function that indicates the number of .fdt files that exist in the folder to be read.
286 282 It also creates an organized list with the names of the files to read.
287 283 '''
288 284
289 285 files = glob.glob(os.path.join(self.path, '*{}'.format(self.ext)))
290 286 files = sorted(files)
291 287 for f in files:
292 288 filename = f.split('/')[-1]
293 289 if folder_in_range(filename.split('.')[1], self.startDate, self.endDate, '%Y%m%d'):
294 290 self.filenameList.append(f)
295 291
296 292 def run(self, **kwargs):
297 293 '''
298 294 This method will be the one that will initiate the data entry, will be called constantly.
299 295 You should first verify that your Setup () is set up and then continue to acquire
300 296 the data to be processed with getData ().
301 297 '''
302 298 if not self.isConfig:
303 299 self.setup(**kwargs)
304 300 self.isConfig = True
305 301
306 302 self.getData()
307 303
308 304 def setup(self,
309 305 path=None,
310 306 startDate=None,
311 307 endDate=None,
312 308 startTime=None,
313 309 endTime=None,
314 310 walk=True,
315 311 code=None,
316 312 online=False,
317 313 mode=None,
318 314 **kwargs):
319 315
320 316 self.isConfig = True
321 317
322 318 self.path = path
323 319 self.startDate = startDate
324 320 self.endDate = endDate
325 321 self.startTime = startTime
326 322 self.endTime = endTime
327 323 self.walk = walk
328 324 self.mode = int(mode)
329 325 self.search_files()
330 326 if self.filenameList:
331 327 self.readFile()
332 328
333 329 def getData(self):
334 330 '''
335 331 Before starting this function, you should check that there is still an unread file,
336 332 If there are still blocks to read or if the data block is empty.
337 333
338 334 You should call the file "read".
339 335
340 336 '''
341 337
342 338 if self.flagNoMoreFiles:
343 339 self.dataOut.flagNoData = True
344 self.dataOut.error = 'No more files'
340 raise schainpy.admin.SchainError('No more files')
345 341
346 342 self.readBlock()
347 343
348 344 def readFile(self):
349 345 '''
350 346 You must indicate if you are reading in Online or Offline mode and load the
351 347 The parameters for this file reading mode.
352 348
353 349 Then you must do 2 actions:
354 350
355 351 1. Get the BLTR FileHeader.
356 352 2. Start reading the first block.
357 353 '''
358 354
359 355 if self.fileSelector < len(self.filenameList):
360 356 log.success('Opening file: {}'.format(self.filenameList[self.fileSelector]), self.name)
361 357 self.fp = open(self.filenameList[self.fileSelector])
362 358 self.fheader = FileHeaderBLTR(self.fp)
363 359 self.rheader = RecordHeaderBLTR(self.fp)
364 360 self.nFDTdataRecors = self.fheader.nFDTdataRecors
365 361 self.fileSelector += 1
366 362 self.BlockCounter = 0
367 363 return 1
368 364 else:
369 365 self.flagNoMoreFiles = True
370 366 self.dataOut.flagNoData = True
371 367 return 0
372 368
373 369 def readBlock(self):
374 370 '''
375 371 It should be checked if the block has data, if it is not passed to the next file.
376 372
377 373 Then the following is done:
378 374
379 375 1. Read the RecordHeader
380 376 2. Fill the buffer with the current block number.
381 377
382 378 '''
383 379
384 380 if self.BlockCounter == self.nFDTdataRecors:
385 381 if not self.readFile():
386 382 return
387 383
388 384 if self.mode == 1:
389 385 self.rheader.read(self.BlockCounter+1)
390 386 elif self.mode == 0:
391 387 self.rheader.read(self.BlockCounter)
392 388
393 389 self.RecCounter = self.rheader.RecCounter
394 390 self.OffsetStartHeader = self.rheader.OffsetStartHeader
395 391 self.Off2StartNxtRec = self.rheader.Off2StartNxtRec
396 392 self.Off2StartData = self.rheader.Off2StartData
397 393 self.nProfiles = self.rheader.nProfiles
398 394 self.nChannels = self.rheader.nChannels
399 395 self.nHeights = self.rheader.nHeights
400 396 self.frequency = self.rheader.TransmitFrec
401 397 self.DualModeIndex = self.rheader.DualModeIndex
402 398 self.pairsList = [(0, 1), (0, 2), (1, 2)]
403 399 self.dataOut.pairsList = self.pairsList
404 400 self.nRdPairs = len(self.dataOut.pairsList)
405 401 self.dataOut.nRdPairs = self.nRdPairs
406 402 self.dataOut.heightList = (self.rheader.StartRangeSamp + numpy.arange(self.nHeights) * self.rheader.SampResolution) / 1000.
407 403 self.dataOut.channelList = range(self.nChannels)
408 404 self.dataOut.nProfiles=self.rheader.nProfiles
409 405 self.dataOut.nIncohInt=self.rheader.nIncohInt
410 406 self.dataOut.nCohInt=self.rheader.nCohInt
411 407 self.dataOut.ippSeconds= 1/float(self.rheader.PRFhz)
412 408 self.dataOut.PRF=self.rheader.PRFhz
413 409 self.dataOut.nFFTPoints=self.rheader.nProfiles
414 410 self.dataOut.utctime = self.rheader.nUtime + self.rheader.nMilisec/1000.
415 411 self.dataOut.timeZone = 0
416 412 self.dataOut.useLocalTime = False
417 413 self.dataOut.nmodes = 2
418 414 log.log('Reading block {} - {}'.format(self.BlockCounter, self.dataOut.datatime), self.name)
419 415 OffDATA = self.OffsetStartHeader + self.RecCounter * \
420 416 self.Off2StartNxtRec + self.Off2StartData
421 417 self.fp.seek(OffDATA, os.SEEK_SET)
422 418
423 419 self.data_fft = numpy.fromfile(self.fp, [('complex','<c8')], self.nProfiles*self.nChannels*self.nHeights )
424 420 self.data_fft = self.data_fft.astype(numpy.dtype('complex'))
425 421 self.data_block = numpy.reshape(self.data_fft,(self.nHeights, self.nChannels, self.nProfiles))
426 422 self.data_block = numpy.transpose(self.data_block, (1,2,0))
427 423 copy = self.data_block.copy()
428 424 spc = copy * numpy.conjugate(copy)
429 425 self.data_spc = numpy.absolute(spc) # valor absoluto o magnitud
430 426 self.dataOut.data_spc = self.data_spc
431 427
432 428 cspc = self.data_block.copy()
433 429 self.data_cspc = self.data_block.copy()
434 430
435 431 for i in range(self.nRdPairs):
436 432
437 433 chan_index0 = self.dataOut.pairsList[i][0]
438 434 chan_index1 = self.dataOut.pairsList[i][1]
439 435
440 436 self.data_cspc[i, :, :] = cspc[chan_index0, :, :] * numpy.conjugate(cspc[chan_index1, :, :])
441 437
442 438 '''Getting Eij and Nij'''
443 439 (AntennaX0, AntennaY0) = pol2cart(
444 440 self.rheader.AntennaCoord0, self.rheader.AntennaAngl0 * numpy.pi / 180)
445 441 (AntennaX1, AntennaY1) = pol2cart(
446 442 self.rheader.AntennaCoord1, self.rheader.AntennaAngl1 * numpy.pi / 180)
447 443 (AntennaX2, AntennaY2) = pol2cart(
448 444 self.rheader.AntennaCoord2, self.rheader.AntennaAngl2 * numpy.pi / 180)
449 445
450 446 E01 = AntennaX0 - AntennaX1
451 447 N01 = AntennaY0 - AntennaY1
452 448
453 449 E02 = AntennaX0 - AntennaX2
454 450 N02 = AntennaY0 - AntennaY2
455 451
456 452 E12 = AntennaX1 - AntennaX2
457 453 N12 = AntennaY1 - AntennaY2
458 454
459 455 self.ChanDist = numpy.array(
460 456 [[E01, N01], [E02, N02], [E12, N12]])
461 457
462 458 self.dataOut.ChanDist = self.ChanDist
463 459
464 460 self.BlockCounter += 2
465 461 self.dataOut.data_spc = self.data_spc
466 462 self.dataOut.data_cspc =self.data_cspc
Show file before | Show file after | Hide comments
@@ -1,692 +1,691
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4
5 5 import os
6 6 import sys
7 7 import glob
8 8 import fnmatch
9 9 import datetime
10 10 import time
11 11 import re
12 12 import h5py
13 13 import numpy
14 14
15 15 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
16 16 from schainpy.model.data.jroamisr import AMISR
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 class RadacHeader():
24 24 def __init__(self, fp):
25 25 header = 'Raw11/Data/RadacHeader'
26 26 self.beamCodeByPulse = fp.get(header+'/BeamCode')
27 27 self.beamCode = fp.get('Raw11/Data/Beamcodes')
28 28 self.code = fp.get(header+'/Code')
29 29 self.frameCount = fp.get(header+'/FrameCount')
30 30 self.modeGroup = fp.get(header+'/ModeGroup')
31 31 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
32 32 self.pulseCount = fp.get(header+'/PulseCount')
33 33 self.radacTime = fp.get(header+'/RadacTime')
34 34 self.timeCount = fp.get(header+'/TimeCount')
35 35 self.timeStatus = fp.get(header+'/TimeStatus')
36 36
37 37 self.nrecords = self.pulseCount.shape[0] #nblocks
38 38 self.npulses = self.pulseCount.shape[1] #nprofile
39 39 self.nsamples = self.nsamplesPulse[0,0] #ngates
40 40 self.nbeams = self.beamCode.shape[1]
41 41
42 42
43 43 def getIndexRangeToPulse(self, idrecord=0):
44 44 #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
45 45 #startPulseCountId = indexToZero[0][0]
46 46 #endPulseCountId = startPulseCountId - 1
47 47 #range1 = numpy.arange(startPulseCountId,self.npulses,1)
48 48 #range2 = numpy.arange(0,startPulseCountId,1)
49 49 #return range1, range2
50 50 zero = 0
51 51 npulse = max(self.pulseCount[0,:]+1)-1
52 52 looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
53 53 getLastIndex = looking_index[-1]
54 54 index_data = numpy.arange(0,getLastIndex+1,1)
55 55 index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
56 56 return index_data, index_buffer
57 57
58 58 class AMISRReader(ProcessingUnit):
59 59
60 60 path = None
61 61 startDate = None
62 62 endDate = None
63 63 startTime = None
64 64 endTime = None
65 65 walk = None
66 66 isConfig = False
67 67
68 68 def __init__(self):
69 69 self.set = None
70 70 self.subset = None
71 71 self.extension_file = '.h5'
72 72 self.dtc_str = 'dtc'
73 73 self.dtc_id = 0
74 74 self.status = True
75 75 self.isConfig = False
76 76 self.dirnameList = []
77 77 self.filenameList = []
78 78 self.fileIndex = None
79 79 self.flagNoMoreFiles = False
80 80 self.flagIsNewFile = 0
81 81 self.filename = ''
82 82 self.amisrFilePointer = None
83 83 self.radacHeaderObj = None
84 84 self.dataOut = self.__createObjByDefault()
85 85 self.datablock = None
86 86 self.rest_datablock = None
87 87 self.range = None
88 88 self.idrecord_count = 0
89 89 self.profileIndex = 0
90 90 self.index_amisr_sample = None
91 91 self.index_amisr_buffer = None
92 92 self.beamCodeByFrame = None
93 93 self.radacTimeByFrame = None
94 94 #atributos originales tal y como esta en el archivo de datos
95 95 self.beamCodesFromFile = None
96 96 self.radacTimeFromFile = None
97 97 self.rangeFromFile = None
98 98 self.dataByFrame = None
99 99 self.dataset = None
100 100
101 101 self.beamCodeDict = {}
102 102 self.beamRangeDict = {}
103 103
104 104 #experiment cgf file
105 105 self.npulsesint_fromfile = None
106 106 self.recordsperfile_fromfile = None
107 107 self.nbeamcodes_fromfile = None
108 108 self.ngates_fromfile = None
109 109 self.ippSeconds_fromfile = None
110 110 self.frequency_h5file = None
111 111
112 112
113 113 self.__firstFile = True
114 114 self.buffer_radactime = None
115 115
116 116 self.index4_schain_datablock = None
117 117 self.index4_buffer = None
118 118 self.schain_datablock = None
119 119 self.buffer = None
120 120 self.linear_pulseCount = None
121 121 self.npulseByFrame = None
122 122 self.profileIndex_offset = None
123 123 self.timezone = 'ut'
124 124
125 125 self.__waitForNewFile = 20
126 126 self.__filename_online = None
127 127
128 128 def __createObjByDefault(self):
129 129
130 130 dataObj = AMISR()
131 131
132 132 return dataObj
133 133
134 134 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
135 135 self.path = path
136 136 self.startDate = startDate
137 137 self.endDate = endDate
138 138 self.startTime = startTime
139 139 self.endTime = endTime
140 140 self.walk = walk
141 141
142 142 def __checkPath(self):
143 143 if os.path.exists(self.path):
144 144 self.status = 1
145 145 else:
146 146 self.status = 0
147 147 print('Path:%s does not exists'%self.path)
148 148
149 149 return
150 150
151 151 def __selDates(self, amisr_dirname_format):
152 152 try:
153 153 year = int(amisr_dirname_format[0:4])
154 154 month = int(amisr_dirname_format[4:6])
155 155 dom = int(amisr_dirname_format[6:8])
156 156 thisDate = datetime.date(year,month,dom)
157 157
158 158 if (thisDate>=self.startDate and thisDate <= self.endDate):
159 159 return amisr_dirname_format
160 160 except:
161 161 return None
162 162
163 163 def __findDataForDates(self,online=False):
164 164
165 165
166 166
167 167 if not(self.status):
168 168 return None
169 169
170 170 pat = '\d+.\d+'
171 171 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
172 172 dirnameList = [x for x in dirnameList if x!=None]
173 173 dirnameList = [x.string for x in dirnameList]
174 174 if not(online):
175 175 dirnameList = [self.__selDates(x) for x in dirnameList]
176 176 dirnameList = [x for x in dirnameList if x!=None]
177 177 if len(dirnameList)>0:
178 178 self.status = 1
179 179 self.dirnameList = dirnameList
180 180 self.dirnameList.sort()
181 181 else:
182 182 self.status = 0
183 183 return None
184 184
185 185 def __getTimeFromData(self):
186 186 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
187 187 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
188 188
189 189 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
190 190 print('........................................')
191 191 filter_filenameList = []
192 192 self.filenameList.sort()
193 193 for i in range(len(self.filenameList)-1):
194 194 filename = self.filenameList[i]
195 195 fp = h5py.File(filename,'r')
196 196 time_str = fp.get('Time/RadacTimeString')
197 197
198 198 startDateTimeStr_File = time_str[0][0].split('.')[0]
199 199 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
200 200 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
201 201
202 202 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
203 203 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
204 204 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
205 205
206 206 fp.close()
207 207
208 208 if self.timezone == 'lt':
209 209 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
210 210 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
211 211
212 212 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
213 213 #self.filenameList.remove(filename)
214 214 filter_filenameList.append(filename)
215 215
216 216 filter_filenameList.sort()
217 217 self.filenameList = filter_filenameList
218 218 return 1
219 219
220 220 def __filterByGlob1(self, dirName):
221 221 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
222 222 filterDict = {}
223 223 filterDict.setdefault(dirName)
224 224 filterDict[dirName] = filter_files
225 225 return filterDict
226 226
227 227 def __getFilenameList(self, fileListInKeys, dirList):
228 228 for value in fileListInKeys:
229 229 dirName = list(value.keys())[0]
230 230 for file in value[dirName]:
231 231 filename = os.path.join(dirName, file)
232 232 self.filenameList.append(filename)
233 233
234 234
235 235 def __selectDataForTimes(self, online=False):
236 236 #aun no esta implementado el filtro for tiempo
237 237 if not(self.status):
238 238 return None
239 239
240 240 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
241 241
242 242 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
243 243
244 244 self.__getFilenameList(fileListInKeys, dirList)
245 245 if not(online):
246 246 #filtro por tiempo
247 247 if not(self.all):
248 248 self.__getTimeFromData()
249 249
250 250 if len(self.filenameList)>0:
251 251 self.status = 1
252 252 self.filenameList.sort()
253 253 else:
254 254 self.status = 0
255 255 return None
256 256
257 257 else:
258 258 #get the last file - 1
259 259 self.filenameList = [self.filenameList[-2]]
260 260
261 261 new_dirnameList = []
262 262 for dirname in self.dirnameList:
263 263 junk = numpy.array([dirname in x for x in self.filenameList])
264 264 junk_sum = junk.sum()
265 265 if junk_sum > 0:
266 266 new_dirnameList.append(dirname)
267 267 self.dirnameList = new_dirnameList
268 268 return 1
269 269
270 270 def searchFilesOnLine(self,
271 271 path,
272 272 walk=True):
273 273
274 274 startDate = datetime.datetime.utcnow().date()
275 275 endDate = datetime.datetime.utcnow().date()
276 276
277 277 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
278 278
279 279 self.__checkPath()
280 280
281 281 self.__findDataForDates(online=True)
282 282
283 283 self.dirnameList = [self.dirnameList[-1]]
284 284
285 285 self.__selectDataForTimes(online=True)
286 286
287 287 return
288 288
289 289
290 290 def searchFilesOffLine(self,
291 291 path,
292 292 startDate,
293 293 endDate,
294 294 startTime=datetime.time(0,0,0),
295 295 endTime=datetime.time(23,59,59),
296 296 walk=True):
297 297
298 298 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
299 299
300 300 self.__checkPath()
301 301
302 302 self.__findDataForDates()
303 303
304 304 self.__selectDataForTimes()
305 305
306 306 for i in range(len(self.filenameList)):
307 307 print("%s" %(self.filenameList[i]))
308 308
309 309 return
310 310
311 311 def __setNextFileOffline(self):
312 312 idFile = self.fileIndex
313 313
314 314 while (True):
315 315 idFile += 1
316 316 if not(idFile < len(self.filenameList)):
317 317 self.flagNoMoreFiles = 1
318 318 print("No more Files")
319 319 return 0
320 320
321 321 filename = self.filenameList[idFile]
322 322
323 323 amisrFilePointer = h5py.File(filename,'r')
324 324
325 325 break
326 326
327 327 self.flagIsNewFile = 1
328 328 self.fileIndex = idFile
329 329 self.filename = filename
330 330
331 331 self.amisrFilePointer = amisrFilePointer
332 332
333 333 print("Setting the file: %s"%self.filename)
334 334
335 335 return 1
336 336
337 337
338 338 def __setNextFileOnline(self):
339 339 filename = self.filenameList[0]
340 340 if self.__filename_online != None:
341 341 self.__selectDataForTimes(online=True)
342 342 filename = self.filenameList[0]
343 343 while self.__filename_online == filename:
344 344 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
345 345 sleep(self.__waitForNewFile)
346 346 self.__selectDataForTimes(online=True)
347 347 filename = self.filenameList[0]
348 348
349 349 self.__filename_online = filename
350 350
351 351 self.amisrFilePointer = h5py.File(filename,'r')
352 352 self.flagIsNewFile = 1
353 353 self.filename = filename
354 354 print("Setting the file: %s"%self.filename)
355 355 return 1
356 356
357 357
358 358 def __readHeader(self):
359 359 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
360 360
361 361 #update values from experiment cfg file
362 362 if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
363 363 self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
364 364 self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
365 365 self.radacHeaderObj.npulses = self.npulsesint_fromfile
366 366 self.radacHeaderObj.nsamples = self.ngates_fromfile
367 367
368 368 #looking index list for data
369 369 start_index = self.radacHeaderObj.pulseCount[0,:][0]
370 370 end_index = self.radacHeaderObj.npulses
371 371 range4data = list(range(start_index, end_index))
372 372 self.index4_schain_datablock = numpy.array(range4data)
373 373
374 374 buffer_start_index = 0
375 375 buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
376 376 range4buffer = list(range(buffer_start_index, buffer_end_index))
377 377 self.index4_buffer = numpy.array(range4buffer)
378 378
379 379 self.linear_pulseCount = numpy.array(range4data + range4buffer)
380 380 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
381 381
382 382 #get tuning frequency
383 383 frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
384 384 self.frequency_h5file = frequency_h5file_dataset[0,0]
385 385
386 386 self.flagIsNewFile = 1
387 387
388 388 def __getBeamCode(self):
389 389 self.beamCodeDict = {}
390 390 self.beamRangeDict = {}
391 391
392 392 beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
393 393
394 394 for i in range(len(self.radacHeaderObj.beamCode[0,:])):
395 395 self.beamCodeDict.setdefault(i)
396 396 self.beamRangeDict.setdefault(i)
397 397 beamcodeValue = self.radacHeaderObj.beamCode[0,i]
398 398 beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
399 399 x = beamCodeMap[beamcodeIndex][1]
400 400 y = beamCodeMap[beamcodeIndex][2]
401 401 z = beamCodeMap[beamcodeIndex][3]
402 402 self.beamCodeDict[i] = [beamcodeValue, x, y, z]
403 403
404 404 just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
405 405
406 406 for i in range(len(list(self.beamCodeDict.values()))):
407 407 xx = numpy.where(just4record0==list(self.beamCodeDict.values())[i][0])
408 408 indexPulseByBeam = self.linear_pulseCount[xx[0]]
409 409 self.beamRangeDict[i] = indexPulseByBeam
410 410
411 411 def __getExpParameters(self):
412 412 if not(self.status):
413 413 return None
414 414
415 415 experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
416 416
417 417 expFinder = glob.glob1(experimentCfgPath,'*.exp')
418 418 if len(expFinder)== 0:
419 419 self.status = 0
420 420 return None
421 421
422 422 experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
423 423
424 424 f = open(experimentFilename)
425 425 lines = f.readlines()
426 426 f.close()
427 427
428 428 parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
429 429 filterList = [fnmatch.filter(lines, x) for x in parmsList]
430 430
431 431
432 432 values = [re.sub(r'\D',"",x[0]) for x in filterList]
433 433
434 434 self.npulsesint_fromfile = int(values[0])
435 435 self.recordsperfile_fromfile = int(values[1])
436 436 self.nbeamcodes_fromfile = int(values[2])
437 437 self.ngates_fromfile = int(values[3])
438 438
439 439 tufileFinder = fnmatch.filter(lines, 'tufile=*')
440 440 tufile = tufileFinder[0].split('=')[1].split('\n')[0]
441 441 tufile = tufile.split('\r')[0]
442 442 tufilename = os.path.join(experimentCfgPath,tufile)
443 443
444 444 f = open(tufilename)
445 445 lines = f.readlines()
446 446 f.close()
447 447 self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
448 448
449 449
450 450 self.status = 1
451 451
452 452 def __setIdsAndArrays(self):
453 453 self.dataByFrame = self.__setDataByFrame()
454 454 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
455 455 self.readRanges()
456 456 self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
457 457 self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
458 458 if len(self.index_amisr_buffer) > 0:
459 459 self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
460 460
461 461
462 462 def __setNextFile(self,online=False):
463 463
464 464 if not(online):
465 465 newFile = self.__setNextFileOffline()
466 466 else:
467 467 newFile = self.__setNextFileOnline()
468 468
469 469 if not(newFile):
470 470 return 0
471 471
472 472 self.__readHeader()
473 473
474 474 if self.__firstFile:
475 475 self.__setIdsAndArrays()
476 476 self.__firstFile = False
477 477
478 478 self.__getBeamCode()
479 479 self.readDataBlock()
480 480
481 481
482 482 def setup(self,path=None,
483 483 startDate=None,
484 484 endDate=None,
485 485 startTime=datetime.time(0,0,0),
486 486 endTime=datetime.time(23,59,59),
487 487 walk=True,
488 488 timezone='ut',
489 489 all=0,
490 490 online=False):
491 491
492 492 self.timezone = timezone
493 493 self.all = all
494 494 self.online = online
495 495 if not(online):
496 496 #Busqueda de archivos offline
497 497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 498 else:
499 499 self.searchFilesOnLine(path, walk)
500 500
501 501 if not(self.filenameList):
502 502 print("There is no files into the folder: %s"%(path))
503 503
504 504 sys.exit(-1)
505 505
506 506 self.__getExpParameters()
507 507
508 508 self.fileIndex = -1
509 509
510 510 self.__setNextFile(online)
511 511
512 512 # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
513 513 # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
514 514 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
515 515 self.profileIndex = self.profileIndex_offset
516 516
517 517 def readRanges(self):
518 518 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
519 519
520 520 self.rangeFromFile = numpy.reshape(dataset.value,(-1))
521 521 return self.rangeFromFile
522 522
523 523
524 524 def readRadacTime(self,idrecord, range1, range2):
525 525 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
526 526
527 527 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
528 528 #radacTimeByFrame = dataset[idrecord - 1,range1]
529 529 #radacTimeByFrame = dataset[idrecord,range2]
530 530
531 531 return radacTimeByFrame
532 532
533 533 def readBeamCode(self, idrecord, range1, range2):
534 534 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
535 535 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
536 536 self.beamCodesFromFile = dataset.value
537 537
538 538 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
539 539 #beamcodeByFrame[range2] = dataset[idrecord, range2]
540 540 beamcodeByFrame[range1] = dataset[idrecord, range1]
541 541 beamcodeByFrame[range2] = dataset[idrecord, range2]
542 542
543 543 return beamcodeByFrame
544 544
545 545
546 546 def __setDataByFrame(self):
547 547 ndata = 2 # porque es complejo
548 548 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
549 549 return dataByFrame
550 550
551 551 def __readDataSet(self):
552 552 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
553 553 return dataset
554 554
555 555 def __setDataBlock(self,):
556 556 real = self.dataByFrame[:,:,0] #asumo que 0 es real
557 557 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
558 558 datablock = real + imag*1j #armo el complejo
559 559 return datablock
560 560
561 561 def readSamples_version1(self,idrecord):
562 562 #estas tres primeras lineas solo se deben ejecutar una vez
563 563 if self.flagIsNewFile:
564 564 #reading dataset
565 565 self.dataset = self.__readDataSet()
566 566 self.flagIsNewFile = 0
567 567
568 568 if idrecord == 0:
569 569 self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
570 570 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
571 571 datablock = self.__setDataBlock()
572 572 if len(self.index_amisr_buffer) > 0:
573 573 self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
574 574 self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
575 575
576 576 return datablock
577 577 if len(self.index_amisr_buffer) > 0:
578 578 self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
579 579 self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
580 580 self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
581 581 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
582 582 datablock = self.__setDataBlock()
583 583 if len(self.index_amisr_buffer) > 0:
584 584 self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
585 585 self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
586 586
587 587 return datablock
588 588
589 589
590 590 def readSamples(self,idrecord):
591 591 if self.flagIsNewFile:
592 592 self.dataByFrame = self.__setDataByFrame()
593 593 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
594 594
595 595 #reading ranges
596 596 self.readRanges()
597 597 #reading dataset
598 598 self.dataset = self.__readDataSet()
599 599
600 600 self.flagIsNewFile = 0
601 601 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
602 602 self.dataByFrame = self.dataset[idrecord, :, :, :]
603 603 datablock = self.__setDataBlock()
604 604 return datablock
605 605
606 606
607 607 def readDataBlock(self):
608 608
609 609 self.datablock = self.readSamples_version1(self.idrecord_count)
610 610 #self.datablock = self.readSamples(self.idrecord_count)
611 611 #print 'record:', self.idrecord_count
612 612
613 613 self.idrecord_count += 1
614 614 self.profileIndex = 0
615 615
616 616 if self.idrecord_count >= self.radacHeaderObj.nrecords:
617 617 self.idrecord_count = 0
618 618 self.flagIsNewFile = 1
619 619
620 620 def readNextBlock(self):
621 621
622 622 self.readDataBlock()
623 623
624 624 if self.flagIsNewFile:
625 625 self.__setNextFile(self.online)
626 626 pass
627 627
628 628 def __hasNotDataInBuffer(self):
629 629 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
630 630 if self.profileIndex >= self.radacHeaderObj.npulses:
631 631 return 1
632 632 return 0
633 633
634 634 def printUTC(self):
635 635 print(self.dataOut.utctime)
636 636 print('')
637 637
638 638 def setObjProperties(self):
639 639
640 640 self.dataOut.heightList = self.rangeFromFile/1000.0 #km
641 641 self.dataOut.nProfiles = self.radacHeaderObj.npulses
642 642 self.dataOut.nRecords = self.radacHeaderObj.nrecords
643 643 self.dataOut.nBeams = self.radacHeaderObj.nbeams
644 644 self.dataOut.ippSeconds = self.ippSeconds_fromfile
645 645 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
646 646 self.dataOut.frequency = self.frequency_h5file
647 647 self.dataOut.npulseByFrame = self.npulseByFrame
648 648 self.dataOut.nBaud = None
649 649 self.dataOut.nCode = None
650 650 self.dataOut.code = None
651 651
652 652 self.dataOut.beamCodeDict = self.beamCodeDict
653 653 self.dataOut.beamRangeDict = self.beamRangeDict
654 654
655 655 if self.timezone == 'lt':
656 656 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
657 657 else:
658 658 self.dataOut.timeZone = 0 #by default time is UTC
659 659
660 660 def getData(self):
661 661
662 662 if self.flagNoMoreFiles:
663 663 self.dataOut.flagNoData = True
664 print('Process finished')
665 664 return 0
666 665
667 666 if self.__hasNotDataInBuffer():
668 667 self.readNextBlock()
669 668
670 669
671 670 if self.datablock is None: # setear esta condicion cuando no hayan datos por leers
672 671 self.dataOut.flagNoData = True
673 672 return 0
674 673
675 674 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
676 675
677 676 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
678 677 self.dataOut.profileIndex = self.profileIndex
679 678 self.dataOut.flagNoData = False
680 679
681 680 self.profileIndex += 1
682 681
683 682 return self.dataOut.data
684 683
685 684
686 685 def run(self, **kwargs):
687 686 if not(self.isConfig):
688 687 self.setup(**kwargs)
689 688 self.setObjProperties()
690 689 self.isConfig = True
691 690
692 691 self.getData() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1831 +1,1831
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 try:
19 from gevent import sleep
20 except:
21 from time import sleep
22
23 18 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 19 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 20 from schainpy.utils import log
26 21 import schainpy.admin
27 22
28 23 LOCALTIME = True
29 24
30 25
31 26 def isNumber(cad):
32 27 """
33 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34 29
35 30 Excepciones:
36 31 Si un determinado string no puede ser convertido a numero
37 32 Input:
38 33 str, string al cual se le analiza para determinar si convertible a un numero o no
39 34
40 35 Return:
41 36 True : si el string es uno numerico
42 37 False : no es un string numerico
43 38 """
44 39 try:
45 40 float(cad)
46 41 return True
47 42 except:
48 43 return False
49 44
50 45
51 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 47 """
53 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54 49
55 50 Inputs:
56 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57 52
58 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 54 segundos contados desde 01/01/1970.
60 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 56 segundos contados desde 01/01/1970.
62 57
63 58 Return:
64 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 60 fecha especificado, de lo contrario retorna False.
66 61
67 62 Excepciones:
68 63 Si el archivo no existe o no puede ser abierto
69 64 Si la cabecera no puede ser leida.
70 65
71 66 """
72 67 basicHeaderObj = BasicHeader(LOCALTIME)
73 68
74 69 try:
75 70 fp = open(filename, 'rb')
76 71 except IOError:
77 72 print("The file %s can't be opened" % (filename))
78 73 return 0
79 74
80 75 sts = basicHeaderObj.read(fp)
81 76 fp.close()
82 77
83 78 if not(sts):
84 79 print("Skipping the file %s because it has not a valid header" % (filename))
85 80 return 0
86 81
87 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 83 return 0
89 84
90 85 return 1
91 86
92 87
93 88 def isTimeInRange(thisTime, startTime, endTime):
94 89 if endTime >= startTime:
95 90 if (thisTime < startTime) or (thisTime > endTime):
96 91 return 0
97 92 return 1
98 93 else:
99 94 if (thisTime < startTime) and (thisTime > endTime):
100 95 return 0
101 96 return 1
102 97
103 98
104 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 100 """
106 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107 102
108 103 Inputs:
109 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110 105
111 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
112 107
113 108 endDate : fecha final del rango seleccionado en formato datetime.date
114 109
115 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116 111
117 112 endTime : tiempo final del rango seleccionado en formato datetime.time
118 113
119 114 Return:
120 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 116 fecha especificado, de lo contrario retorna False.
122 117
123 118 Excepciones:
124 119 Si el archivo no existe o no puede ser abierto
125 120 Si la cabecera no puede ser leida.
126 121
127 122 """
128 123
129 124 try:
130 125 fp = open(filename, 'rb')
131 126 except IOError:
132 127 print("The file %s can't be opened" % (filename))
133 128 return None
134 129
135 130 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 131 systemHeaderObj = SystemHeader()
137 132 radarControllerHeaderObj = RadarControllerHeader()
138 133 processingHeaderObj = ProcessingHeader()
139 134
140 135 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141 136
142 137 sts = firstBasicHeaderObj.read(fp)
143 138
144 139 if not(sts):
145 140 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
146 141 return None
147 142
148 143 if not systemHeaderObj.read(fp):
149 144 return None
150 145
151 146 if not radarControllerHeaderObj.read(fp):
152 147 return None
153 148
154 149 if not processingHeaderObj.read(fp):
155 150 return None
156 151
157 152 filesize = os.path.getsize(filename)
158 153
159 154 offset = processingHeaderObj.blockSize + 24 # header size
160 155
161 156 if filesize <= offset:
162 157 print("[Reading] %s: This file has not enough data" % filename)
163 158 return None
164 159
165 160 fp.seek(-offset, 2)
166 161
167 162 sts = lastBasicHeaderObj.read(fp)
168 163
169 164 fp.close()
170 165
171 166 thisDatetime = lastBasicHeaderObj.datatime
172 167 thisTime_last_block = thisDatetime.time()
173 168
174 169 thisDatetime = firstBasicHeaderObj.datatime
175 170 thisDate = thisDatetime.date()
176 171 thisTime_first_block = thisDatetime.time()
177 172
178 173 # General case
179 174 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 175 #-----------o----------------------------o-----------
181 176 # startTime endTime
182 177
183 178 if endTime >= startTime:
184 179 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 180 return None
186 181
187 182 return thisDatetime
188 183
189 184 # If endTime < startTime then endTime belongs to the next day
190 185
191 186 #<<<<<<<<<<<o o>>>>>>>>>>>
192 187 #-----------o----------------------------o-----------
193 188 # endTime startTime
194 189
195 190 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 191 return None
197 192
198 193 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 194 return None
200 195
201 196 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 197 return None
203 198
204 199 return thisDatetime
205 200
206 201
207 202 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 203 """
209 204 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210 205
211 206 Inputs:
212 207 folder : nombre completo del directorio.
213 208 Su formato deberia ser "/path_root/?YYYYDDD"
214 209
215 210 siendo:
216 211 YYYY : Anio (ejemplo 2015)
217 212 DDD : Dia del anio (ejemplo 305)
218 213
219 214 startDate : fecha inicial del rango seleccionado en formato datetime.date
220 215
221 216 endDate : fecha final del rango seleccionado en formato datetime.date
222 217
223 218 Return:
224 219 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 220 fecha especificado, de lo contrario retorna False.
226 221 Excepciones:
227 222 Si el directorio no tiene el formato adecuado
228 223 """
229 224
230 225 basename = os.path.basename(folder)
231 226
232 227 if not isRadarFolder(basename):
233 228 print("The folder %s has not the rigth format" % folder)
234 229 return 0
235 230
236 231 if startDate and endDate:
237 232 thisDate = getDateFromRadarFolder(basename)
238 233
239 234 if thisDate < startDate:
240 235 return 0
241 236
242 237 if thisDate > endDate:
243 238 return 0
244 239
245 240 return 1
246 241
247 242
248 243 def isFileInDateRange(filename, startDate=None, endDate=None):
249 244 """
250 245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251 246
252 247 Inputs:
253 248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254 249
255 250 Su formato deberia ser "?YYYYDDDsss"
256 251
257 252 siendo:
258 253 YYYY : Anio (ejemplo 2015)
259 254 DDD : Dia del anio (ejemplo 305)
260 255 sss : set
261 256
262 257 startDate : fecha inicial del rango seleccionado en formato datetime.date
263 258
264 259 endDate : fecha final del rango seleccionado en formato datetime.date
265 260
266 261 Return:
267 262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 263 fecha especificado, de lo contrario retorna False.
269 264 Excepciones:
270 265 Si el archivo no tiene el formato adecuado
271 266 """
272 267
273 268 basename = os.path.basename(filename)
274 269
275 270 if not isRadarFile(basename):
276 271 print("The filename %s has not the rigth format" % filename)
277 272 return 0
278 273
279 274 if startDate and endDate:
280 275 thisDate = getDateFromRadarFile(basename)
281 276
282 277 if thisDate < startDate:
283 278 return 0
284 279
285 280 if thisDate > endDate:
286 281 return 0
287 282
288 283 return 1
289 284
290 285
291 286 def getFileFromSet(path, ext, set):
292 287 validFilelist = []
293 288 fileList = os.listdir(path)
294 289
295 290 # 0 1234 567 89A BCDE
296 291 # H YYYY DDD SSS .ext
297 292
298 293 for thisFile in fileList:
299 294 try:
300 295 year = int(thisFile[1:5])
301 296 doy = int(thisFile[5:8])
302 297 except:
303 298 continue
304 299
305 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 301 continue
307 302
308 303 validFilelist.append(thisFile)
309 304
310 305 myfile = fnmatch.filter(
311 306 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312 307
313 308 if len(myfile) != 0:
314 309 return myfile[0]
315 310 else:
316 311 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 312 print('the filename %s does not exist' % filename)
318 313 print('...going to the last file: ')
319 314
320 315 if validFilelist:
321 316 validFilelist = sorted(validFilelist, key=str.lower)
322 317 return validFilelist[-1]
323 318
324 319 return None
325 320
326 321
327 322 def getlastFileFromPath(path, ext):
328 323 """
329 324 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 325 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331 326
332 327 Input:
333 328 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 329 ext : extension de los files contenidos en una carpeta
335 330
336 331 Return:
337 332 El ultimo file de una determinada carpeta, no se considera el path.
338 333 """
339 334 validFilelist = []
340 335 fileList = os.listdir(path)
341 336
342 337 # 0 1234 567 89A BCDE
343 338 # H YYYY DDD SSS .ext
344 339
345 340 for thisFile in fileList:
346 341
347 342 year = thisFile[1:5]
348 343 if not isNumber(year):
349 344 continue
350 345
351 346 doy = thisFile[5:8]
352 347 if not isNumber(doy):
353 348 continue
354 349
355 350 year = int(year)
356 351 doy = int(doy)
357 352
358 353 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 354 continue
360 355
361 356 validFilelist.append(thisFile)
362 357
363 358 if validFilelist:
364 359 validFilelist = sorted(validFilelist, key=str.lower)
365 360 return validFilelist[-1]
366 361
367 362 return None
368 363
369 364
370 365 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 366 """
372 367 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 368 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 369 el path exacto de un determinado file.
375 370
376 371 Example :
377 372 nombre correcto del file es .../.../D2009307/P2009307367.ext
378 373
379 374 Entonces la funcion prueba con las siguientes combinaciones
380 375 .../.../y2009307367.ext
381 376 .../.../Y2009307367.ext
382 377 .../.../x2009307/y2009307367.ext
383 378 .../.../x2009307/Y2009307367.ext
384 379 .../.../X2009307/y2009307367.ext
385 380 .../.../X2009307/Y2009307367.ext
386 381 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387 382
388 383 Return:
389 384 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 385 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 386 para el filename
392 387 """
393 388 fullfilename = None
394 389 find_flag = False
395 390 filename = None
396 391
397 392 prefixDirList = [None, 'd', 'D']
398 393 if ext.lower() == ".r": # voltage
399 394 prefixFileList = ['d', 'D']
400 395 elif ext.lower() == ".pdata": # spectra
401 396 prefixFileList = ['p', 'P']
402 397 else:
403 398 return None, filename
404 399
405 400 # barrido por las combinaciones posibles
406 401 for prefixDir in prefixDirList:
407 402 thispath = path
408 403 if prefixDir != None:
409 404 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 405 if foldercounter == 0:
411 406 thispath = os.path.join(path, "%s%04d%03d" %
412 407 (prefixDir, year, doy))
413 408 else:
414 409 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 410 prefixDir, year, doy, foldercounter))
416 411 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 412 # formo el nombre del file xYYYYDDDSSS.ext
418 413 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 414 fullfilename = os.path.join(
420 415 thispath, filename) # formo el path completo
421 416
422 417 if os.path.exists(fullfilename): # verifico que exista
423 418 find_flag = True
424 419 break
425 420 if find_flag:
426 421 break
427 422
428 423 if not(find_flag):
429 424 return None, filename
430 425
431 426 return fullfilename, filename
432 427
433 428
434 429 def isRadarFolder(folder):
435 430 try:
436 431 year = int(folder[1:5])
437 432 doy = int(folder[5:8])
438 433 except:
439 434 return 0
440 435
441 436 return 1
442 437
443 438
444 439 def isRadarFile(file):
445 440 try:
446 441 year = int(file[1:5])
447 442 doy = int(file[5:8])
448 443 set = int(file[8:11])
449 444 except:
450 445 return 0
451 446
452 447 return 1
453 448
454 449
455 450 def getDateFromRadarFile(file):
456 451 try:
457 452 year = int(file[1:5])
458 453 doy = int(file[5:8])
459 454 set = int(file[8:11])
460 455 except:
461 456 return None
462 457
463 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 459 return thisDate
465 460
466 461
467 462 def getDateFromRadarFolder(folder):
468 463 try:
469 464 year = int(folder[1:5])
470 465 doy = int(folder[5:8])
471 466 except:
472 467 return None
473 468
474 469 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 470 return thisDate
476 471
477 472
478 473 class JRODataIO:
479 474
480 475 c = 3E8
481 476
482 477 isConfig = False
483 478
484 479 basicHeaderObj = None
485 480
486 481 systemHeaderObj = None
487 482
488 483 radarControllerHeaderObj = None
489 484
490 485 processingHeaderObj = None
491 486
492 487 dtype = None
493 488
494 489 pathList = []
495 490
496 491 filenameList = []
497 492
498 493 filename = None
499 494
500 495 ext = None
501 496
502 497 flagIsNewFile = 1
503 498
504 499 flagDiscontinuousBlock = 0
505 500
506 501 flagIsNewBlock = 0
507 502
508 503 fp = None
509 504
510 505 firstHeaderSize = 0
511 506
512 507 basicHeaderSize = 24
513 508
514 509 versionFile = 1103
515 510
516 511 fileSize = None
517 512
518 513 # ippSeconds = None
519 514
520 515 fileSizeByHeader = None
521 516
522 517 fileIndex = None
523 518
524 519 profileIndex = None
525 520
526 521 blockIndex = None
527 522
528 523 nTotalBlocks = None
529 524
530 525 maxTimeStep = 30
531 526
532 527 lastUTTime = None
533 528
534 529 datablock = None
535 530
536 531 dataOut = None
537 532
538 533 blocksize = None
539 534
540 535 getByBlock = False
541 536
542 537 def __init__(self):
543 538
544 539 raise NotImplementedError
545 540
546 541 def run(self):
547 542
548 543 raise NotImplementedError
549 544
550 545 def getDtypeWidth(self):
551 546
552 547 dtype_index = get_dtype_index(self.dtype)
553 548 dtype_width = get_dtype_width(dtype_index)
554 549
555 550 return dtype_width
556 551
557 552 def getAllowedArgs(self):
558 553 if hasattr(self, '__attrs__'):
559 554 return self.__attrs__
560 555 else:
561 556 return inspect.getargspec(self.run).args
562 557
563 558
564 559 class JRODataReader(JRODataIO):
565 560
566 561 online = 0
567 562
568 563 realtime = 0
569 564
570 565 nReadBlocks = 0
571 566
572 567 delay = 10 # number of seconds waiting a new file
573 568
574 569 nTries = 3 # quantity tries
575 570
576 571 nFiles = 3 # number of files for searching
577 572
578 573 path = None
579 574
580 575 foldercounter = 0
581 576
582 577 flagNoMoreFiles = 0
583 578
584 579 datetimeList = []
585 580
586 581 __isFirstTimeOnline = 1
587 582
588 583 __printInfo = True
589 584
590 585 profileIndex = None
591 586
592 587 nTxs = 1
593 588
594 589 txIndex = None
595 590
596 591 # Added--------------------
597 592
598 593 selBlocksize = None
599 594
600 595 selBlocktime = None
601 596
602 597 def __init__(self):
603 598 """
604 599 This class is used to find data files
605 600
606 601 Example:
607 602 reader = JRODataReader()
608 603 fileList = reader.findDataFiles()
609 604
610 605 """
611 606 pass
612 607
613 608 def createObjByDefault(self):
614 609 """
615 610
616 611 """
617 612 raise NotImplementedError
618 613
619 614 def getBlockDimension(self):
620 615
621 616 raise NotImplementedError
622 617
623 618 def searchFilesOffLine(self,
624 619 path,
625 620 startDate=None,
626 621 endDate=None,
627 622 startTime=datetime.time(0, 0, 0),
628 623 endTime=datetime.time(23, 59, 59),
629 624 set=None,
630 625 expLabel='',
631 626 ext='.r',
632 627 cursor=None,
633 628 skip=None,
634 629 walk=True):
635 630
636 631 self.filenameList = []
637 632 self.datetimeList = []
638 633
639 634 pathList = []
640 635
641 636 dateList, pathList = self.findDatafiles(
642 637 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643 638
644 639 if dateList == []:
645 640 return [], []
646 641
647 642 if len(dateList) > 1:
648 643 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
649 644 else:
650 645 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
651 646
652 647 filenameList = []
653 648 datetimeList = []
654 649
655 650 for thisPath in pathList:
656 651
657 652 fileList = glob.glob1(thisPath, "*%s" % ext)
658 653 fileList.sort()
659 654
660 655 for file in fileList:
661 656
662 657 filename = os.path.join(thisPath, file)
663 658
664 659 if not isFileInDateRange(filename, startDate, endDate):
665 660 continue
666 661
667 662 thisDatetime = isFileInTimeRange(
668 663 filename, startDate, endDate, startTime, endTime)
669 664
670 665 if not(thisDatetime):
671 666 continue
672 667
673 668 filenameList.append(filename)
674 669 datetimeList.append(thisDatetime)
675 670
676 671 if cursor is not None and skip is not None:
677 672 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 673 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679 674
680 675 if not(filenameList):
681 676 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
682 677 return [], []
683 678
684 679 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
685 680
686 681 # for i in range(len(filenameList)):
687 682 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688 683
689 684 self.filenameList = filenameList
690 685 self.datetimeList = datetimeList
691 686
692 687 return pathList, filenameList
693 688
694 689 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 690 """
696 691 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 692 devuelve el archivo encontrado ademas de otros datos.
698 693
699 694 Input:
700 695 path : carpeta donde estan contenidos los files que contiene data
701 696
702 697 expLabel : Nombre del subexperimento (subfolder)
703 698
704 699 ext : extension de los files
705 700
706 701 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707 702
708 703 Return:
709 704 directory : eL directorio donde esta el file encontrado
710 705 filename : el ultimo file de una determinada carpeta
711 706 year : el anho
712 707 doy : el numero de dia del anho
713 708 set : el set del archivo
714 709
715 710
716 711 """
717 712 if not os.path.isdir(path):
718 713 return None, None, None, None, None, None
719 714
720 715 dirList = []
721 716
722 717 if not walk:
723 718 fullpath = path
724 719 foldercounter = 0
725 720 else:
726 721 # Filtra solo los directorios
727 722 for thisPath in os.listdir(path):
728 723 if not os.path.isdir(os.path.join(path, thisPath)):
729 724 continue
730 725 if not isRadarFolder(thisPath):
731 726 continue
732 727
733 728 dirList.append(thisPath)
734 729
735 730 if not(dirList):
736 731 return None, None, None, None, None, None
737 732
738 733 dirList = sorted(dirList, key=str.lower)
739 734
740 735 doypath = dirList[-1]
741 736 foldercounter = int(doypath.split('_')[1]) if len(
742 737 doypath.split('_')) > 1 else 0
743 738 fullpath = os.path.join(path, doypath, expLabel)
744 739
745 740 print("[Reading] %s folder was found: " % (fullpath))
746 741
747 742 if set == None:
748 743 filename = getlastFileFromPath(fullpath, ext)
749 744 else:
750 745 filename = getFileFromSet(fullpath, ext, set)
751 746
752 747 if not(filename):
753 748 return None, None, None, None, None, None
754 749
755 750 print("[Reading] %s file was found" % (filename))
756 751
757 752 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 753 return None, None, None, None, None, None
759 754
760 755 year = int(filename[1:5])
761 756 doy = int(filename[5:8])
762 757 set = int(filename[8:11])
763 758
764 759 return fullpath, foldercounter, filename, year, doy, set
765 760
766 761 def __setNextFileOffline(self):
767 762
768 763 idFile = self.fileIndex
769 764
770 765 while (True):
771 766 idFile += 1
772 767 if not(idFile < len(self.filenameList)):
773 768 self.flagNoMoreFiles = 1
774 # print "[Reading] No more Files"
775 769 return 0
776 770
777 771 filename = self.filenameList[idFile]
778 772
779 773 if not(self.__verifyFile(filename)):
780 774 continue
781 775
782 776 fileSize = os.path.getsize(filename)
783 777 fp = open(filename, 'rb')
784 778 break
785 779
786 780 self.flagIsNewFile = 1
787 781 self.fileIndex = idFile
788 782 self.filename = filename
789 783 self.fileSize = fileSize
790 784 self.fp = fp
791 785
792 786 # print "[Reading] Setting the file: %s"%self.filename
793 787
794 788 return 1
795 789
796 790 def __setNextFileOnline(self):
797 791 """
798 792 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 793 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 794 siguientes.
801 795
802 796 Affected:
803 797 self.flagIsNewFile
804 798 self.filename
805 799 self.fileSize
806 800 self.fp
807 801 self.set
808 802 self.flagNoMoreFiles
809 803
810 804 Return:
811 805 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 806 1 : si el file fue abierto con exito y esta listo a ser leido
813 807
814 808 Excepciones:
815 809 Si un determinado file no puede ser abierto
816 810 """
817 811 nFiles = 0
818 812 fileOk_flag = False
819 813 firstTime_flag = True
820 814
821 815 self.set += 1
822 816
823 817 if self.set > 999:
824 818 self.set = 0
825 819 self.foldercounter += 1
826 820
827 821 # busca el 1er file disponible
828 822 fullfilename, filename = checkForRealPath(
829 823 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 824 if fullfilename:
831 825 if self.__verifyFile(fullfilename, False):
832 826 fileOk_flag = True
833 827
834 828 # si no encuentra un file entonces espera y vuelve a buscar
835 829 if not(fileOk_flag):
836 830 # busco en los siguientes self.nFiles+1 files posibles
837 831 for nFiles in range(self.nFiles + 1):
838 832
839 833 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 834 tries = self.nTries
841 835 else:
842 836 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843 837
844 838 for nTries in range(tries):
845 839 if firstTime_flag:
846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
847 sleep(self.delay)
840 log.warning(
841 "Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1),
842 self.name)
843 time.sleep(self.delay)
848 844 else:
849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
845 log.warning(
846 "Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext),
847 self.name)
850 848
851 849 fullfilename, filename = checkForRealPath(
852 850 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 851 if fullfilename:
854 852 if self.__verifyFile(fullfilename):
855 853 fileOk_flag = True
856 854 break
857 855
858 856 if fileOk_flag:
859 857 break
860 858
861 859 firstTime_flag = False
862 860
863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
861 log.warning(
862 'Skipping the file {} due to this file doesn\'t exist'.format(filename),
863 self.name)
864 864 self.set += 1
865 865
866 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 867 if nFiles == (self.nFiles - 1):
868 868 self.set = 0
869 869 self.doy += 1
870 870 self.foldercounter = 0
871 871
872 872 if fileOk_flag:
873 873 self.fileSize = os.path.getsize(fullfilename)
874 874 self.filename = fullfilename
875 875 self.flagIsNewFile = 1
876 876 if self.fp != None:
877 877 self.fp.close()
878 878 self.fp = open(fullfilename, 'rb')
879 879 self.flagNoMoreFiles = 0
880 # print '[Reading] Setting the file: %s' % fullfilename
881 880 else:
881 raise schainpy.admin.SchainError('Time for waiting new files reach')
882 882 self.fileSize = 0
883 883 self.filename = None
884 884 self.flagIsNewFile = 0
885 885 self.fp = None
886 886 self.flagNoMoreFiles = 1
887 # print '[Reading] No more files to read'
888 887
889 888 return fileOk_flag
890 889
891 890 def setNextFile(self):
892 891 if self.fp != None:
893 892 self.fp.close()
894 893
895 894 if self.online:
896 895 newFile = self.__setNextFileOnline()
897 896 else:
898 897 newFile = self.__setNextFileOffline()
899 898
900 899 if not(newFile):
901 self.dataOut.error = 'No more files to read'
902 return 0
900 raise schainpy.admin.SchainWarning('No more files to read')
901
903 902
904 903 if self.verbose:
905 904 print('[Reading] Setting the file: %s' % self.filename)
906 905
907 906 self.__readFirstHeader()
908 907 self.nReadBlocks = 0
909 908 return 1
910 909
911 910 def __waitNewBlock(self):
912 911 """
913 912 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 913
915 914 Si el modo de lectura es OffLine siempre retorn 0
916 915 """
917 916 if not self.online:
918 917 return 0
919 918
920 919 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 920 return 0
922 921
923 922 currentPointer = self.fp.tell()
924 923
925 924 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 925
927 926 for nTries in range(self.nTries):
928 927
929 928 self.fp.close()
930 929 self.fp = open(self.filename, 'rb')
931 930 self.fp.seek(currentPointer)
932 931
933 932 self.fileSize = os.path.getsize(self.filename)
934 933 currentSize = self.fileSize - currentPointer
935 934
936 935 if (currentSize >= neededSize):
937 936 self.basicHeaderObj.read(self.fp)
938 937 return 1
939 938
940 939 if self.fileSize == self.fileSizeByHeader:
941 940 # self.flagEoF = True
942 941 return 0
943 942
944 943 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 sleep(self.delay)
944 time.sleep(self.delay)
946 945
947 946 return 0
948 947
949 948 def waitDataBlock(self, pointer_location, blocksize=None):
950 949
951 950 currentPointer = pointer_location
952 951 if blocksize is None:
953 952 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 953 else:
955 954 neededSize = blocksize
956 955
957 956 for nTries in range(self.nTries):
958 957 self.fp.close()
959 958 self.fp = open(self.filename, 'rb')
960 959 self.fp.seek(currentPointer)
961 960
962 961 self.fileSize = os.path.getsize(self.filename)
963 962 currentSize = self.fileSize - currentPointer
964 963
965 964 if (currentSize >= neededSize):
966 965 return 1
967 966
968 967 log.warning(
969 968 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
970 969 self.name
971 970 )
972 sleep(self.delay)
971 time.sleep(self.delay)
973 972
974 973 return 0
975 974
976 975 def __jumpToLastBlock(self):
977 976
978 977 if not(self.__isFirstTimeOnline):
979 978 return
980 979
981 980 csize = self.fileSize - self.fp.tell()
982 981 blocksize = self.processingHeaderObj.blockSize
983 982
984 983 # salta el primer bloque de datos
985 984 if csize > self.processingHeaderObj.blockSize:
986 985 self.fp.seek(self.fp.tell() + blocksize)
987 986 else:
988 987 return
989 988
990 989 csize = self.fileSize - self.fp.tell()
991 990 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
992 991 while True:
993 992
994 993 if self.fp.tell() < self.fileSize:
995 994 self.fp.seek(self.fp.tell() + neededsize)
996 995 else:
997 996 self.fp.seek(self.fp.tell() - neededsize)
998 997 break
999 998
1000 999 # csize = self.fileSize - self.fp.tell()
1001 1000 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1002 1001 # factor = int(csize/neededsize)
1003 1002 # if factor > 0:
1004 1003 # self.fp.seek(self.fp.tell() + factor*neededsize)
1005 1004
1006 1005 self.flagIsNewFile = 0
1007 1006 self.__isFirstTimeOnline = 0
1008 1007
1009 1008 def __setNewBlock(self):
1010 1009 # if self.server is None:
1011 1010 if self.fp == None:
1012 1011 return 0
1013 1012
1014 1013 # if self.online:
1015 1014 # self.__jumpToLastBlock()
1016
1015
1017 1016 if self.flagIsNewFile:
1018 1017 self.lastUTTime = self.basicHeaderObj.utc
1019 1018 return 1
1020 1019
1021 1020 if self.realtime:
1022 1021 self.flagDiscontinuousBlock = 1
1023 1022 if not(self.setNextFile()):
1024 1023 return 0
1025 1024 else:
1026 1025 return 1
1027 1026 # if self.server is None:
1028 1027 currentSize = self.fileSize - self.fp.tell()
1029 1028 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1030 1029 if (currentSize >= neededSize):
1031 1030 self.basicHeaderObj.read(self.fp)
1032 1031 self.lastUTTime = self.basicHeaderObj.utc
1033 1032 return 1
1034 1033 # else:
1035 1034 # self.basicHeaderObj.read(self.zHeader)
1036 1035 # self.lastUTTime = self.basicHeaderObj.utc
1037 1036 # return 1
1038 1037 if self.__waitNewBlock():
1039 1038 self.lastUTTime = self.basicHeaderObj.utc
1040 1039 return 1
1041 1040 # if self.server is None:
1042 1041 if not(self.setNextFile()):
1043 1042 return 0
1044 1043
1045 1044 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1046 1045 self.lastUTTime = self.basicHeaderObj.utc
1047 1046
1048 1047 self.flagDiscontinuousBlock = 0
1049 1048
1050 1049 if deltaTime > self.maxTimeStep:
1051 1050 self.flagDiscontinuousBlock = 1
1052 1051
1053 1052 return 1
1054 1053
1055 1054 def readNextBlock(self):
1056 1055
1057 1056 # Skip block out of startTime and endTime
1058 1057 while True:
1059 1058 if not(self.__setNewBlock()):
1060 self.dataOut.error = 'No more files to read'
1061 return 0
1059 raise schainpy.admin.SchainWarning('No more files to read')
1062 1060
1063 1061 if not(self.readBlock()):
1064 1062 return 0
1065 1063
1066 1064 self.getBasicHeader()
1067 1065 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1068 1066 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1069 1067 self.processingHeaderObj.dataBlocksPerFile,
1070 1068 self.dataOut.datatime.ctime()))
1071 1069 continue
1072 1070
1073 1071 break
1074 1072
1075 1073 if self.verbose:
1076 1074 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1077 1075 self.processingHeaderObj.dataBlocksPerFile,
1078 1076 self.dataOut.datatime.ctime()))
1079 1077 return 1
1080 1078
1081 1079 def __readFirstHeader(self):
1082 1080
1083 1081 self.basicHeaderObj.read(self.fp)
1084 1082 self.systemHeaderObj.read(self.fp)
1085 1083 self.radarControllerHeaderObj.read(self.fp)
1086 1084 self.processingHeaderObj.read(self.fp)
1087 1085
1088 1086 self.firstHeaderSize = self.basicHeaderObj.size
1089 1087
1090 1088 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1091 1089 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1092 1090 if datatype == 0:
1093 1091 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1094 1092 elif datatype == 1:
1095 1093 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1096 1094 elif datatype == 2:
1097 1095 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1098 1096 elif datatype == 3:
1099 1097 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1100 1098 elif datatype == 4:
1101 1099 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1102 1100 elif datatype == 5:
1103 1101 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1104 1102 else:
1105 1103 raise ValueError('Data type was not defined')
1106 1104
1107 1105 self.dtype = datatype_str
1108 1106 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1109 1107 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1110 1108 self.firstHeaderSize + self.basicHeaderSize * \
1111 1109 (self.processingHeaderObj.dataBlocksPerFile - 1)
1112 1110 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1113 1111 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1114 1112 self.getBlockDimension()
1115 1113
1116 1114 def __verifyFile(self, filename, msgFlag=True):
1117 1115
1118 1116 msg = None
1119 1117
1120 1118 try:
1121 1119 fp = open(filename, 'rb')
1122 1120 except IOError:
1123 1121
1124 1122 if msgFlag:
1125 1123 print("[Reading] File %s can't be opened" % (filename))
1126 1124
1127 1125 return False
1128 1126
1129 1127 currentPosition = fp.tell()
1130 1128 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1131 1129
1132 1130 if neededSize == 0:
1133 1131 basicHeaderObj = BasicHeader(LOCALTIME)
1134 1132 systemHeaderObj = SystemHeader()
1135 1133 radarControllerHeaderObj = RadarControllerHeader()
1136 1134 processingHeaderObj = ProcessingHeader()
1137 1135
1138 1136 if not(basicHeaderObj.read(fp)):
1139 1137 fp.close()
1140 1138 return False
1141 1139
1142 1140 if not(systemHeaderObj.read(fp)):
1143 1141 fp.close()
1144 1142 return False
1145 1143
1146 1144 if not(radarControllerHeaderObj.read(fp)):
1147 1145 fp.close()
1148 1146 return False
1149 1147
1150 1148 if not(processingHeaderObj.read(fp)):
1151 1149 fp.close()
1152 1150 return False
1153 1151
1154 1152 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1155 1153 else:
1156 1154 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1157 1155
1158 1156 fp.close()
1159 1157
1160 1158 fileSize = os.path.getsize(filename)
1161 1159 currentSize = fileSize - currentPosition
1162 1160
1163 1161 if currentSize < neededSize:
1164 1162 if msgFlag and (msg != None):
1165 1163 print(msg)
1166 1164 return False
1167 1165
1168 1166 return True
1169 1167
1170 1168 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1171 1169
1172 1170 path_empty = True
1173 1171
1174 1172 dateList = []
1175 1173 pathList = []
1176 1174
1177 1175 multi_path = path.split(',')
1178 1176
1179 1177 if not walk:
1180 1178
1181 1179 for single_path in multi_path:
1182 1180
1183 1181 if not os.path.isdir(single_path):
1184 1182 continue
1185 1183
1186 1184 fileList = glob.glob1(single_path, "*" + ext)
1187 1185
1188 1186 if not fileList:
1189 1187 continue
1190 1188
1191 1189 path_empty = False
1192 1190
1193 1191 fileList.sort()
1194 1192
1195 1193 for thisFile in fileList:
1196 1194
1197 1195 if not os.path.isfile(os.path.join(single_path, thisFile)):
1198 1196 continue
1199 1197
1200 1198 if not isRadarFile(thisFile):
1201 1199 continue
1202 1200
1203 1201 if not isFileInDateRange(thisFile, startDate, endDate):
1204 1202 continue
1205 1203
1206 1204 thisDate = getDateFromRadarFile(thisFile)
1207 1205
1208 1206 if thisDate in dateList or single_path in pathList:
1209 1207 continue
1210 1208
1211 1209 dateList.append(thisDate)
1212 1210 pathList.append(single_path)
1213 1211
1214 1212 else:
1215 1213 for single_path in multi_path:
1216 1214
1217 1215 if not os.path.isdir(single_path):
1218 1216 continue
1219 1217
1220 1218 dirList = []
1221 1219
1222 1220 for thisPath in os.listdir(single_path):
1223 1221
1224 1222 if not os.path.isdir(os.path.join(single_path, thisPath)):
1225 1223 continue
1226 1224
1227 1225 if not isRadarFolder(thisPath):
1228 1226 continue
1229 1227
1230 1228 if not isFolderInDateRange(thisPath, startDate, endDate):
1231 1229 continue
1232 1230
1233 1231 dirList.append(thisPath)
1234 1232
1235 1233 if not dirList:
1236 1234 continue
1237 1235
1238 1236 dirList.sort()
1239 1237
1240 1238 for thisDir in dirList:
1241 1239
1242 1240 datapath = os.path.join(single_path, thisDir, expLabel)
1243 1241 fileList = glob.glob1(datapath, "*" + ext)
1244 1242
1245 1243 if not fileList:
1246 1244 continue
1247 1245
1248 1246 path_empty = False
1249 1247
1250 1248 thisDate = getDateFromRadarFolder(thisDir)
1251 1249
1252 1250 pathList.append(datapath)
1253 1251 dateList.append(thisDate)
1254 1252
1255 1253 dateList.sort()
1256 1254
1257 1255 if walk:
1258 1256 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1259 1257 else:
1260 1258 pattern_path = multi_path[0]
1261 1259
1262 1260 if path_empty:
1263 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1261 raise schainpy.admin.SchainError("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1264 1262 else:
1265 1263 if not dateList:
1266 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1264 raise schainpy.admin.SchainError("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1267 1265
1268 1266 if include_path:
1269 1267 return dateList, pathList
1270 1268
1271 1269 return dateList
1272 1270
1273 1271 def setup(self,
1274 1272 path=None,
1275 1273 startDate=None,
1276 1274 endDate=None,
1277 1275 startTime=datetime.time(0, 0, 0),
1278 1276 endTime=datetime.time(23, 59, 59),
1279 1277 set=None,
1280 1278 expLabel="",
1281 1279 ext=None,
1282 1280 online=False,
1283 1281 delay=60,
1284 1282 walk=True,
1285 1283 getblock=False,
1286 1284 nTxs=1,
1287 1285 realtime=False,
1288 1286 blocksize=None,
1289 1287 blocktime=None,
1290 1288 skip=None,
1291 1289 cursor=None,
1292 1290 warnings=True,
1293 1291 verbose=True,
1294 1292 server=None,
1295 1293 format=None,
1296 1294 oneDDict=None,
1297 1295 twoDDict=None,
1298 1296 independentParam=None):
1297
1298 self.online = online
1299 self.realtime = realtime
1300 self.delay = delay
1301 self.getByBlock = getblock
1302 self.nTxs = nTxs
1303 self.startTime = startTime
1304 self.endTime = endTime
1305 self.endDate = endDate
1306 self.startDate = startDate
1307
1299 1308 if server is not None:
1300 1309 if 'tcp://' in server:
1301 1310 address = server
1302 1311 else:
1303 1312 address = 'ipc:///tmp/%s' % server
1304 1313 self.server = address
1305 1314 self.context = zmq.Context()
1306 1315 self.receiver = self.context.socket(zmq.PULL)
1307 1316 self.receiver.connect(self.server)
1308 1317 time.sleep(0.5)
1309 1318 print('[Starting] ReceiverData from {}'.format(self.server))
1310 1319 else:
1311 1320 self.server = None
1312 1321 if path == None:
1313 1322 raise ValueError("[Reading] The path is not valid")
1314 1323
1315 1324 if ext == None:
1316 1325 ext = self.ext
1317 1326
1318 1327 if online:
1319 1328 print("[Reading] Searching files in online mode...")
1320 1329
1321 1330 for nTries in range(self.nTries):
1322 1331 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1323 1332 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1324 1333
1325 1334 if fullpath:
1326 1335 break
1327 1336
1328 1337 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1329 sleep(self.delay)
1338 time.sleep(self.delay)
1330 1339
1331 1340 if not(fullpath):
1332 self.dataOut.error = 'There isn\'t any valid file in {}'.format(path)
1341 raise schainpy.admin.SchainError('There isn\'t any valid file in {}'.format(path))
1333 1342 return
1334 1343
1335 1344 self.year = year
1336 1345 self.doy = doy
1337 1346 self.set = set - 1
1338 1347 self.path = path
1339 1348 self.foldercounter = foldercounter
1340 1349 last_set = None
1341 1350 else:
1342 1351 print("[Reading] Searching files in offline mode ...")
1343 1352 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1344 1353 startTime=startTime, endTime=endTime,
1345 1354 set=set, expLabel=expLabel, ext=ext,
1346 1355 walk=walk, cursor=cursor,
1347 1356 skip=skip)
1348 1357
1349 1358 if not(pathList):
1350 1359 self.fileIndex = -1
1351 1360 self.pathList = []
1352 1361 self.filenameList = []
1353 1362 return
1354 1363
1355 1364 self.fileIndex = -1
1356 1365 self.pathList = pathList
1357 1366 self.filenameList = filenameList
1358 1367 file_name = os.path.basename(filenameList[-1])
1359 1368 basename, ext = os.path.splitext(file_name)
1360 1369 last_set = int(basename[-3:])
1361 1370
1362 self.online = online
1363 self.realtime = realtime
1364 self.delay = delay
1371
1365 1372 ext = ext.lower()
1366 1373 self.ext = ext
1367 self.getByBlock = getblock
1368 self.nTxs = nTxs
1369 self.startTime = startTime
1370 self.endTime = endTime
1371 self.endDate = endDate
1372 self.startDate = startDate
1374
1373 1375 # Added-----------------
1374 1376 self.selBlocksize = blocksize
1375 1377 self.selBlocktime = blocktime
1376 1378
1377 1379 # Verbose-----------
1378 1380 self.verbose = verbose
1379 1381 self.warnings = warnings
1380 1382
1381 1383 if not(self.setNextFile()):
1382 1384 if (startDate != None) and (endDate != None):
1383 1385 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1384 1386 elif startDate != None:
1385 1387 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1386 1388 else:
1387 1389 print("[Reading] No files")
1388 1390
1389 1391 self.fileIndex = -1
1390 1392 self.pathList = []
1391 1393 self.filenameList = []
1392 1394 return
1393 1395
1394 # self.getBasicHeader()
1395
1396 1396 if last_set != None:
1397 1397 self.dataOut.last_block = last_set * \
1398 1398 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1399 1399 return
1400 1400
1401 1401 def getBasicHeader(self):
1402 1402
1403 1403 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1404 1404 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1405 1405
1406 1406 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1407 1407
1408 1408 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1409 1409
1410 1410 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1411 1411
1412 1412 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1413 1413
1414 1414 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1415 1415
1416 1416 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1417 1417
1418 1418 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1419 1419
1420 1420 def getFirstHeader(self):
1421 1421
1422 1422 raise NotImplementedError
1423 1423
1424 1424 def getData(self):
1425 1425
1426 1426 raise NotImplementedError
1427 1427
1428 1428 def hasNotDataInBuffer(self):
1429 1429
1430 1430 raise NotImplementedError
1431 1431
1432 1432 def readBlock(self):
1433 1433
1434 1434 raise NotImplementedError
1435 1435
1436 1436 def isEndProcess(self):
1437 1437
1438 1438 return self.flagNoMoreFiles
1439 1439
1440 1440 def printReadBlocks(self):
1441 1441
1442 1442 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1443 1443
1444 1444 def printTotalBlocks(self):
1445 1445
1446 1446 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1447 1447
1448 1448 def printNumberOfBlock(self):
1449 1449 'SPAM!'
1450 1450
1451 1451 # if self.flagIsNewBlock:
1452 1452 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1453 1453 # self.processingHeaderObj.dataBlocksPerFile,
1454 1454 # self.dataOut.datatime.ctime())
1455 1455
1456 1456 def printInfo(self):
1457 1457
1458 1458 if self.__printInfo == False:
1459 1459 return
1460 1460
1461 1461 self.basicHeaderObj.printInfo()
1462 1462 self.systemHeaderObj.printInfo()
1463 1463 self.radarControllerHeaderObj.printInfo()
1464 1464 self.processingHeaderObj.printInfo()
1465 1465
1466 1466 self.__printInfo = False
1467 1467
1468 1468 def run(self,
1469 1469 path=None,
1470 1470 startDate=None,
1471 1471 endDate=None,
1472 1472 startTime=datetime.time(0, 0, 0),
1473 1473 endTime=datetime.time(23, 59, 59),
1474 1474 set=None,
1475 1475 expLabel="",
1476 1476 ext=None,
1477 1477 online=False,
1478 1478 delay=60,
1479 1479 walk=True,
1480 1480 getblock=False,
1481 1481 nTxs=1,
1482 1482 realtime=False,
1483 1483 blocksize=None,
1484 1484 blocktime=None,
1485 1485 skip=None,
1486 1486 cursor=None,
1487 1487 warnings=True,
1488 1488 server=None,
1489 1489 verbose=True,
1490 1490 format=None,
1491 1491 oneDDict=None,
1492 1492 twoDDict=None,
1493 1493 independentParam=None, **kwargs):
1494 1494
1495 1495 if not(self.isConfig):
1496 1496 self.setup(path=path,
1497 1497 startDate=startDate,
1498 1498 endDate=endDate,
1499 1499 startTime=startTime,
1500 1500 endTime=endTime,
1501 1501 set=set,
1502 1502 expLabel=expLabel,
1503 1503 ext=ext,
1504 1504 online=online,
1505 1505 delay=delay,
1506 1506 walk=walk,
1507 1507 getblock=getblock,
1508 1508 nTxs=nTxs,
1509 1509 realtime=realtime,
1510 1510 blocksize=blocksize,
1511 1511 blocktime=blocktime,
1512 1512 skip=skip,
1513 1513 cursor=cursor,
1514 1514 warnings=warnings,
1515 1515 server=server,
1516 1516 verbose=verbose,
1517 1517 format=format,
1518 1518 oneDDict=oneDDict,
1519 1519 twoDDict=twoDDict,
1520 1520 independentParam=independentParam)
1521 1521 self.isConfig = True
1522 1522 if server is None:
1523 1523 self.getData()
1524 1524 else:
1525 1525 self.getFromServer()
1526 1526
1527 1527
1528 1528 class JRODataWriter(JRODataIO):
1529 1529
1530 1530 """
1531 1531 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1532 1532 de los datos siempre se realiza por bloques.
1533 1533 """
1534 1534
1535 1535 blockIndex = 0
1536 1536
1537 1537 path = None
1538 1538
1539 1539 setFile = None
1540 1540
1541 1541 profilesPerBlock = None
1542 1542
1543 1543 blocksPerFile = None
1544 1544
1545 1545 nWriteBlocks = 0
1546 1546
1547 1547 fileDate = None
1548 1548
1549 1549 def __init__(self, dataOut=None):
1550 1550 raise NotImplementedError
1551 1551
1552 1552 def hasAllDataInBuffer(self):
1553 1553 raise NotImplementedError
1554 1554
1555 1555 def setBlockDimension(self):
1556 1556 raise NotImplementedError
1557 1557
1558 1558 def writeBlock(self):
1559 1559 raise NotImplementedError
1560 1560
1561 1561 def putData(self):
1562 1562 raise NotImplementedError
1563 1563
1564 1564 def getProcessFlags(self):
1565 1565
1566 1566 processFlags = 0
1567 1567
1568 1568 dtype_index = get_dtype_index(self.dtype)
1569 1569 procflag_dtype = get_procflag_dtype(dtype_index)
1570 1570
1571 1571 processFlags += procflag_dtype
1572 1572
1573 1573 if self.dataOut.flagDecodeData:
1574 1574 processFlags += PROCFLAG.DECODE_DATA
1575 1575
1576 1576 if self.dataOut.flagDeflipData:
1577 1577 processFlags += PROCFLAG.DEFLIP_DATA
1578 1578
1579 1579 if self.dataOut.code is not None:
1580 1580 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1581 1581
1582 1582 if self.dataOut.nCohInt > 1:
1583 1583 processFlags += PROCFLAG.COHERENT_INTEGRATION
1584 1584
1585 1585 if self.dataOut.type == "Spectra":
1586 1586 if self.dataOut.nIncohInt > 1:
1587 1587 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1588 1588
1589 1589 if self.dataOut.data_dc is not None:
1590 1590 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1591 1591
1592 1592 if self.dataOut.flagShiftFFT:
1593 1593 processFlags += PROCFLAG.SHIFT_FFT_DATA
1594 1594
1595 1595 return processFlags
1596 1596
1597 1597 def setBasicHeader(self):
1598 1598
1599 1599 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1600 1600 self.basicHeaderObj.version = self.versionFile
1601 1601 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1602 1602 utc = numpy.floor(self.dataOut.utctime)
1603 1603 milisecond = (self.dataOut.utctime - utc) * 1000.0
1604 1604 self.basicHeaderObj.utc = utc
1605 1605 self.basicHeaderObj.miliSecond = milisecond
1606 1606 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1607 1607 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1608 1608 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1609 1609
1610 1610 def setFirstHeader(self):
1611 1611 """
1612 1612 Obtiene una copia del First Header
1613 1613
1614 1614 Affected:
1615 1615
1616 1616 self.basicHeaderObj
1617 1617 self.systemHeaderObj
1618 1618 self.radarControllerHeaderObj
1619 1619 self.processingHeaderObj self.
1620 1620
1621 1621 Return:
1622 1622 None
1623 1623 """
1624 1624
1625 1625 raise NotImplementedError
1626 1626
1627 1627 def __writeFirstHeader(self):
1628 1628 """
1629 1629 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1630 1630
1631 1631 Affected:
1632 1632 __dataType
1633 1633
1634 1634 Return:
1635 1635 None
1636 1636 """
1637 1637
1638 1638 # CALCULAR PARAMETROS
1639 1639
1640 1640 sizeLongHeader = self.systemHeaderObj.size + \
1641 1641 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1642 1642 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1643 1643
1644 1644 self.basicHeaderObj.write(self.fp)
1645 1645 self.systemHeaderObj.write(self.fp)
1646 1646 self.radarControllerHeaderObj.write(self.fp)
1647 1647 self.processingHeaderObj.write(self.fp)
1648 1648
1649 1649 def __setNewBlock(self):
1650 1650 """
1651 1651 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1652 1652
1653 1653 Return:
1654 1654 0 : si no pudo escribir nada
1655 1655 1 : Si escribio el Basic el First Header
1656 1656 """
1657 1657 if self.fp == None:
1658 1658 self.setNextFile()
1659 1659
1660 1660 if self.flagIsNewFile:
1661 1661 return 1
1662 1662
1663 1663 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1664 1664 self.basicHeaderObj.write(self.fp)
1665 1665 return 1
1666 1666
1667 1667 if not(self.setNextFile()):
1668 1668 return 0
1669 1669
1670 1670 return 1
1671 1671
1672 1672 def writeNextBlock(self):
1673 1673 """
1674 1674 Selecciona el bloque siguiente de datos y los escribe en un file
1675 1675
1676 1676 Return:
1677 1677 0 : Si no hizo pudo escribir el bloque de datos
1678 1678 1 : Si no pudo escribir el bloque de datos
1679 1679 """
1680 1680 if not(self.__setNewBlock()):
1681 1681 return 0
1682 1682
1683 1683 self.writeBlock()
1684 1684
1685 1685 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1686 1686 self.processingHeaderObj.dataBlocksPerFile))
1687 1687
1688 1688 return 1
1689 1689
1690 1690 def setNextFile(self):
1691 1691 """
1692 1692 Determina el siguiente file que sera escrito
1693 1693
1694 1694 Affected:
1695 1695 self.filename
1696 1696 self.subfolder
1697 1697 self.fp
1698 1698 self.setFile
1699 1699 self.flagIsNewFile
1700 1700
1701 1701 Return:
1702 1702 0 : Si el archivo no puede ser escrito
1703 1703 1 : Si el archivo esta listo para ser escrito
1704 1704 """
1705 1705 ext = self.ext
1706 1706 path = self.path
1707 1707
1708 1708 if self.fp != None:
1709 1709 self.fp.close()
1710 1710
1711 1711 timeTuple = time.localtime(self.dataOut.utctime)
1712 1712 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1713 1713
1714 1714 fullpath = os.path.join(path, subfolder)
1715 1715 setFile = self.setFile
1716 1716
1717 1717 if not(os.path.exists(fullpath)):
1718 1718 os.mkdir(fullpath)
1719 1719 setFile = -1 # inicializo mi contador de seteo
1720 1720 else:
1721 1721 filesList = os.listdir(fullpath)
1722 1722 if len(filesList) > 0:
1723 1723 filesList = sorted(filesList, key=str.lower)
1724 1724 filen = filesList[-1]
1725 1725 # el filename debera tener el siguiente formato
1726 1726 # 0 1234 567 89A BCDE (hex)
1727 1727 # x YYYY DDD SSS .ext
1728 1728 if isNumber(filen[8:11]):
1729 1729 # inicializo mi contador de seteo al seteo del ultimo file
1730 1730 setFile = int(filen[8:11])
1731 1731 else:
1732 1732 setFile = -1
1733 1733 else:
1734 1734 setFile = -1 # inicializo mi contador de seteo
1735 1735
1736 1736 setFile += 1
1737 1737
1738 1738 # If this is a new day it resets some values
1739 1739 if self.dataOut.datatime.date() > self.fileDate:
1740 1740 setFile = 0
1741 1741 self.nTotalBlocks = 0
1742 1742
1743 1743 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1744 1744 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1745 1745
1746 1746 filename = os.path.join(path, subfolder, filen)
1747 1747
1748 1748 fp = open(filename, 'wb')
1749 1749
1750 1750 self.blockIndex = 0
1751 1751
1752 1752 # guardando atributos
1753 1753 self.filename = filename
1754 1754 self.subfolder = subfolder
1755 1755 self.fp = fp
1756 1756 self.setFile = setFile
1757 1757 self.flagIsNewFile = 1
1758 1758 self.fileDate = self.dataOut.datatime.date()
1759 1759
1760 1760 self.setFirstHeader()
1761 1761
1762 1762 print('[Writing] Opening file: %s' % self.filename)
1763 1763
1764 1764 self.__writeFirstHeader()
1765 1765
1766 1766 return 1
1767 1767
1768 1768 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1769 1769 """
1770 1770 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1771 1771
1772 1772 Inputs:
1773 1773 path : directory where data will be saved
1774 1774 profilesPerBlock : number of profiles per block
1775 1775 set : initial file set
1776 1776 datatype : An integer number that defines data type:
1777 1777 0 : int8 (1 byte)
1778 1778 1 : int16 (2 bytes)
1779 1779 2 : int32 (4 bytes)
1780 1780 3 : int64 (8 bytes)
1781 1781 4 : float32 (4 bytes)
1782 1782 5 : double64 (8 bytes)
1783 1783
1784 1784 Return:
1785 1785 0 : Si no realizo un buen seteo
1786 1786 1 : Si realizo un buen seteo
1787 1787 """
1788 1788
1789 1789 if ext == None:
1790 1790 ext = self.ext
1791 1791
1792 1792 self.ext = ext.lower()
1793 1793
1794 1794 self.path = path
1795 1795
1796 1796 if set is None:
1797 1797 self.setFile = -1
1798 1798 else:
1799 1799 self.setFile = set - 1
1800 1800
1801 1801 self.blocksPerFile = blocksPerFile
1802 1802
1803 1803 self.profilesPerBlock = profilesPerBlock
1804 1804
1805 1805 self.dataOut = dataOut
1806 1806 self.fileDate = self.dataOut.datatime.date()
1807 1807 # By default
1808 1808 self.dtype = self.dataOut.dtype
1809 1809
1810 1810 if datatype is not None:
1811 1811 self.dtype = get_numpy_dtype(datatype)
1812 1812
1813 1813 if not(self.setNextFile()):
1814 1814 print("[Writing] There isn't a next file")
1815 1815 return 0
1816 1816
1817 1817 self.setBlockDimension()
1818 1818
1819 1819 return 1
1820 1820
1821 1821 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1822 1822
1823 1823 if not(self.isConfig):
1824 1824
1825 1825 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1826 1826 set=set, ext=ext, datatype=datatype, **kwargs)
1827 1827 self.isConfig = True
1828 1828
1829 1829 self.dataOut = dataOut
1830 1830 self.putData()
1831 1831 return self.dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,796 +1,792
1 1
2 2 '''
3 3 Created on Jul 3, 2014
4 4
5 5 @author: roj-idl71
6 6 '''
7 7 # SUBCHANNELS EN VEZ DE CHANNELS
8 8 # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
9 9 # ACTUALIZACION DE VERSION
10 10 # HEADERS
11 11 # MODULO DE ESCRITURA
12 12 # METADATA
13 13
14 14 import os
15 import time
15 16 import datetime
16 17 import numpy
17 18 import timeit
18 19 from fractions import Fraction
19 20
20 try:
21 from gevent import sleep
22 except:
23 from time import sleep
24
21 import schainpy.admin
25 22 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
26 23 from schainpy.model.data.jrodata import Voltage
27 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
28 from time import time
29 25
30 26 import pickle
31 27 try:
32 28 import digital_rf
33 29 except:
34 30 print('You should install "digital_rf" module if you want to read Digital RF data')
35 31
36 32 @MPDecorator
37 33 class DigitalRFReader(ProcessingUnit):
38 34 '''
39 35 classdocs
40 36 '''
41 37
42 38 def __init__(self):
43 39 '''
44 40 Constructor
45 41 '''
46 42
47 43 ProcessingUnit.__init__(self)
48 44
49 45 self.dataOut = Voltage()
50 46 self.__printInfo = True
51 47 self.__flagDiscontinuousBlock = False
52 48 self.__bufferIndex = 9999999
53 49 self.__ippKm = None
54 50 self.__codeType = 0
55 51 self.__nCode = None
56 52 self.__nBaud = None
57 53 self.__code = None
58 54 self.dtype = None
59 55 self.oldAverage = None
60 56
61 57 def close(self):
62 58 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
63 59 return
64 60
65 61 def __getCurrentSecond(self):
66 62
67 63 return self.__thisUnixSample / self.__sample_rate
68 64
69 65 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
70 66
71 67 def __setFileHeader(self):
72 68 '''
73 69 In this method will be initialized every parameter of dataOut object (header, no data)
74 70 '''
75 71 ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
76 72
77 73 nProfiles = 1.0 / ippSeconds # Number of profiles in one second
78 74
79 75 try:
80 76 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
81 77 self.__radarControllerHeader)
82 78 except:
83 79 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
84 80 txA=0,
85 81 txB=0,
86 82 nWindows=1,
87 83 nHeights=self.__nSamples,
88 84 firstHeight=self.__firstHeigth,
89 85 deltaHeight=self.__deltaHeigth,
90 86 codeType=self.__codeType,
91 87 nCode=self.__nCode, nBaud=self.__nBaud,
92 88 code=self.__code)
93 89
94 90 try:
95 91 self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
96 92 except:
97 93 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
98 94 nProfiles=nProfiles,
99 95 nChannels=len(
100 96 self.__channelList),
101 97 adcResolution=14)
102 98 self.dataOut.type = "Voltage"
103 99
104 100 self.dataOut.data = None
105 101
106 102 self.dataOut.dtype = self.dtype
107 103
108 104 # self.dataOut.nChannels = 0
109 105
110 106 # self.dataOut.nHeights = 0
111 107
112 108 self.dataOut.nProfiles = int(nProfiles)
113 109
114 110 self.dataOut.heightList = self.__firstHeigth + \
115 111 numpy.arange(self.__nSamples, dtype=numpy.float) * \
116 112 self.__deltaHeigth
117 113
118 114 self.dataOut.channelList = list(range(self.__num_subchannels))
119 115
120 116 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
121 117
122 118 # self.dataOut.channelIndexList = None
123 119
124 120 self.dataOut.flagNoData = True
125 121
126 122 self.dataOut.flagDataAsBlock = False
127 123 # Set to TRUE if the data is discontinuous
128 124 self.dataOut.flagDiscontinuousBlock = False
129 125
130 126 self.dataOut.utctime = None
131 127
132 128 # timezone like jroheader, difference in minutes between UTC and localtime
133 129 self.dataOut.timeZone = self.__timezone / 60
134 130
135 131 self.dataOut.dstFlag = 0
136 132
137 133 self.dataOut.errorCount = 0
138 134
139 135 try:
140 136 self.dataOut.nCohInt = self.fixed_metadata_dict.get(
141 137 'nCohInt', self.nCohInt)
142 138
143 139 # asumo que la data esta decodificada
144 140 self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
145 141 'flagDecodeData', self.flagDecodeData)
146 142
147 143 # asumo que la data esta sin flip
148 144 self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
149 145
150 146 self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
151 147
152 148 self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
153 149 except:
154 150 pass
155 151
156 152 self.dataOut.ippSeconds = ippSeconds
157 153
158 154 # Time interval between profiles
159 155 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
160 156
161 157 self.dataOut.frequency = self.__frequency
162 158
163 159 self.dataOut.realtime = self.__online
164 160
165 161 def findDatafiles(self, path, startDate=None, endDate=None):
166 162
167 163 if not os.path.isdir(path):
168 164 return []
169 165
170 166 try:
171 167 digitalReadObj = digital_rf.DigitalRFReader(
172 168 path, load_all_metadata=True)
173 169 except:
174 170 digitalReadObj = digital_rf.DigitalRFReader(path)
175 171
176 172 channelNameList = digitalReadObj.get_channels()
177 173
178 174 if not channelNameList:
179 175 return []
180 176
181 177 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
182 178
183 179 sample_rate = metadata_dict['sample_rate'][0]
184 180
185 181 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
186 182
187 183 try:
188 184 timezone = this_metadata_file['timezone'].value
189 185 except:
190 186 timezone = 0
191 187
192 188 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
193 189 channelNameList[0]) / sample_rate - timezone
194 190
195 191 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
196 192 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
197 193
198 194 if not startDate:
199 195 startDate = startDatetime.date()
200 196
201 197 if not endDate:
202 198 endDate = endDatatime.date()
203 199
204 200 dateList = []
205 201
206 202 thisDatetime = startDatetime
207 203
208 204 while(thisDatetime <= endDatatime):
209 205
210 206 thisDate = thisDatetime.date()
211 207
212 208 if thisDate < startDate:
213 209 continue
214 210
215 211 if thisDate > endDate:
216 212 break
217 213
218 214 dateList.append(thisDate)
219 215 thisDatetime += datetime.timedelta(1)
220 216
221 217 return dateList
222 218
223 219 def setup(self, path=None,
224 220 startDate=None,
225 221 endDate=None,
226 222 startTime=datetime.time(0, 0, 0),
227 223 endTime=datetime.time(23, 59, 59),
228 224 channelList=None,
229 225 nSamples=None,
230 226 online=False,
231 227 delay=60,
232 228 buffer_size=1024,
233 229 ippKm=None,
234 230 nCohInt=1,
235 231 nCode=1,
236 232 nBaud=1,
237 233 flagDecodeData=False,
238 234 code=numpy.ones((1, 1), dtype=numpy.int),
239 235 **kwargs):
240 236 '''
241 237 In this method we should set all initial parameters.
242 238
243 239 Inputs:
244 240 path
245 241 startDate
246 242 endDate
247 243 startTime
248 244 endTime
249 245 set
250 246 expLabel
251 247 ext
252 248 online
253 249 delay
254 250 '''
255 251 self.nCohInt = nCohInt
256 252 self.flagDecodeData = flagDecodeData
257 253 self.i = 0
258 254 if not os.path.isdir(path):
259 255 raise ValueError("[Reading] Directory %s does not exist" % path)
260 256
261 257 try:
262 258 self.digitalReadObj = digital_rf.DigitalRFReader(
263 259 path, load_all_metadata=True)
264 260 except:
265 261 self.digitalReadObj = digital_rf.DigitalRFReader(path)
266 262
267 263 channelNameList = self.digitalReadObj.get_channels()
268 264
269 265 if not channelNameList:
270 266 raise ValueError("[Reading] Directory %s does not have any files" % path)
271 267
272 268 if not channelList:
273 269 channelList = list(range(len(channelNameList)))
274 270
275 271 ########## Reading metadata ######################
276 272
277 273 top_properties = self.digitalReadObj.get_properties(
278 274 channelNameList[channelList[0]])
279 275
280 276 self.__num_subchannels = top_properties['num_subchannels']
281 277 self.__sample_rate = 1.0 * \
282 278 top_properties['sample_rate_numerator'] / \
283 279 top_properties['sample_rate_denominator']
284 280 # self.__samples_per_file = top_properties['samples_per_file'][0]
285 281 self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate # why 0.15?
286 282
287 283 this_metadata_file = self.digitalReadObj.get_digital_metadata(
288 284 channelNameList[channelList[0]])
289 285 metadata_bounds = this_metadata_file.get_bounds()
290 286 self.fixed_metadata_dict = this_metadata_file.read(
291 287 metadata_bounds[0])[metadata_bounds[0]] # GET FIRST HEADER
292 288
293 289 try:
294 290 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
295 291 self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
296 292 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
297 293 self.dtype = pickle.loads(self.fixed_metadata_dict['dtype'])
298 294 except:
299 295 pass
300 296
301 297 self.__frequency = None
302 298
303 299 self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
304 300
305 301 self.__timezone = self.fixed_metadata_dict.get('timezone', 300)
306 302
307 303 try:
308 304 nSamples = self.fixed_metadata_dict['nSamples']
309 305 except:
310 306 nSamples = None
311 307
312 308 self.__firstHeigth = 0
313 309
314 310 try:
315 311 codeType = self.__radarControllerHeader['codeType']
316 312 except:
317 313 codeType = 0
318 314
319 315 try:
320 316 if codeType:
321 317 nCode = self.__radarControllerHeader['nCode']
322 318 nBaud = self.__radarControllerHeader['nBaud']
323 319 code = self.__radarControllerHeader['code']
324 320 except:
325 321 pass
326 322
327 323 if not ippKm:
328 324 try:
329 325 # seconds to km
330 326 ippKm = self.__radarControllerHeader['ipp']
331 327 except:
332 328 ippKm = None
333 329 ####################################################
334 330 self.__ippKm = ippKm
335 331 startUTCSecond = None
336 332 endUTCSecond = None
337 333
338 334 if startDate:
339 335 startDatetime = datetime.datetime.combine(startDate, startTime)
340 336 startUTCSecond = (
341 337 startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
342 338
343 339 if endDate:
344 340 endDatetime = datetime.datetime.combine(endDate, endTime)
345 341 endUTCSecond = (endDatetime - datetime.datetime(1970,
346 342 1, 1)).total_seconds() + self.__timezone
347 343
348 344 start_index, end_index = self.digitalReadObj.get_bounds(
349 345 channelNameList[channelList[0]])
350 346
351 347 if not startUTCSecond:
352 348 startUTCSecond = start_index / self.__sample_rate
353 349
354 350 if start_index > startUTCSecond * self.__sample_rate:
355 351 startUTCSecond = start_index / self.__sample_rate
356 352
357 353 if not endUTCSecond:
358 354 endUTCSecond = end_index / self.__sample_rate
359 355
360 356 if end_index < endUTCSecond * self.__sample_rate:
361 357 endUTCSecond = end_index / self.__sample_rate
362 358 if not nSamples:
363 359 if not ippKm:
364 360 raise ValueError("[Reading] nSamples or ippKm should be defined")
365 361 nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
366 362 channelBoundList = []
367 363 channelNameListFiltered = []
368 364
369 365 for thisIndexChannel in channelList:
370 366 thisChannelName = channelNameList[thisIndexChannel]
371 367 start_index, end_index = self.digitalReadObj.get_bounds(
372 368 thisChannelName)
373 369 channelBoundList.append((start_index, end_index))
374 370 channelNameListFiltered.append(thisChannelName)
375 371
376 372 self.profileIndex = 0
377 373 self.i = 0
378 374 self.__delay = delay
379 375
380 376 self.__codeType = codeType
381 377 self.__nCode = nCode
382 378 self.__nBaud = nBaud
383 379 self.__code = code
384 380
385 381 self.__datapath = path
386 382 self.__online = online
387 383 self.__channelList = channelList
388 384 self.__channelNameList = channelNameListFiltered
389 385 self.__channelBoundList = channelBoundList
390 386 self.__nSamples = nSamples
391 387 self.__samples_to_read = int(nSamples) # FIJO: AHORA 40
392 388 self.__nChannels = len(self.__channelList)
393 389
394 390 self.__startUTCSecond = startUTCSecond
395 391 self.__endUTCSecond = endUTCSecond
396 392
397 393 self.__timeInterval = 1.0 * self.__samples_to_read / \
398 394 self.__sample_rate # Time interval
399 395
400 396 if online:
401 397 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
402 398 startUTCSecond = numpy.floor(endUTCSecond)
403 399
404 400 # por que en el otro metodo lo primero q se hace es sumar samplestoread
405 401 self.__thisUnixSample = int(
406 402 startUTCSecond * self.__sample_rate) - self.__samples_to_read
407 403
408 404 self.__data_buffer = numpy.zeros(
409 405 (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
410 406
411 407 self.__setFileHeader()
412 408 self.isConfig = True
413 409
414 410 print("[Reading] Digital RF Data was found from %s to %s " % (
415 411 datetime.datetime.utcfromtimestamp(
416 412 self.__startUTCSecond - self.__timezone),
417 413 datetime.datetime.utcfromtimestamp(
418 414 self.__endUTCSecond - self.__timezone)
419 415 ))
420 416
421 417 print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
422 418 datetime.datetime.utcfromtimestamp(
423 419 endUTCSecond - self.__timezone)
424 420 ))
425 421 self.oldAverage = None
426 422 self.count = 0
427 423 self.executionTime = 0
428 424
429 425 def __reload(self):
430 426 # print
431 427 # print "%s not in range [%s, %s]" %(
432 428 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
433 429 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
434 430 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
435 431 # )
436 432 print("[Reading] reloading metadata ...")
437 433
438 434 try:
439 435 self.digitalReadObj.reload(complete_update=True)
440 436 except:
441 437 self.digitalReadObj.reload()
442 438
443 439 start_index, end_index = self.digitalReadObj.get_bounds(
444 440 self.__channelNameList[self.__channelList[0]])
445 441
446 442 if start_index > self.__startUTCSecond * self.__sample_rate:
447 443 self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
448 444
449 445 if end_index > self.__endUTCSecond * self.__sample_rate:
450 446 self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
451 447 print()
452 448 print("[Reading] New timerange found [%s, %s] " % (
453 449 datetime.datetime.utcfromtimestamp(
454 450 self.__startUTCSecond - self.__timezone),
455 451 datetime.datetime.utcfromtimestamp(
456 452 self.__endUTCSecond - self.__timezone)
457 453 ))
458 454
459 455 return True
460 456
461 457 return False
462 458
463 459 def timeit(self, toExecute):
464 t0 = time()
460 t0 = time.time()
465 461 toExecute()
466 self.executionTime = time() - t0
462 self.executionTime = time.time() - t0
467 463 if self.oldAverage is None:
468 464 self.oldAverage = self.executionTime
469 465 self.oldAverage = (self.executionTime + self.count *
470 466 self.oldAverage) / (self.count + 1.0)
471 467 self.count = self.count + 1.0
472 468 return
473 469
474 470 def __readNextBlock(self, seconds=30, volt_scale=1):
475 471 '''
476 472 '''
477 473
478 474 # Set the next data
479 475 self.__flagDiscontinuousBlock = False
480 476 self.__thisUnixSample += self.__samples_to_read
481 477
482 478 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
483 479 print("[Reading] There are no more data into selected time-range")
484 480 if self.__online:
485 481 self.__reload()
486 482 else:
487 483 return False
488 484
489 485 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
490 486 return False
491 487 self.__thisUnixSample -= self.__samples_to_read
492 488
493 489 indexChannel = 0
494 490
495 491 dataOk = False
496 492 for thisChannelName in self.__channelNameList: # TODO VARIOS CHANNELS?
497 493 for indexSubchannel in range(self.__num_subchannels):
498 494 try:
499 495 t0 = time()
500 496 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
501 497 self.__samples_to_read,
502 498 thisChannelName, sub_channel=indexSubchannel)
503 499 self.executionTime = time() - t0
504 500 if self.oldAverage is None:
505 501 self.oldAverage = self.executionTime
506 502 self.oldAverage = (
507 503 self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
508 504 self.count = self.count + 1.0
509 505
510 506 except IOError as e:
511 507 # read next profile
512 508 self.__flagDiscontinuousBlock = True
513 509 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
514 510 break
515 511
516 512 if result.shape[0] != self.__samples_to_read:
517 513 self.__flagDiscontinuousBlock = True
518 514 print("[Reading] %s: Too few samples were found, just %d/%d samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
519 515 result.shape[0],
520 516 self.__samples_to_read))
521 517 break
522 518
523 519 self.__data_buffer[indexSubchannel, :] = result * volt_scale
524 520
525 521 indexChannel += 1
526 522
527 523 dataOk = True
528 524
529 525 self.__utctime = self.__thisUnixSample / self.__sample_rate
530 526
531 527 if not dataOk:
532 528 return False
533 529
534 530 print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
535 531 self.__samples_to_read,
536 532 self.__timeInterval))
537 533
538 534 self.__bufferIndex = 0
539 535
540 536 return True
541 537
542 538 def __isBufferEmpty(self):
543 539 return self.__bufferIndex > self.__samples_to_read - self.__nSamples # 40960 - 40
544 540
545 541 def getData(self, seconds=30, nTries=5):
546 542 '''
547 543 This method gets the data from files and put the data into the dataOut object
548 544
549 545 In addition, increase el the buffer counter in one.
550 546
551 547 Return:
552 548 data : retorna un perfil de voltages (alturas * canales) copiados desde el
553 549 buffer. Si no hay mas archivos a leer retorna None.
554 550
555 551 Affected:
556 552 self.dataOut
557 553 self.profileIndex
558 554 self.flagDiscontinuousBlock
559 555 self.flagIsNewBlock
560 556 '''
561 557
562 558 err_counter = 0
563 559 self.dataOut.flagNoData = True
564 560
565 561 if self.__isBufferEmpty():
566 562 self.__flagDiscontinuousBlock = False
567 563
568 564 while True:
569 565 if self.__readNextBlock():
570 566 break
571 567 if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
572 self.dataOut.error = 'Error'
568 raise schainpy.admin.SchainError('Error')
573 569 return
574 570
575 571 if self.__flagDiscontinuousBlock:
576 self.dataOut.error = 'discontinuous block found'
572 raise schainpy.admin.SchainError('discontinuous block found')
577 573 return
578 574
579 575 if not self.__online:
580 self.dataOut.error = 'Online?'
576 raise schainpy.admin.SchainError('Online?')
581 577 return
582 578
583 579 err_counter += 1
584 580 if err_counter > nTries:
585 self.dataOut.error = 'Max retrys reach'
581 raise schainpy.admin.SchainError('Max retrys reach')
586 582 return
587 583
588 584 print('[Reading] waiting %d seconds to read a new block' % seconds)
589 sleep(seconds)
585 time.sleep(seconds)
590 586
591 587 self.dataOut.data = self.__data_buffer[:,
592 588 self.__bufferIndex:self.__bufferIndex + self.__nSamples]
593 589 self.dataOut.utctime = (
594 590 self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
595 591 self.dataOut.flagNoData = False
596 592 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
597 593 self.dataOut.profileIndex = self.profileIndex
598 594
599 595 self.__bufferIndex += self.__nSamples
600 596 self.profileIndex += 1
601 597
602 598 if self.profileIndex == self.dataOut.nProfiles:
603 599 self.profileIndex = 0
604 600
605 601 return
606 602
607 603 def printInfo(self):
608 604 '''
609 605 '''
610 606 if self.__printInfo == False:
611 607 return
612 608
613 609 # self.systemHeaderObj.printInfo()
614 610 # self.radarControllerHeaderObj.printInfo()
615 611
616 612 self.__printInfo = False
617 613
618 614 def printNumberOfBlock(self):
619 615 '''
620 616 '''
621 617 return
622 618 # print self.profileIndex
623 619
624 620 def run(self, **kwargs):
625 621 '''
626 622 This method will be called many times so here you should put all your code
627 623 '''
628 624
629 625 if not self.isConfig:
630 626 self.setup(**kwargs)
631 627 #self.i = self.i+1
632 628 self.getData(seconds=self.__delay)
633 629
634 630 return
635 631
636 632
637 633 class DigitalRFWriter(Operation):
638 634 '''
639 635 classdocs
640 636 '''
641 637
642 638 def __init__(self, **kwargs):
643 639 '''
644 640 Constructor
645 641 '''
646 642 Operation.__init__(self, **kwargs)
647 643 self.metadata_dict = {}
648 644 self.dataOut = None
649 645 self.dtype = None
650 646 self.oldAverage = 0
651 647
652 648 def setHeader(self):
653 649
654 650 self.metadata_dict['frequency'] = self.dataOut.frequency
655 651 self.metadata_dict['timezone'] = self.dataOut.timeZone
656 652 self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype)
657 653 self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
658 654 self.metadata_dict['heightList'] = self.dataOut.heightList
659 655 self.metadata_dict['channelList'] = self.dataOut.channelList
660 656 self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
661 657 self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
662 658 self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
663 659 self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
664 660 self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
665 661 self.metadata_dict['type'] = self.dataOut.type
666 662 self.metadata_dict['flagDataAsBlock'] = getattr(
667 663 self.dataOut, 'flagDataAsBlock', None) # chequear
668 664
669 665 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
670 666 '''
671 667 In this method we should set all initial parameters.
672 668 Input:
673 669 dataOut: Input data will also be outputa data
674 670 '''
675 671 self.setHeader()
676 672 self.__ippSeconds = dataOut.ippSeconds
677 673 self.__deltaH = dataOut.getDeltaH()
678 674 self.__sample_rate = 1e6 * 0.15 / self.__deltaH
679 675 self.__dtype = dataOut.dtype
680 676 if len(dataOut.dtype) == 2:
681 677 self.__dtype = dataOut.dtype[0]
682 678 self.__nSamples = dataOut.systemHeaderObj.nSamples
683 679 self.__nProfiles = dataOut.nProfiles
684 680
685 681 if self.dataOut.type != 'Voltage':
686 682 raise 'Digital RF cannot be used with this data type'
687 683 self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
688 684 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
689 685 else:
690 686 self.arr_data = numpy.ones((self.__nSamples, len(
691 687 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
692 688
693 689 file_cadence_millisecs = 1000
694 690
695 691 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
696 692 sample_rate_numerator = int(sample_rate_fraction.numerator)
697 693 sample_rate_denominator = int(sample_rate_fraction.denominator)
698 694 start_global_index = dataOut.utctime * self.__sample_rate
699 695
700 696 uuid = 'prueba'
701 697 compression_level = 0
702 698 checksum = False
703 699 is_complex = True
704 700 num_subchannels = len(dataOut.channelList)
705 701 is_continuous = True
706 702 marching_periods = False
707 703
708 704 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
709 705 fileCadence, start_global_index,
710 706 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
711 707 is_complex, num_subchannels, is_continuous, marching_periods)
712 708 metadata_dir = os.path.join(path, 'metadata')
713 709 os.system('mkdir %s' % (metadata_dir))
714 710 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, # 236, file_cadence_millisecs / 1000
715 711 sample_rate_numerator, sample_rate_denominator,
716 712 metadataFile)
717 713 self.isConfig = True
718 714 self.currentSample = 0
719 715 self.oldAverage = 0
720 716 self.count = 0
721 717 return
722 718
723 719 def writeMetadata(self):
724 720 start_idx = self.__sample_rate * self.dataOut.utctime
725 721
726 722 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict(
727 723 )
728 724 self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
729 725 )
730 726 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict(
731 727 )
732 728 self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
733 729 return
734 730
735 731 def timeit(self, toExecute):
736 732 t0 = time()
737 733 toExecute()
738 734 self.executionTime = time() - t0
739 735 if self.oldAverage is None:
740 736 self.oldAverage = self.executionTime
741 737 self.oldAverage = (self.executionTime + self.count *
742 738 self.oldAverage) / (self.count + 1.0)
743 739 self.count = self.count + 1.0
744 740 return
745 741
746 742 def writeData(self):
747 743 if self.dataOut.type != 'Voltage':
748 744 raise 'Digital RF cannot be used with this data type'
749 745 for channel in self.dataOut.channelList:
750 746 for i in range(self.dataOut.nFFTPoints):
751 747 self.arr_data[1][channel * self.dataOut.nFFTPoints +
752 748 i]['r'] = self.dataOut.data[channel][i].real
753 749 self.arr_data[1][channel * self.dataOut.nFFTPoints +
754 750 i]['i'] = self.dataOut.data[channel][i].imag
755 751 else:
756 752 for i in range(self.dataOut.systemHeaderObj.nSamples):
757 753 for channel in self.dataOut.channelList:
758 754 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
759 755 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
760 756
761 757 def f(): return self.digitalWriteObj.rf_write(self.arr_data)
762 758 self.timeit(f)
763 759
764 760 return
765 761
766 762 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
767 763 '''
768 764 This method will be called many times so here you should put all your code
769 765 Inputs:
770 766 dataOut: object with the data
771 767 '''
772 768 # print dataOut.__dict__
773 769 self.dataOut = dataOut
774 770 if not self.isConfig:
775 771 self.setup(dataOut, path, frequency, fileCadence,
776 772 dirCadence, metadataCadence, **kwargs)
777 773 self.writeMetadata()
778 774
779 775 self.writeData()
780 776
781 777 ## self.currentSample += 1
782 778 # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
783 779 # self.writeMetadata()
784 780 ## if self.currentSample == self.__nProfiles: self.currentSample = 0
785 781
786 782 return dataOut
787 783
788 784 def close(self):
789 785 print('[Writing] - Closing files ')
790 786 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
791 787 try:
792 788 self.digitalWriteObj.close()
793 789 except:
794 790 pass
795 791
796 792
Show file before | Show file after | Hide comments
@@ -1,863 +1,862
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-com0419
5 5 '''
6 6
7 7 import os,sys
8 8 import time,datetime
9 9 import h5py
10 10 import numpy
11 11 import fnmatch
12 12 import re
13 13
14 14 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 15 from schainpy.model.data.jrodata import Voltage
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
17 17
18 18
19 19 def isNumber(str):
20 20 """
21 21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22 22
23 23 Excepciones:
24 24 Si un determinado string no puede ser convertido a numero
25 25 Input:
26 26 str, string al cual se le analiza para determinar si convertible a un numero o no
27 27
28 28 Return:
29 29 True : si el string es uno numerico
30 30 False : no es un string numerico
31 31 """
32 32 try:
33 33 float( str )
34 34 return True
35 35 except:
36 36 return False
37 37
38 38 def getFileFromSet(path, ext, set=None):
39 39 validFilelist = []
40 40 fileList = os.listdir(path)
41 41
42 42
43 43 if len(fileList) < 1:
44 44 return None
45 45
46 46 # 0 1234 567 89A BCDE
47 47 # H YYYY DDD SSS .ext
48 48
49 49 for thisFile in fileList:
50 50 try:
51 51 number= int(thisFile[6:16])
52 52
53 53 # year = int(thisFile[1:5])
54 54 # doy = int(thisFile[5:8])
55 55 except:
56 56 continue
57 57
58 58 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
59 59 continue
60 60
61 61 validFilelist.append(thisFile)
62 62
63 63 if len(validFilelist) < 1:
64 64 return None
65 65
66 66 validFilelist = sorted( validFilelist, key=str.lower )
67 67
68 68 if set == None:
69 69 return validFilelist[-1]
70 70
71 71 print("set =" ,set)
72 72 for thisFile in validFilelist:
73 73 if set <= int(thisFile[6:16]):
74 74 print(thisFile,int(thisFile[6:16]))
75 75 return thisFile
76 76
77 77 return validFilelist[-1]
78 78
79 79 myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
80 80 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
81 81
82 82 if len(myfile)!= 0:
83 83 return myfile[0]
84 84 else:
85 85 filename = '*%10.10d%s'%(set,ext.lower())
86 86 print('the filename %s does not exist'%filename)
87 87 print('...going to the last file: ')
88 88
89 89 if validFilelist:
90 90 validFilelist = sorted( validFilelist, key=str.lower )
91 91 return validFilelist[-1]
92 92
93 93 return None
94 94
95 95 def getlastFileFromPath(path, ext):
96 96 """
97 97 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
98 98 al final de la depuracion devuelve el ultimo file de la lista que quedo.
99 99
100 100 Input:
101 101 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
102 102 ext : extension de los files contenidos en una carpeta
103 103
104 104 Return:
105 105 El ultimo file de una determinada carpeta, no se considera el path.
106 106 """
107 107 validFilelist = []
108 108 fileList = os.listdir(path)
109 109
110 110 # 0 1234 567 89A BCDE
111 111 # H YYYY DDD SSS .ext
112 112
113 113 for thisFile in fileList:
114 114
115 115 try:
116 116 number= int(thisFile[6:16])
117 117 except:
118 118 print("There is a file or folder with different format")
119 119 if not isNumber(number):
120 120 continue
121 121
122 122 # year = thisFile[1:5]
123 123 # if not isNumber(year):
124 124 # continue
125 125
126 126 # doy = thisFile[5:8]
127 127 # if not isNumber(doy):
128 128 # continue
129 129
130 130 number= int(number)
131 131 # year = int(year)
132 132 # doy = int(doy)
133 133
134 134 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
135 135 continue
136 136
137 137
138 138 validFilelist.append(thisFile)
139 139
140 140
141 141 if validFilelist:
142 142 validFilelist = sorted( validFilelist, key=str.lower )
143 143 return validFilelist[-1]
144 144
145 145 return None
146 146
147 147
148 148
149 149 class HFReader(ProcessingUnit):
150 150 '''
151 151 classdocs
152 152 '''
153 153 path = None
154 154 startDate= None
155 155 endDate = None
156 156 startTime= None
157 157 endTime = None
158 158 walk = None
159 159 isConfig = False
160 160 dataOut=None
161 161 nTries = 3
162 162 ext = ".hdf5"
163 163
164 164 def __init__(self, **kwargs):
165 165 '''
166 166 Constructor
167 167 '''
168 168 ProcessingUnit.__init__(self, **kwargs)
169 169
170 170 self.isConfig =False
171 171
172 172 self.datablock = None
173 173
174 174 self.filename_current=None
175 175
176 176 self.utc = 0
177 177
178 178 self.ext='.hdf5'
179 179
180 180 self.flagIsNewFile = 1
181 181
182 182 #-------------------------------------------------
183 183 self.fileIndex=None
184 184
185 185 self.profileIndex_offset=None
186 186
187 187 self.filenameList=[]
188 188
189 189 self.hfFilePointer= None
190 190
191 191 self.filename_online = None
192 192
193 193 self.status=True
194 194
195 195 self.flagNoMoreFiles= False
196 196
197 197 self.__waitForNewFile = 20
198 198
199 199
200 200 #--------------------------------------------------
201 201
202 202 self.dataOut = self.createObjByDefault()
203 203
204 204
205 205 def createObjByDefault(self):
206 206
207 207 dataObj = Voltage()
208 208
209 209 return dataObj
210 210
211 211 def setObjProperties(self):
212 212
213 213 pass
214 214
215 215 def getBlockDimension(self):
216 216 """
217 217 Obtiene la cantidad de puntos a leer por cada bloque de datos
218 218
219 219 Affected:
220 220 self.blocksize
221 221
222 222 Return:
223 223 None
224 224 """
225 225 pts2read =self.nChannels*self.nHeights*self.nProfiles
226 226 self.blocksize = pts2read
227 227
228 228 def __readHeader(self):
229 229
230 230 self.nProfiles = 100
231 231 self.nHeights = 1000
232 232 self.nChannels = 2
233 233 self.__firstHeigth=0
234 234 self.__nSamples=1000
235 235 self.__deltaHeigth=1.5
236 236 self.__sample_rate=1e5
237 237 #self.__frequency=2.72e6
238 238 #self.__frequency=3.64e6
239 239 self.__frequency=None
240 240 self.__online = False
241 241 self.filename_next_set=None
242 242
243 243 #print "Frequency of Operation:", self.__frequency
244 244
245 245
246 246 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
247 247 self.path = path
248 248 self.startDate = startDate
249 249 self.endDate = endDate
250 250 self.startTime = startTime
251 251 self.endTime = endTime
252 252 self.walk = walk
253 253
254 254 def __checkPath(self):
255 255 if os.path.exists(self.path):
256 256 self.status=1
257 257 else:
258 258 self.status=0
259 259 print('Path %s does not exits'%self.path)
260 260 return
261 261 return
262 262
263 263 def __selDates(self, hf_dirname_format):
264 264 try:
265 265 dir_hf_filename= self.path+"/"+hf_dirname_format
266 266 fp= h5py.File(dir_hf_filename,'r')
267 267 hipoc=fp['t'].value
268 268 fp.close()
269 269 date_time=datetime.datetime.utcfromtimestamp(hipoc)
270 270 year =int(date_time[0:4])
271 271 month=int(date_time[5:7])
272 272 dom =int(date_time[8:10])
273 273 thisDate= datetime.date(year,month,dom)
274 274 if (thisDate>=self.startDate and thisDate <= self.endDate):
275 275 return hf_dirname_format
276 276 except:
277 277 return None
278 278
279 279 def __findDataForDates(self,online=False):
280 280 if not(self.status):
281 281 return None
282 282
283 283 pat = '\d+.\d+'
284 284 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
285 285 dirnameList = [x for x in dirnameList if x!=None]
286 286 dirnameList = [x.string for x in dirnameList]
287 287 if not(online):
288 288
289 289 dirnameList = [self.__selDates(x) for x in dirnameList]
290 290 dirnameList = [x for x in dirnameList if x!=None]
291 291
292 292 if len(dirnameList)>0:
293 293 self.status = 1
294 294 self.dirnameList = dirnameList
295 295 self.dirnameList.sort()
296 296
297 297 else:
298 298 self.status = 0
299 299 return None
300 300
301 301 def __getTimeFromData(self):
302 302 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
303 303 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
304 304 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
305 305 print('........................................')
306 306 filter_filenameList=[]
307 307 self.filenameList.sort()
308 308 for i in range(len(self.filenameList)-1):
309 309 filename=self.filenameList[i]
310 310 dir_hf_filename= filename
311 311 fp= h5py.File(dir_hf_filename,'r')
312 312 hipoc=fp['t'].value
313 313 hipoc=hipoc+self.timezone
314 314 date_time=datetime.datetime.utcfromtimestamp(hipoc)
315 315 fp.close()
316 316 year =int(date_time[0:4])
317 317 month=int(date_time[5:7])
318 318 dom =int(date_time[8:10])
319 319 hour =int(date_time[11:13])
320 320 min =int(date_time[14:16])
321 321 sec =int(date_time[17:19])
322 322 this_time=datetime.datetime(year,month,dom,hour,min,sec)
323 323 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
324 324 filter_filenameList.append(filename)
325 325 filter_filenameList.sort()
326 326 self.filenameList = filter_filenameList
327 327 return 1
328 328
329 329 def __getFilenameList(self):
330 330 #print "hola"
331 331 #print self.dirnameList
332 332 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
333 333 self.filenameList= dirList
334 334 #print self.filenameList
335 335 #print "pase",len(self.filenameList)
336 336
337 337 def __selectDataForTimes(self, online=False):
338 338
339 339 if not(self.status):
340 340 return None
341 341 #----------------
342 342 self.__getFilenameList()
343 343 #----------------
344 344 if not(online):
345 345 if not(self.all):
346 346 self.__getTimeFromData()
347 347 if len(self.filenameList)>0:
348 348 self.status=1
349 349 self.filenameList.sort()
350 350 else:
351 351 self.status=0
352 352 return None
353 353 else:
354 354 if self.set != None:
355 355
356 356 filename=getFileFromSet(self.path,self.ext,self.set)
357 357
358 358 if self.flag_nextfile==True:
359 359 self.dirnameList=[filename]
360 360 fullfilename=self.path+"/"+filename
361 361 self.filenameList=[fullfilename]
362 362 self.filename_next_set=int(filename[6:16])+10
363 363
364 364 self.flag_nextfile=False
365 365 else:
366 366 print(filename)
367 367 print("PRIMERA CONDICION")
368 368 #if self.filename_next_set== int(filename[6:16]):
369 369 print("TODO BIEN")
370 370
371 371 if filename == None:
372 372 raise ValueError("corregir")
373 373
374 374 self.dirnameList=[filename]
375 375 fullfilename=self.path+"/"+filename
376 376 self.filenameList=[fullfilename]
377 377 self.filename_next_set=int(filename[6:16])+10
378 378 print("Setting next file",self.filename_next_set)
379 379 self.set=int(filename[6:16])
380 380 if True:
381 381 pass
382 382 else:
383 383 print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
384 384
385 385 else:
386 386 filename =getlastFileFromPath(self.path,self.ext)
387 387
388 388 if self.flag_nextfile==True:
389 389 self.dirnameList=[filename]
390 390 fullfilename=self.path+"/"+filename
391 391 self.filenameList=[self.filenameList[-1]]
392 392 self.filename_next_set=int(filename[6:16])+10
393 393
394 394 self.flag_nextfile=False
395 395 else:
396 396 filename=getFileFromSet(self.path,self.ext,self.set)
397 397 print(filename)
398 398 print("PRIMERA CONDICION")
399 399 #if self.filename_next_set== int(filename[6:16]):
400 400 print("TODO BIEN")
401 401
402 402 if filename == None:
403 403 raise ValueError("corregir")
404 404
405 405 self.dirnameList=[filename]
406 406 fullfilename=self.path+"/"+filename
407 407 self.filenameList=[fullfilename]
408 408 self.filename_next_set=int(filename[6:16])+10
409 409 print("Setting next file",self.filename_next_set)
410 410 self.set=int(filename[6:16])
411 411 if True:
412 412 pass
413 413 else:
414 414 print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
415 415
416 416
417 417
418 418 def searchFilesOffLine(self,
419 419 path,
420 420 startDate,
421 421 endDate,
422 422 ext,
423 423 startTime=datetime.time(0,0,0),
424 424 endTime=datetime.time(23,59,59),
425 425 walk=True):
426 426
427 427 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
428 428
429 429 self.__checkPath()
430 430
431 431 self.__findDataForDates()
432 432 #print self.dirnameList
433 433
434 434 self.__selectDataForTimes()
435 435
436 436 for i in range(len(self.filenameList)):
437 437 print("%s"% (self.filenameList[i]))
438 438
439 439 return
440 440
441 441 def searchFilesOnLine(self,
442 442 path,
443 443 expLabel= "",
444 444 ext=None,
445 445 startDate=None,
446 446 endDate=None,
447 447 walk=True,
448 448 set=None):
449 449
450 450
451 451 startDate = datetime.datetime.utcnow().date()
452 452 endDate = datetime.datetime.utcnow().date()
453 453
454 454 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
455 455
456 456 self.__checkPath()
457 457
458 458 fullpath=path
459 459 print("%s folder was found: " %(fullpath ))
460 460
461 461 if set == None:
462 462 self.set=None
463 463 filename =getlastFileFromPath(fullpath,ext)
464 464 startDate= datetime.datetime.utcnow().date
465 465 endDate= datetime.datetime.utcnow().date()
466 466 #
467 467 else:
468 468 filename= getFileFromSet(fullpath,ext,set)
469 469 startDate=None
470 470 endDate=None
471 471 #
472 472 if not (filename):
473 473 return None,None,None,None,None
474 474 #print "%s file was found" %(filename)
475 475
476 476 #
477 477 # dir_hf_filename= self.path+"/"+filename
478 478 # fp= h5py.File(dir_hf_filename,'r')
479 479 # hipoc=fp['t'].value
480 480 # fp.close()
481 481 # date_time=datetime.datetime.utcfromtimestamp(hipoc)
482 482 #
483 483 # year =int(date_time[0:4])
484 484 # month=int(date_time[5:7])
485 485 # dom =int(date_time[8:10])
486 486 # set= int(filename[4:10])
487 487 # self.set=set-1
488 488 #self.dirnameList=[filename]
489 489 filenameList= fullpath+"/"+filename
490 490 self.dirnameList=[filename]
491 491 self.filenameList=[filenameList]
492 492 self.flag_nextfile=True
493 493
494 494 #self.__findDataForDates(online=True)
495 495 #self.dirnameList=[self.dirnameList[-1]]
496 496 #print self.dirnameList
497 497 #self.__selectDataForTimes(online=True)
498 498 #return fullpath,filename,year,month,dom,set
499 499 return
500 500
501 501 def __setNextFile(self,online=False):
502 502 """
503 503 """
504 504 if not(online):
505 505 newFile = self.__setNextFileOffline()
506 506 else:
507 507 newFile = self.__setNextFileOnline()
508 508
509 509 if not(newFile):
510 510 return 0
511 511 return 1
512 512
513 513 def __setNextFileOffline(self):
514 514 """
515 515 """
516 516 idFile= self.fileIndex
517 517 while(True):
518 518 idFile += 1
519 519 if not (idFile < len(self.filenameList)):
520 520 self.flagNoMoreFiles = 1
521 521 print("No more Files")
522 522 return 0
523 523 filename = self.filenameList[idFile]
524 524 hfFilePointer =h5py.File(filename,'r')
525 525
526 526 epoc=hfFilePointer['t'].value
527 527 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
528 528 break
529 529
530 530 self.flagIsNewFile = 1
531 531 self.fileIndex = idFile
532 532 self.filename = filename
533 533
534 534 self.hfFilePointer = hfFilePointer
535 535 hfFilePointer.close()
536 536 self.__t0=epoc
537 537 print("Setting the file: %s"%self.filename)
538 538
539 539 return 1
540 540
541 541 def __setNextFileOnline(self):
542 542 """
543 543 """
544 544 print("SOY NONE",self.set)
545 545 if self.set==None:
546 546 pass
547 547 else:
548 548 self.set +=10
549 549
550 550 filename = self.filenameList[0]#fullfilename
551 551 if self.filename_online != None:
552 552 self.__selectDataForTimes(online=True)
553 553 filename = self.filenameList[0]
554 554 while self.filename_online == filename:
555 555 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
556 556 time.sleep(self.__waitForNewFile)
557 557 #self.__findDataForDates(online=True)
558 558 self.set=self.filename_next_set
559 559 self.__selectDataForTimes(online=True)
560 560 filename = self.filenameList[0]
561 561 sizeoffile=os.path.getsize(filename)
562 562
563 563 #print filename
564 564 sizeoffile=os.path.getsize(filename)
565 565 if sizeoffile<1670240:
566 566 print("%s is not the rigth size"%filename)
567 567 delay=50
568 568 print('waiting %d seconds for delay...'%(delay))
569 569 time.sleep(delay)
570 570 sizeoffile=os.path.getsize(filename)
571 571 if sizeoffile<1670240:
572 572 delay=50
573 573 print('waiting %d more seconds for delay...'%(delay))
574 574 time.sleep(delay)
575 575
576 576 sizeoffile=os.path.getsize(filename)
577 577 if sizeoffile<1670240:
578 578 delay=50
579 579 print('waiting %d more seconds for delay...'%(delay))
580 580 time.sleep(delay)
581 581
582 582 try:
583 583 hfFilePointer=h5py.File(filename,'r')
584 584
585 585 except:
586 586 print("Error reading file %s"%filename)
587 587
588 588 self.filename_online=filename
589 589 epoc=hfFilePointer['t'].value
590 590
591 591 self.hfFilePointer=hfFilePointer
592 592 hfFilePointer.close()
593 593 self.__t0=epoc
594 594
595 595
596 596 self.flagIsNewFile = 1
597 597 self.filename = filename
598 598
599 599 print("Setting the file: %s"%self.filename)
600 600 return 1
601 601
602 602 def __getExpParameters(self):
603 603 if not(self.status):
604 604 return None
605 605
606 606 def setup(self,
607 607 path = None,
608 608 startDate = None,
609 609 endDate = None,
610 610 startTime = datetime.time(0,0,0),
611 611 endTime = datetime.time(23,59,59),
612 612 set = None,
613 613 expLabel = "",
614 614 ext = None,
615 615 all=0,
616 616 timezone=0,
617 617 online = False,
618 618 delay = 60,
619 619 walk = True):
620 620 '''
621 621 In this method we should set all initial parameters.
622 622
623 623 '''
624 624 if path==None:
625 625 raise ValueError("The path is not valid")
626 626
627 627 if ext==None:
628 628 ext = self.ext
629 629
630 630 self.timezone= timezone
631 631 self.online= online
632 632 self.all=all
633 633 #if set==None:
634 634
635 635 #print set
636 636 if not(online):
637 637 print("Searching files in offline mode...")
638 638
639 639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 640 else:
641 641 print("Searching files in online mode...")
642 642 self.searchFilesOnLine(path, walk,ext,set=set)
643 643 if set==None:
644 644 pass
645 645 else:
646 646 self.set=set-10
647 647
648 648 # for nTries in range(self.nTries):
649 649 #
650 650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 651 #
652 652 # if fullpath:
653 653 # break
654 654 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
655 655 # time.sleep(self.delay)
656 656 # if not(fullpath):
657 657 # print "There ins't valid files in %s" % path
658 658 # return None
659 659
660 660
661 661 if not(self.filenameList):
662 662 print("There is no files into the folder: %s"%(path))
663 663 sys.exit(-1)
664 664
665 665 self.__getExpParameters()
666 666
667 667
668 668 self.fileIndex = -1
669 669
670 670 self.__setNextFile(online)
671 671
672 672 self.__readMetadata()
673 673
674 674 self.__setLocalVariables()
675 675
676 676 self.__setHeaderDO()
677 677 #self.profileIndex_offset= 0
678 678
679 679 #self.profileIndex = self.profileIndex_offset
680 680
681 681 self.isConfig = True
682 682
683 683 def __readMetadata(self):
684 684 self.__readHeader()
685 685
686 686
687 687 def __setLocalVariables(self):
688 688
689 689 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
690 690 #
691 691
692 692
693 693
694 694 self.profileIndex = 9999
695 695
696 696
697 697 def __setHeaderDO(self):
698 698
699 699
700 700 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
701 701
702 702 self.dataOut.systemHeaderObj = SystemHeader()
703 703
704 704
705 705 #---------------------------------------------------------
706 706 self.dataOut.systemHeaderObj.nProfiles=100
707 707 self.dataOut.systemHeaderObj.nSamples=1000
708 708
709 709
710 710 SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
711 711 self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
712 712 self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
713 713 self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
714 714 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
715 715 self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
716 716 self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
717 717 self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
718 718 self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
719 719
720 720 self.dataOut.radarControllerHeaderObj.nWindows=1
721 721 self.dataOut.radarControllerHeaderObj.codetype=0
722 722 self.dataOut.radarControllerHeaderObj.numTaus=0
723 723 #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
724 724
725 725
726 726 #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
727 727 #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
728 728 #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
729 729
730 730 self.dataOut.radarControllerHeaderObj.code_size=0
731 731 self.dataOut.nBaud=0
732 732 self.dataOut.nCode=0
733 733 self.dataOut.nPairs=0
734 734
735 735
736 736 #---------------------------------------------------------
737 737
738 738 self.dataOut.type = "Voltage"
739 739
740 740 self.dataOut.data = None
741 741
742 742 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
743 743
744 744 self.dataOut.nProfiles = 1
745 745
746 746 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
747 747
748 748 self.dataOut.channelList = list(range(self.nChannels))
749 749
750 750 #self.dataOut.channelIndexList = None
751 751
752 752 self.dataOut.flagNoData = True
753 753
754 754 #Set to TRUE if the data is discontinuous
755 755 self.dataOut.flagDiscontinuousBlock = False
756 756
757 757 self.dataOut.utctime = None
758 758
759 759 self.dataOut.timeZone = self.timezone
760 760
761 761 self.dataOut.dstFlag = 0
762 762
763 763 self.dataOut.errorCount = 0
764 764
765 765 self.dataOut.nCohInt = 1
766 766
767 767 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
768 768
769 769 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
770 770
771 771 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
772 772
773 773 self.dataOut.flagShiftFFT = False
774 774
775 775 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
776 776
777 777 #Time interval between profiles
778 778 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
779 779
780 780
781 781 self.dataOut.frequency = self.__frequency
782 782
783 783 self.dataOut.realtime = self.__online
784 784
785 785 def __hasNotDataInBuffer(self):
786 786
787 787 if self.profileIndex >= self.nProfiles:
788 788 return 1
789 789
790 790 return 0
791 791
792 792 def readNextBlock(self):
793 793 if not(self.__setNewBlock()):
794 794 return 0
795 795
796 796 if not(self.readBlock()):
797 797 return 0
798 798
799 799 return 1
800 800
801 801 def __setNewBlock(self):
802 802
803 803 if self.hfFilePointer==None:
804 804 return 0
805 805
806 806 if self.flagIsNewFile:
807 807 return 1
808 808
809 809 if self.profileIndex < self.nProfiles:
810 810 return 1
811 811
812 812 self.__setNextFile(self.online)
813 813
814 814 return 1
815 815
816 816
817 817
818 818 def readBlock(self):
819 819 fp=h5py.File(self.filename,'r')
820 820 #Puntero que apunta al archivo hdf5
821 821 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
822 822 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
823 823 fp.close()
824 824 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
825 825 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
826 826 self.datablock = numpy.array([ch0,ch1])
827 827 self.flagIsNewFile=0
828 828
829 829 self.profileIndex=0
830 830
831 831 return 1
832 832
833 833 def getData(self):
834 834 if self.flagNoMoreFiles:
835 835 self.dataOut.flagNoData = True
836 print('Process finished')
837 836 return 0
838 837
839 838 if self.__hasNotDataInBuffer():
840 839 if not(self.readNextBlock()):
841 840 self.dataOut.flagNodata=True
842 841 return 0
843 842
844 843 ##############################
845 844 ##############################
846 845 self.dataOut.data = self.datablock[:,:,self.profileIndex]
847 846 self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
848 847 self.dataOut.profileIndex= self.profileIndex
849 848 self.dataOut.flagNoData=False
850 849 self.profileIndex +=1
851 850
852 851 return self.dataOut.data
853 852
854 853
855 854 def run(self, **kwargs):
856 855 '''
857 856 This method will be called many times so here you should put all your code
858 857 '''
859 858
860 859 if not self.isConfig:
861 860 self.setup(**kwargs)
862 861 self.isConfig = True
863 862 self.getData() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,675 +1,674
1 1 '''
2 2 Created on Set 9, 2015
3 3
4 4 @author: roj-idl71 Karim Kuyeng
5 5 '''
6 6
7 7 import os
8 8 import sys
9 9 import glob
10 10 import fnmatch
11 11 import datetime
12 12 import time
13 13 import re
14 14 import h5py
15 15 import numpy
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
23 23 from schainpy.model.data.jrodata import Voltage
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25 from numpy import imag
26 26
27 27 class AMISRReader(ProcessingUnit):
28 28 '''
29 29 classdocs
30 30 '''
31 31
32 32 def __init__(self):
33 33 '''
34 34 Constructor
35 35 '''
36 36
37 37 ProcessingUnit.__init__(self)
38 38
39 39 self.set = None
40 40 self.subset = None
41 41 self.extension_file = '.h5'
42 42 self.dtc_str = 'dtc'
43 43 self.dtc_id = 0
44 44 self.status = True
45 45 self.isConfig = False
46 46 self.dirnameList = []
47 47 self.filenameList = []
48 48 self.fileIndex = None
49 49 self.flagNoMoreFiles = False
50 50 self.flagIsNewFile = 0
51 51 self.filename = ''
52 52 self.amisrFilePointer = None
53 53
54 54
55 55 self.dataset = None
56 56
57 57
58 58
59 59
60 60 self.profileIndex = 0
61 61
62 62
63 63 self.beamCodeByFrame = None
64 64 self.radacTimeByFrame = None
65 65
66 66 self.dataset = None
67 67
68 68
69 69
70 70
71 71 self.__firstFile = True
72 72
73 73 self.buffer = None
74 74
75 75
76 76 self.timezone = 'ut'
77 77
78 78 self.__waitForNewFile = 20
79 79 self.__filename_online = None
80 80 #Is really necessary create the output object in the initializer
81 81 self.dataOut = Voltage()
82 82
83 83 def setup(self,path=None,
84 84 startDate=None,
85 85 endDate=None,
86 86 startTime=None,
87 87 endTime=None,
88 88 walk=True,
89 89 timezone='ut',
90 90 all=0,
91 91 code = None,
92 92 nCode = 0,
93 93 nBaud = 0,
94 94 online=False):
95 95
96 96 self.timezone = timezone
97 97 self.all = all
98 98 self.online = online
99 99
100 100 self.code = code
101 101 self.nCode = int(nCode)
102 102 self.nBaud = int(nBaud)
103 103
104 104
105 105
106 106 #self.findFiles()
107 107 if not(online):
108 108 #Busqueda de archivos offline
109 109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 110 else:
111 111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
112 112
113 113 if not(self.filenameList):
114 114 print("There is no files into the folder: %s"%(path))
115 115
116 116 sys.exit(-1)
117 117
118 118 self.fileIndex = -1
119 119
120 120 self.readNextFile(online)
121 121
122 122 '''
123 123 Add code
124 124 '''
125 125 self.isConfig = True
126 126
127 127 pass
128 128
129 129
130 130 def readAMISRHeader(self,fp):
131 131 header = 'Raw11/Data/RadacHeader'
132 132 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
133 133 self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
134 134 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
135 135 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
136 136 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
137 137 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
138 138 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
139 139 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
140 140 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
141 141 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
142 142 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
143 143 self.frequency = fp.get('Rx/Frequency')
144 144 txAus = fp.get('Raw11/Data/Pulsewidth')
145 145
146 146
147 147 self.nblocks = self.pulseCount.shape[0] #nblocks
148 148
149 149 self.nprofiles = self.pulseCount.shape[1] #nprofile
150 150 self.nsa = self.nsamplesPulse[0,0] #ngates
151 151 self.nchannels = self.beamCode.shape[1]
152 152 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
153 153 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
154 154 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
155 155
156 156 #filling radar controller header parameters
157 157 self.__ippKm = self.ippSeconds *.15*1e6 # in km
158 158 self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
159 159 self.__txB = 0
160 160 nWindows=1
161 161 self.__nSamples = self.nsa
162 162 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
163 163 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
164 164
165 165 #for now until understand why the code saved is different (code included even though code not in tuf file)
166 166 #self.__codeType = 0
167 167 # self.__nCode = None
168 168 # self.__nBaud = None
169 169 self.__code = self.code
170 170 self.__codeType = 0
171 171 if self.code != None:
172 172 self.__codeType = 1
173 173 self.__nCode = self.nCode
174 174 self.__nBaud = self.nBaud
175 175 #self.__code = 0
176 176
177 177 #filling system header parameters
178 178 self.__nSamples = self.nsa
179 179 self.newProfiles = self.nprofiles/self.nchannels
180 180 self.__channelList = list(range(self.nchannels))
181 181
182 182 self.__frequency = self.frequency[0][0]
183 183
184 184
185 185
186 186 def createBuffers(self):
187 187
188 188 pass
189 189
190 190 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
191 191 self.path = path
192 192 self.startDate = startDate
193 193 self.endDate = endDate
194 194 self.startTime = startTime
195 195 self.endTime = endTime
196 196 self.walk = walk
197 197
198 198 def __checkPath(self):
199 199 if os.path.exists(self.path):
200 200 self.status = 1
201 201 else:
202 202 self.status = 0
203 203 print('Path:%s does not exists'%self.path)
204 204
205 205 return
206 206
207 207
208 208 def __selDates(self, amisr_dirname_format):
209 209 try:
210 210 year = int(amisr_dirname_format[0:4])
211 211 month = int(amisr_dirname_format[4:6])
212 212 dom = int(amisr_dirname_format[6:8])
213 213 thisDate = datetime.date(year,month,dom)
214 214
215 215 if (thisDate>=self.startDate and thisDate <= self.endDate):
216 216 return amisr_dirname_format
217 217 except:
218 218 return None
219 219
220 220
221 221 def __findDataForDates(self,online=False):
222 222
223 223 if not(self.status):
224 224 return None
225 225
226 226 pat = '\d+.\d+'
227 227 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
228 228 dirnameList = [x for x in dirnameList if x!=None]
229 229 dirnameList = [x.string for x in dirnameList]
230 230 if not(online):
231 231 dirnameList = [self.__selDates(x) for x in dirnameList]
232 232 dirnameList = [x for x in dirnameList if x!=None]
233 233 if len(dirnameList)>0:
234 234 self.status = 1
235 235 self.dirnameList = dirnameList
236 236 self.dirnameList.sort()
237 237 else:
238 238 self.status = 0
239 239 return None
240 240
241 241 def __getTimeFromData(self):
242 242 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
243 243 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
244 244
245 245 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
246 246 print('........................................')
247 247 filter_filenameList = []
248 248 self.filenameList.sort()
249 249 #for i in range(len(self.filenameList)-1):
250 250 for i in range(len(self.filenameList)):
251 251 filename = self.filenameList[i]
252 252 fp = h5py.File(filename,'r')
253 253 time_str = fp.get('Time/RadacTimeString')
254 254
255 255 startDateTimeStr_File = time_str[0][0].split('.')[0]
256 256 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
257 257 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
258 258
259 259 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
260 260 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
261 261 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
262 262
263 263 fp.close()
264 264
265 265 if self.timezone == 'lt':
266 266 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
267 267 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
268 268
269 269 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
270 270 #self.filenameList.remove(filename)
271 271 filter_filenameList.append(filename)
272 272
273 273 if (endDateTime_File>=endDateTime_Reader):
274 274 break
275 275
276 276
277 277 filter_filenameList.sort()
278 278 self.filenameList = filter_filenameList
279 279 return 1
280 280
281 281 def __filterByGlob1(self, dirName):
282 282 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
283 283 filter_files.sort()
284 284 filterDict = {}
285 285 filterDict.setdefault(dirName)
286 286 filterDict[dirName] = filter_files
287 287 return filterDict
288 288
289 289 def __getFilenameList(self, fileListInKeys, dirList):
290 290 for value in fileListInKeys:
291 291 dirName = list(value.keys())[0]
292 292 for file in value[dirName]:
293 293 filename = os.path.join(dirName, file)
294 294 self.filenameList.append(filename)
295 295
296 296
297 297 def __selectDataForTimes(self, online=False):
298 298 #aun no esta implementado el filtro for tiempo
299 299 if not(self.status):
300 300 return None
301 301
302 302 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
303 303
304 304 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
305 305
306 306 self.__getFilenameList(fileListInKeys, dirList)
307 307 if not(online):
308 308 #filtro por tiempo
309 309 if not(self.all):
310 310 self.__getTimeFromData()
311 311
312 312 if len(self.filenameList)>0:
313 313 self.status = 1
314 314 self.filenameList.sort()
315 315 else:
316 316 self.status = 0
317 317 return None
318 318
319 319 else:
320 320 #get the last file - 1
321 321 self.filenameList = [self.filenameList[-2]]
322 322
323 323 new_dirnameList = []
324 324 for dirname in self.dirnameList:
325 325 junk = numpy.array([dirname in x for x in self.filenameList])
326 326 junk_sum = junk.sum()
327 327 if junk_sum > 0:
328 328 new_dirnameList.append(dirname)
329 329 self.dirnameList = new_dirnameList
330 330 return 1
331 331
332 332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 333 endTime=datetime.time(23,59,59),walk=True):
334 334
335 335 if endDate ==None:
336 336 startDate = datetime.datetime.utcnow().date()
337 337 endDate = datetime.datetime.utcnow().date()
338 338
339 339 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
340 340
341 341 self.__checkPath()
342 342
343 343 self.__findDataForDates(online=True)
344 344
345 345 self.dirnameList = [self.dirnameList[-1]]
346 346
347 347 self.__selectDataForTimes(online=True)
348 348
349 349 return
350 350
351 351
352 352 def searchFilesOffLine(self,
353 353 path,
354 354 startDate,
355 355 endDate,
356 356 startTime=datetime.time(0,0,0),
357 357 endTime=datetime.time(23,59,59),
358 358 walk=True):
359 359
360 360 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
361 361
362 362 self.__checkPath()
363 363
364 364 self.__findDataForDates()
365 365
366 366 self.__selectDataForTimes()
367 367
368 368 for i in range(len(self.filenameList)):
369 369 print("%s" %(self.filenameList[i]))
370 370
371 371 return
372 372
373 373 def __setNextFileOffline(self):
374 374 idFile = self.fileIndex
375 375
376 376 while (True):
377 377 idFile += 1
378 378 if not(idFile < len(self.filenameList)):
379 379 self.flagNoMoreFiles = 1
380 380 print("No more Files")
381 381 return 0
382 382
383 383 filename = self.filenameList[idFile]
384 384
385 385 amisrFilePointer = h5py.File(filename,'r')
386 386
387 387 break
388 388
389 389 self.flagIsNewFile = 1
390 390 self.fileIndex = idFile
391 391 self.filename = filename
392 392
393 393 self.amisrFilePointer = amisrFilePointer
394 394
395 395 print("Setting the file: %s"%self.filename)
396 396
397 397 return 1
398 398
399 399
400 400 def __setNextFileOnline(self):
401 401 filename = self.filenameList[0]
402 402 if self.__filename_online != None:
403 403 self.__selectDataForTimes(online=True)
404 404 filename = self.filenameList[0]
405 405 wait = 0
406 406 while self.__filename_online == filename:
407 407 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
408 408 if wait == 5:
409 409 return 0
410 410 sleep(self.__waitForNewFile)
411 411 self.__selectDataForTimes(online=True)
412 412 filename = self.filenameList[0]
413 413 wait += 1
414 414
415 415 self.__filename_online = filename
416 416
417 417 self.amisrFilePointer = h5py.File(filename,'r')
418 418 self.flagIsNewFile = 1
419 419 self.filename = filename
420 420 print("Setting the file: %s"%self.filename)
421 421 return 1
422 422
423 423
424 424 def readData(self):
425 425 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
426 426 re = buffer[:,:,:,0]
427 427 im = buffer[:,:,:,1]
428 428 dataset = re + im*1j
429 429 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
430 430 timeset = self.radacTime[:,0]
431 431 return dataset,timeset
432 432
433 433 def reshapeData(self):
434 434 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
435 435 channels = self.beamCodeByPulse[0,:]
436 436 nchan = self.nchannels
437 437 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
438 438 nblocks = self.nblocks
439 439 nsamples = self.nsa
440 440
441 441 #Dimensions : nChannels, nProfiles, nSamples
442 442 new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
443 443 ############################################
444 444
445 445 for thisChannel in range(nchan):
446 446 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
447 447
448 448
449 449 new_block = numpy.transpose(new_block, (1,0,2,3))
450 450 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
451 451
452 452 return new_block
453 453
454 454 def updateIndexes(self):
455 455
456 456 pass
457 457
458 458 def fillJROHeader(self):
459 459
460 460 #fill radar controller header
461 461 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
462 462 txA=self.__txA,
463 463 txB=0,
464 464 nWindows=1,
465 465 nHeights=self.__nSamples,
466 466 firstHeight=self.__firstHeight,
467 467 deltaHeight=self.__deltaHeight,
468 468 codeType=self.__codeType,
469 469 nCode=self.__nCode, nBaud=self.__nBaud,
470 470 code = self.__code,
471 471 fClock=1)
472 472
473 473
474 474
475 475 #fill system header
476 476 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
477 477 nProfiles=self.newProfiles,
478 478 nChannels=len(self.__channelList),
479 479 adcResolution=14,
480 480 pciDioBusWith=32)
481 481
482 482 self.dataOut.type = "Voltage"
483 483
484 484 self.dataOut.data = None
485 485
486 486 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
487 487
488 488 # self.dataOut.nChannels = 0
489 489
490 490 # self.dataOut.nHeights = 0
491 491
492 492 self.dataOut.nProfiles = self.newProfiles*self.nblocks
493 493
494 494 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
495 495 ranges = numpy.reshape(self.rangeFromFile.value,(-1))
496 496 self.dataOut.heightList = ranges/1000.0 #km
497 497
498 498
499 499 self.dataOut.channelList = self.__channelList
500 500
501 501 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
502 502
503 503 # self.dataOut.channelIndexList = None
504 504
505 505 self.dataOut.flagNoData = True
506 506
507 507 #Set to TRUE if the data is discontinuous
508 508 self.dataOut.flagDiscontinuousBlock = False
509 509
510 510 self.dataOut.utctime = None
511 511
512 512 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
513 513 if self.timezone == 'lt':
514 514 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
515 515 else:
516 516 self.dataOut.timeZone = 0 #by default time is UTC
517 517
518 518 self.dataOut.dstFlag = 0
519 519
520 520 self.dataOut.errorCount = 0
521 521
522 522 self.dataOut.nCohInt = 1
523 523
524 524 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
525 525
526 526 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
527 527
528 528 self.dataOut.flagShiftFFT = False
529 529
530 530 self.dataOut.ippSeconds = self.ippSeconds
531 531
532 532 #Time interval between profiles
533 533 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
534 534
535 535 self.dataOut.frequency = self.__frequency
536 536
537 537 self.dataOut.realtime = self.online
538 538 pass
539 539
540 540 def readNextFile(self,online=False):
541 541
542 542 if not(online):
543 543 newFile = self.__setNextFileOffline()
544 544 else:
545 545 newFile = self.__setNextFileOnline()
546 546
547 547 if not(newFile):
548 548 return 0
549 549
550 550 #if self.__firstFile:
551 551 self.readAMISRHeader(self.amisrFilePointer)
552 552 self.createBuffers()
553 553 self.fillJROHeader()
554 554 #self.__firstFile = False
555 555
556 556
557 557
558 558 self.dataset,self.timeset = self.readData()
559 559
560 560 if self.endDate!=None:
561 561 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
562 562 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
563 563 startDateTimeStr_File = time_str[0][0].split('.')[0]
564 564 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
565 565 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
566 566 if self.timezone == 'lt':
567 567 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
568 568 if (startDateTime_File>endDateTime_Reader):
569 569 return 0
570 570
571 571 self.jrodataset = self.reshapeData()
572 572 #----self.updateIndexes()
573 573 self.profileIndex = 0
574 574
575 575 return 1
576 576
577 577
578 578 def __hasNotDataInBuffer(self):
579 579 if self.profileIndex >= (self.newProfiles*self.nblocks):
580 580 return 1
581 581 return 0
582 582
583 583
584 584 def getData(self):
585 585
586 586 if self.flagNoMoreFiles:
587 587 self.dataOut.flagNoData = True
588 print('Process finished')
589 588 return 0
590 589
591 590 if self.__hasNotDataInBuffer():
592 591 if not (self.readNextFile(self.online)):
593 592 return 0
594 593
595 594
596 595 if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
597 596 self.dataOut.flagNoData = True
598 597 return 0
599 598
600 599 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
601 600
602 601 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
603 602
604 603 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
605 604 #verificar basic header de jro data y ver si es compatible con este valor
606 605 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
607 606 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
608 607 indexblock = self.profileIndex/self.newProfiles
609 608 #print indexblock, indexprof
610 609 self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
611 610 self.dataOut.profileIndex = self.profileIndex
612 611 self.dataOut.flagNoData = False
613 612 # if indexprof == 0:
614 613 # print self.dataOut.utctime
615 614
616 615 self.profileIndex += 1
617 616
618 617 return self.dataOut.data
619 618
620 619
621 620 def run(self, **kwargs):
622 621 '''
623 622 This method will be called many times so here you should put all your code
624 623 '''
625 624
626 625 if not self.isConfig:
627 626 self.setup(**kwargs)
628 627 self.isConfig = True
629 628
630 629 self.getData()
631 630
632 631 class Writer(Operation):
633 632 '''
634 633 classdocs
635 634 '''
636 635
637 636 def __init__(self):
638 637 '''
639 638 Constructor
640 639 '''
641 640 self.dataOut = None
642 641
643 642 self.isConfig = False
644 643
645 644 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
646 645 '''
647 646 In this method we should set all initial parameters.
648 647
649 648 Input:
650 649 dataIn : Input data will also be outputa data
651 650
652 651 '''
653 652 self.dataOut = dataIn
654 653
655 654
656 655
657 656
658 657
659 658 self.isConfig = True
660 659
661 660 return
662 661
663 662 def run(self, dataIn, **kwargs):
664 663 '''
665 664 This method will be called many times so here you should put all your code
666 665
667 666 Inputs:
668 667
669 668 dataIn : object with the data
670 669
671 670 '''
672 671
673 672 if not self.isConfig:
674 673 self.setup(dataIn, **kwargs)
675 674 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,637 +1,639
1 1 '''
2 2 Created on Aug 1, 2017
3 3
4 4 @author: Juan C. Espinoza
5 5 '''
6 6
7 7 import os
8 8 import sys
9 9 import time
10 10 import json
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15 import h5py
16
17 import schainpy.admin
16 18 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
17 19 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
18 20 from schainpy.model.data.jrodata import Parameters
19 21 from schainpy.utils import log
20 22
21 23 try:
22 24 import madrigal.cedar
23 25 except:
24 26 log.warning(
25 27 'You should install "madrigal library" module if you want to read/write Madrigal data'
26 28 )
27 29
28 30 try:
29 31 basestring
30 32 except:
31 33 basestring = str
32 34
33 35 DEF_CATALOG = {
34 36 'principleInvestigator': 'Marco Milla',
35 37 'expPurpose': '',
36 38 'cycleTime': '',
37 39 'correlativeExp': '',
38 40 'sciRemarks': '',
39 41 'instRemarks': ''
40 42 }
41 43
42 44 DEF_HEADER = {
43 45 'kindatDesc': '',
44 46 'analyst': 'Jicamarca User',
45 47 'comments': '',
46 48 'history': ''
47 49 }
48 50
49 51 MNEMONICS = {
50 52 10: 'jro',
51 53 11: 'jbr',
52 54 840: 'jul',
53 55 13: 'jas',
54 56 1000: 'pbr',
55 57 1001: 'hbr',
56 58 1002: 'obr',
57 59 400: 'clr'
58 60
59 61 }
60 62
61 63 UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
62 64
63 65 def load_json(obj):
64 66 '''
65 67 Parse json as string instead of unicode
66 68 '''
67 69
68 70 if isinstance(obj, str):
69 71 iterable = json.loads(obj)
70 72 else:
71 73 iterable = obj
72 74
73 75 if isinstance(iterable, dict):
74 76 return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
75 77 for k, v in list(iterable.items())}
76 78 elif isinstance(iterable, (list, tuple)):
77 79 return [str(v) if isinstance(v, basestring) else v for v in iterable]
78 80
79 81 return iterable
80 82
81 83 @MPDecorator
82 84 class MADReader(JRODataReader, ProcessingUnit):
83 85
84 86 def __init__(self):
85 87
86 88 ProcessingUnit.__init__(self)
87 89
88 90 self.dataOut = Parameters()
89 91 self.counter_records = 0
90 92 self.nrecords = None
91 93 self.flagNoMoreFiles = 0
92 94 self.isConfig = False
93 95 self.filename = None
94 96 self.intervals = set()
95 97
96 98 def setup(self,
97 99 path=None,
98 100 startDate=None,
99 101 endDate=None,
100 102 format=None,
101 103 startTime=datetime.time(0, 0, 0),
102 104 endTime=datetime.time(23, 59, 59),
103 105 **kwargs):
104 106
105 107 self.path = path
106 108 self.startDate = startDate
107 109 self.endDate = endDate
108 110 self.startTime = startTime
109 111 self.endTime = endTime
110 112 self.datatime = datetime.datetime(1900,1,1)
111 113 self.oneDDict = load_json(kwargs.get('oneDDict',
112 114 "{\"GDLATR\":\"lat\", \"GDLONR\":\"lon\"}"))
113 115 self.twoDDict = load_json(kwargs.get('twoDDict',
114 116 "{\"GDALT\": \"heightList\"}"))
115 117 self.independentParam = 'GDALT'
116 118
117 119 if self.path is None:
118 120 raise ValueError('The path is not valid')
119 121
120 122 if format is None:
121 123 raise ValueError('The format is not valid choose simple or hdf5')
122 124 elif format.lower() in ('simple', 'txt'):
123 125 self.ext = '.txt'
124 126 elif format.lower() in ('cedar',):
125 127 self.ext = '.001'
126 128 else:
127 129 self.ext = '.hdf5'
128 130
129 131 self.search_files(self.path)
130 132 self.fileId = 0
131 133
132 134 if not self.fileList:
133 135 raise Warning('There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path))
134 136
135 137 self.setNextFile()
136 138
137 139 def search_files(self, path):
138 140 '''
139 141 Searching for madrigal files in path
140 142 Creating a list of files to procces included in [startDate,endDate]
141 143
142 144 Input:
143 145 path - Path to find files
144 146 '''
145 147
146 148 log.log('Searching files {} in {} '.format(self.ext, path), 'MADReader')
147 149 fileList0 = glob.glob1(path, '*{}'.format(self.ext))
148 150 fileList0.sort()
149 151
150 152 self.fileList = []
151 153 self.dateFileList = []
152 154
153 155 startDate = self.startDate - datetime.timedelta(1)
154 156 endDate = self.endDate + datetime.timedelta(1)
155 157
156 158 for thisFile in fileList0:
157 159 year = thisFile[3:7]
158 160 if not year.isdigit():
159 161 continue
160 162
161 163 month = thisFile[7:9]
162 164 if not month.isdigit():
163 165 continue
164 166
165 167 day = thisFile[9:11]
166 168 if not day.isdigit():
167 169 continue
168 170
169 171 year, month, day = int(year), int(month), int(day)
170 172 dateFile = datetime.date(year, month, day)
171 173
172 174 if (startDate > dateFile) or (endDate < dateFile):
173 175 continue
174 176
175 177 self.fileList.append(thisFile)
176 178 self.dateFileList.append(dateFile)
177 179
178 180 return
179 181
180 182 def parseHeader(self):
181 183 '''
182 184 '''
183 185
184 186 self.output = {}
185 187 self.version = '2'
186 188 s_parameters = None
187 189 if self.ext == '.txt':
188 190 self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
189 191 elif self.ext == '.hdf5':
190 192 self.metadata = self.fp['Metadata']
191 193 if '_record_layout' in self.metadata:
192 194 s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
193 195 self.version = '3'
194 196 self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
195 197
196 198 log.success('Parameters found: {}'.format(self.parameters),
197 199 'MADReader')
198 200 if s_parameters:
199 201 log.success('Spatial parameters found: {}'.format(s_parameters),
200 202 'MADReader')
201 203
202 204 for param in list(self.oneDDict.keys()):
203 205 if param.lower() not in self.parameters:
204 206 log.warning(
205 207 'Parameter {} not found will be ignored'.format(
206 208 param),
207 209 'MADReader')
208 210 self.oneDDict.pop(param, None)
209 211
210 212 for param, value in list(self.twoDDict.items()):
211 213 if param.lower() not in self.parameters:
212 214 log.warning(
213 215 'Parameter {} not found, it will be ignored'.format(
214 216 param),
215 217 'MADReader')
216 218 self.twoDDict.pop(param, None)
217 219 continue
218 220 if isinstance(value, list):
219 221 if value[0] not in self.output:
220 222 self.output[value[0]] = []
221 223 self.output[value[0]].append([])
222 224
223 225 def parseData(self):
224 226 '''
225 227 '''
226 228
227 229 if self.ext == '.txt':
228 230 self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
229 231 self.nrecords = self.data.shape[0]
230 232 self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
231 233 self.counter_records = 0
232 234 elif self.ext == '.hdf5':
233 235 self.data = self.fp['Data']
234 236 self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
235 237 self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
236 238 self.counter_records = int(self.data['Table Layout']['recno'][0])
237 239 self.nrecords = int(self.data['Table Layout']['recno'][-1])
238 240
239 241 def setNextFile(self):
240 242 '''
241 243 '''
242 244
243 245 file_id = self.fileId
244 246
245 247 if file_id == len(self.fileList):
246 248 log.success('No more files', 'MADReader')
247 249 self.flagNoMoreFiles = 1
248 250 return 0
249 251
250 252 log.success(
251 253 'Opening: {}'.format(self.fileList[file_id]),
252 254 'MADReader'
253 255 )
254 256
255 257 filename = os.path.join(self.path, self.fileList[file_id])
256 258
257 259 if self.filename is not None:
258 260 self.fp.close()
259 261
260 262 self.filename = filename
261 263 self.filedate = self.dateFileList[file_id]
262 264
263 265 if self.ext=='.hdf5':
264 266 self.fp = h5py.File(self.filename, 'r')
265 267 else:
266 268 self.fp = open(self.filename, 'rb')
267 269
268 270 self.parseHeader()
269 271 self.parseData()
270 272 self.sizeOfFile = os.path.getsize(self.filename)
271 273 self.flagIsNewFile = 0
272 274 self.fileId += 1
273 275
274 276 return 1
275 277
276 278 def readNextBlock(self):
277 279
278 280 while True:
279 281 self.flagDiscontinuousBlock = 0
280 282 if self.flagIsNewFile:
281 283 if not self.setNextFile():
282 284 return 0
283 285
284 286 self.readBlock()
285 287
286 288 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
287 289 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
288 290 log.warning(
289 291 'Reading Record No. {}/{} -> {} [Skipping]'.format(
290 292 self.counter_records,
291 293 self.nrecords,
292 294 self.datatime.ctime()),
293 295 'MADReader')
294 296 continue
295 297 break
296 298
297 299 log.log(
298 300 'Reading Record No. {}/{} -> {}'.format(
299 301 self.counter_records,
300 302 self.nrecords,
301 303 self.datatime.ctime()),
302 304 'MADReader')
303 305
304 306 return 1
305 307
306 308 def readBlock(self):
307 309 '''
308 310 '''
309 311 dum = []
310 312 if self.ext == '.txt':
311 313 dt = self.data[self.counter_records][:6].astype(int)
312 314 if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
313 315 self.flagDiscontinuousBlock = 1
314 316 self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
315 317 while True:
316 318 dt = self.data[self.counter_records][:6].astype(int)
317 319 datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
318 320 if datatime == self.datatime:
319 321 dum.append(self.data[self.counter_records])
320 322 self.counter_records += 1
321 323 if self.counter_records == self.nrecords:
322 324 self.flagIsNewFile = True
323 325 break
324 326 continue
325 327 self.intervals.add((datatime-self.datatime).seconds)
326 328 break
327 329 elif self.ext == '.hdf5':
328 330 datatime = datetime.datetime.utcfromtimestamp(
329 331 self.times[self.counter_records])
330 332 dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
331 333 self.intervals.add((datatime-self.datatime).seconds)
332 334 if datatime.date()>self.datatime.date():
333 335 self.flagDiscontinuousBlock = 1
334 336 self.datatime = datatime
335 337 self.counter_records += 1
336 338 if self.counter_records == self.nrecords:
337 339 self.flagIsNewFile = True
338 340
339 341 self.buffer = numpy.array(dum)
340 342 return
341 343
342 344 def set_output(self):
343 345 '''
344 346 Storing data from buffer to dataOut object
345 347 '''
346 348
347 349 parameters = [None for __ in self.parameters]
348 350
349 351 for param, attr in list(self.oneDDict.items()):
350 352 x = self.parameters.index(param.lower())
351 353 setattr(self.dataOut, attr, self.buffer[0][x])
352 354
353 355 for param, value in list(self.twoDDict.items()):
354 356 dummy = numpy.zeros(self.ranges.shape) + numpy.nan
355 357 if self.ext == '.txt':
356 358 x = self.parameters.index(param.lower())
357 359 y = self.parameters.index(self.independentParam.lower())
358 360 ranges = self.buffer[:,y]
359 361 #if self.ranges.size == ranges.size:
360 362 # continue
361 363 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
362 364 dummy[index] = self.buffer[:,x]
363 365 else:
364 366 ranges = self.buffer[self.independentParam.lower()]
365 367 index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
366 368 dummy[index] = self.buffer[param.lower()]
367 369
368 370 if isinstance(value, str):
369 371 if value not in self.independentParam:
370 372 setattr(self.dataOut, value, dummy.reshape(1,-1))
371 373 elif isinstance(value, list):
372 374 self.output[value[0]][value[1]] = dummy
373 375 parameters[value[1]] = param
374 376 for key, value in list(self.output.items()):
375 377 setattr(self.dataOut, key, numpy.array(value))
376 378
377 379 self.dataOut.parameters = [s for s in parameters if s]
378 380 self.dataOut.heightList = self.ranges
379 381 self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
380 382 self.dataOut.utctimeInit = self.dataOut.utctime
381 383 self.dataOut.paramInterval = min(self.intervals)
382 384 self.dataOut.useLocalTime = False
383 385 self.dataOut.flagNoData = False
384 386 self.dataOut.nrecords = self.nrecords
385 387 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
386 388
387 389 def getData(self):
388 390 '''
389 391 Storing data from databuffer to dataOut object
390 392 '''
391 393 if self.flagNoMoreFiles:
392 394 self.dataOut.flagNoData = True
393 self.dataOut.error = 'No file left to process'
395 raise schainpy.admin.SchainError('No file left to process')
394 396 return 0
395 397
396 398 if not self.readNextBlock():
397 399 self.dataOut.flagNoData = True
398 400 return 0
399 401
400 402 self.set_output()
401 403
402 404 return 1
403 405
404 406 @MPDecorator
405 407 class MADWriter(Operation):
406 408
407 409 missing = -32767
408 410
409 411 def __init__(self):
410 412
411 413 Operation.__init__(self)
412 414 self.dataOut = Parameters()
413 415 self.counter = 0
414 416 self.path = None
415 417 self.fp = None
416 418
417 419 def run(self, dataOut, path, oneDDict, independentParam='[]', twoDDict='{}',
418 420 metadata='{}', format='cedar', **kwargs):
419 421 '''
420 422 Inputs:
421 423 path - path where files will be created
422 424 oneDDict - json of one-dimensional parameters in record where keys
423 425 are Madrigal codes (integers or mnemonics) and values the corresponding
424 426 dataOut attribute e.g: {
425 427 'gdlatr': 'lat',
426 428 'gdlonr': 'lon',
427 429 'gdlat2':'lat',
428 430 'glon2':'lon'}
429 431 independentParam - list of independent spatial two-dimensional parameters e.g:
430 432 ['heigthList']
431 433 twoDDict - json of two-dimensional parameters in record where keys
432 434 are Madrigal codes (integers or mnemonics) and values the corresponding
433 435 dataOut attribute if multidimensional array specify as tupple
434 436 ('attr', pos) e.g: {
435 437 'gdalt': 'heightList',
436 438 'vn1p2': ('data_output', 0),
437 439 'vn2p2': ('data_output', 1),
438 440 'vn3': ('data_output', 2),
439 441 'snl': ('data_SNR', 'db')
440 442 }
441 443 metadata - json of madrigal metadata (kinst, kindat, catalog and header)
442 444 '''
443 445 if not self.isConfig:
444 446 self.setup(path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs)
445 447 self.isConfig = True
446 448
447 449 self.dataOut = dataOut
448 450 self.putData()
449 451 return 1
450 452
451 453 def setup(self, path, oneDDict, independentParam, twoDDict, metadata, format, **kwargs):
452 454 '''
453 455 Configure Operation
454 456 '''
455 457
456 458 self.path = path
457 459 self.blocks = kwargs.get('blocks', None)
458 460 self.counter = 0
459 461 self.oneDDict = load_json(oneDDict)
460 462 self.twoDDict = load_json(twoDDict)
461 463 self.independentParam = load_json(independentParam)
462 464 meta = load_json(metadata)
463 465 self.kinst = meta.get('kinst')
464 466 self.kindat = meta.get('kindat')
465 467 self.catalog = meta.get('catalog', DEF_CATALOG)
466 468 self.header = meta.get('header', DEF_HEADER)
467 469 if format == 'cedar':
468 470 self.ext = '.dat'
469 471 self.extra_args = {}
470 472 elif format == 'hdf5':
471 473 self.ext = '.hdf5'
472 474 self.extra_args = {'independentParam': self.independentParam}
473 475
474 476 self.keys = [k.lower() for k in self.twoDDict]
475 477 if 'range' in self.keys:
476 478 self.keys.remove('range')
477 479 if 'gdalt' in self.keys:
478 480 self.keys.remove('gdalt')
479 481
480 482 def setFile(self):
481 483 '''
482 484 Create new cedar file object
483 485 '''
484 486
485 487 self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal
486 488 date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
487 489
488 490 filename = '{}{}{}'.format(self.mnemonic,
489 491 date.strftime('%Y%m%d_%H%M%S'),
490 492 self.ext)
491 493
492 494 self.fullname = os.path.join(self.path, filename)
493 495
494 496 if os.path.isfile(self.fullname) :
495 497 log.warning(
496 498 'Destination file {} already exists, previous file deleted.'.format(
497 499 self.fullname),
498 500 'MADWriter')
499 501 os.remove(self.fullname)
500 502
501 503 try:
502 504 log.success(
503 505 'Creating file: {}'.format(self.fullname),
504 506 'MADWriter')
505 507 self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
506 508 except ValueError as e:
507 509 log.error(
508 510 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
509 511 'MADWriter')
510 512 return
511 513
512 514 return 1
513 515
514 516 def writeBlock(self):
515 517 '''
516 518 Add data records to cedar file taking data from oneDDict and twoDDict
517 519 attributes.
518 520 Allowed parameters in: parcodes.tab
519 521 '''
520 522
521 523 startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
522 524 endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
523 525 heights = self.dataOut.heightList
524 526
525 527 if self.ext == '.dat':
526 528 for key, value in list(self.twoDDict.items()):
527 529 if isinstance(value, str):
528 530 data = getattr(self.dataOut, value)
529 531 invalid = numpy.isnan(data)
530 532 data[invalid] = self.missing
531 533 elif isinstance(value, (tuple, list)):
532 534 attr, key = value
533 535 data = getattr(self.dataOut, attr)
534 536 invalid = numpy.isnan(data)
535 537 data[invalid] = self.missing
536 538
537 539 out = {}
538 540 for key, value in list(self.twoDDict.items()):
539 541 key = key.lower()
540 542 if isinstance(value, str):
541 543 if 'db' in value.lower():
542 544 tmp = getattr(self.dataOut, value.replace('_db', ''))
543 545 SNRavg = numpy.average(tmp, axis=0)
544 546 tmp = 10*numpy.log10(SNRavg)
545 547 else:
546 548 tmp = getattr(self.dataOut, value)
547 549 out[key] = tmp.flatten()
548 550 elif isinstance(value, (tuple, list)):
549 551 attr, x = value
550 552 data = getattr(self.dataOut, attr)
551 553 out[key] = data[int(x)]
552 554
553 555 a = numpy.array([out[k] for k in self.keys])
554 556 nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
555 557 index = numpy.where(nrows == False)[0]
556 558
557 559 rec = madrigal.cedar.MadrigalDataRecord(
558 560 self.kinst,
559 561 self.kindat,
560 562 startTime.year,
561 563 startTime.month,
562 564 startTime.day,
563 565 startTime.hour,
564 566 startTime.minute,
565 567 startTime.second,
566 568 startTime.microsecond/10000,
567 569 endTime.year,
568 570 endTime.month,
569 571 endTime.day,
570 572 endTime.hour,
571 573 endTime.minute,
572 574 endTime.second,
573 575 endTime.microsecond/10000,
574 576 list(self.oneDDict.keys()),
575 577 list(self.twoDDict.keys()),
576 578 len(index),
577 579 **self.extra_args
578 580 )
579 581
580 582 # Setting 1d values
581 583 for key in self.oneDDict:
582 584 rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
583 585
584 586 # Setting 2d values
585 587 nrec = 0
586 588 for n in index:
587 589 for key in out:
588 590 rec.set2D(key, nrec, out[key][n])
589 591 nrec += 1
590 592
591 593 self.fp.append(rec)
592 594 if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
593 595 self.fp.dump()
594 596 if self.counter % 20 == 0 and self.counter > 0:
595 597 log.log(
596 598 'Writing {} records'.format(
597 599 self.counter),
598 600 'MADWriter')
599 601
600 602 def setHeader(self):
601 603 '''
602 604 Create an add catalog and header to cedar file
603 605 '''
604 606
605 607 log.success('Closing file {}'.format(self.fullname), 'MADWriter')
606 608
607 609 if self.ext == '.dat':
608 610 self.fp.write()
609 611 else:
610 612 self.fp.dump()
611 613 self.fp.close()
612 614
613 615 header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
614 616 header.createCatalog(**self.catalog)
615 617 header.createHeader(**self.header)
616 618 header.write()
617 619
618 620 def putData(self):
619 621
620 622 if self.dataOut.flagNoData:
621 623 return 0
622 624
623 625 if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
624 626 if self.counter > 0:
625 627 self.setHeader()
626 628 self.counter = 0
627 629
628 630 if self.counter == 0:
629 631 self.setFile()
630 632
631 633 self.writeBlock()
632 634 self.counter += 1
633 635
634 636 def close(self):
635 637
636 638 if self.counter > 0:
637 639 self.setHeader() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1544 +1,1544
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 import schainpy.admin
8 9 from schainpy.model.data.jrodata import *
9 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 11 from schainpy.model.io.jroIO_base import *
11 12 from schainpy.utils import log
12 13
13 14 @MPDecorator
14 15 class ParamReader(JRODataReader,ProcessingUnit):
15 16 '''
16 17 Reads HDF5 format files
17 18 path
18 19 startDate
19 20 endDate
20 21 startTime
21 22 endTime
22 23 '''
23 24
24 25 ext = ".hdf5"
25 26 optchar = "D"
26 27 timezone = None
27 28 startTime = None
28 29 endTime = None
29 30 fileIndex = None
30 31 utcList = None #To select data in the utctime list
31 32 blockList = None #List to blocks to be read from the file
32 33 blocksPerFile = None #Number of blocks to be read
33 34 blockIndex = None
34 35 path = None
35 36 #List of Files
36 37 filenameList = None
37 38 datetimeList = None
38 39 #Hdf5 File
39 40 listMetaname = None
40 41 listMeta = None
41 42 listDataname = None
42 43 listData = None
43 44 listShapes = None
44 45 fp = None
45 46 #dataOut reconstruction
46 47 dataOut = None
47 48
48 49 def __init__(self):#, **kwargs):
49 50 ProcessingUnit.__init__(self) #, **kwargs)
50 51 self.dataOut = Parameters()
51 52 return
52 53
53 54 def setup(self, **kwargs):
54 55
55 56 path = kwargs['path']
56 57 startDate = kwargs['startDate']
57 58 endDate = kwargs['endDate']
58 59 startTime = kwargs['startTime']
59 60 endTime = kwargs['endTime']
60 61 walk = kwargs['walk']
61 62 if 'ext' in kwargs:
62 63 ext = kwargs['ext']
63 64 else:
64 65 ext = '.hdf5'
65 66 if 'timezone' in kwargs:
66 67 self.timezone = kwargs['timezone']
67 68 else:
68 69 self.timezone = 'lt'
69 70
70 71 print("[Reading] Searching files in offline mode ...")
71 72 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
72 73 startTime=startTime, endTime=endTime,
73 74 ext=ext, walk=walk)
74 75
75 76 if not(filenameList):
76 77 print("There is no files into the folder: %s"%(path))
77 78 sys.exit(-1)
78 79
79 80 self.fileIndex = -1
80 81 self.startTime = startTime
81 82 self.endTime = endTime
82 83
83 84 self.__readMetadata()
84 85
85 86 self.__setNextFileOffline()
86 87
87 88 return
88 89
89 90 def searchFilesOffLine(self,
90 91 path,
91 92 startDate=None,
92 93 endDate=None,
93 94 startTime=datetime.time(0,0,0),
94 95 endTime=datetime.time(23,59,59),
95 96 ext='.hdf5',
96 97 walk=True):
97 98
98 99 expLabel = ''
99 100 self.filenameList = []
100 101 self.datetimeList = []
101 102
102 103 pathList = []
103 104
104 105 JRODataObj = JRODataReader()
105 106 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
106 107
107 108 if dateList == []:
108 109 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
109 110 datetime.datetime.combine(startDate,startTime).ctime(),
110 111 datetime.datetime.combine(endDate,endTime).ctime()))
111 112
112 113 return None, None
113 114
114 115 if len(dateList) > 1:
115 116 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
116 117 else:
117 118 print("[Reading] data was found for the date %s" %(dateList[0]))
118 119
119 120 filenameList = []
120 121 datetimeList = []
121 122
122 123 #----------------------------------------------------------------------------------
123 124
124 125 for thisPath in pathList:
125 126
126 127 fileList = glob.glob1(thisPath, "*%s" %ext)
127 128 fileList.sort()
128 129
129 130 for file in fileList:
130 131
131 132 filename = os.path.join(thisPath,file)
132 133
133 134 if not isFileInDateRange(filename, startDate, endDate):
134 135 continue
135 136
136 137 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
137 138
138 139 if not(thisDatetime):
139 140 continue
140 141
141 142 filenameList.append(filename)
142 143 datetimeList.append(thisDatetime)
143 144
144 145 if not(filenameList):
145 146 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
146 147 return None, None
147 148
148 149 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
149 150 print()
150 151
151 152 self.filenameList = filenameList
152 153 self.datetimeList = datetimeList
153 154
154 155 return pathList, filenameList
155 156
156 157 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
157 158
158 159 """
159 160 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
160 161
161 162 Inputs:
162 163 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
163 164 startDate : fecha inicial del rango seleccionado en formato datetime.date
164 165 endDate : fecha final del rango seleccionado en formato datetime.date
165 166 startTime : tiempo inicial del rango seleccionado en formato datetime.time
166 167 endTime : tiempo final del rango seleccionado en formato datetime.time
167 168
168 169 Return:
169 170 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
170 171 fecha especificado, de lo contrario retorna False.
171 172
172 173 Excepciones:
173 174 Si el archivo no existe o no puede ser abierto
174 175 Si la cabecera no puede ser leida.
175 176
176 177 """
177 178
178 179 try:
179 180 fp = h5py.File(filename,'r')
180 181 grp1 = fp['Data']
181 182
182 183 except IOError:
183 184 traceback.print_exc()
184 185 raise IOError("The file %s can't be opened" %(filename))
185 186
186 187 #In case has utctime attribute
187 188 grp2 = grp1['utctime']
188 189 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
189 190 thisUtcTime = grp2.value[0]
190 191
191 192 fp.close()
192 193
193 194 if self.timezone == 'lt':
194 195 thisUtcTime -= 5*3600
195 196
196 197 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
197 198 thisDate = thisDatetime.date()
198 199 thisTime = thisDatetime.time()
199 200
200 201 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
201 202 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
202 203
203 204 #General case
204 205 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
205 206 #-----------o----------------------------o-----------
206 207 # startTime endTime
207 208
208 209 if endTime >= startTime:
209 210 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
210 211 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
211 212 return thisDatetime
212 213 return None
213 214
214 215 #If endTime < startTime then endTime belongs to the next day
215 216 #<<<<<<<<<<<o o>>>>>>>>>>>
216 217 #-----------o----------------------------o-----------
217 218 # endTime startTime
218 219
219 220 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
220 221 return None
221 222
222 223 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
223 224 return None
224 225
225 226 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
226 227 return None
227 228
228 229 return thisDatetime
229 230
230 231 def __setNextFileOffline(self):
231 232
232 233 self.fileIndex += 1
233 234 idFile = self.fileIndex
234 235
235 236 if not(idFile < len(self.filenameList)):
236 self.dataOut.error = "No more Files"
237 raise schainpy.admin.SchainError("No more Files")
237 238 return 0
238 239
239 240 filename = self.filenameList[idFile]
240 241 filePointer = h5py.File(filename,'r')
241 242 self.filename = filename
242 243 self.fp = filePointer
243 244
244 245 print("Setting the file: %s"%self.filename)
245 246
246 247 self.__setBlockList()
247 248 self.__readData()
248 249 self.blockIndex = 0
249 250 return 1
250 251
251 252 def __setBlockList(self):
252 253 '''
253 254 Selects the data within the times defined
254 255
255 256 self.fp
256 257 self.startTime
257 258 self.endTime
258 259
259 260 self.blockList
260 261 self.blocksPerFile
261 262
262 263 '''
263 264 fp = self.fp
264 265 startTime = self.startTime
265 266 endTime = self.endTime
266 267
267 268 grp = fp['Data']
268 269 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
269 270
270 271 #ERROOOOR
271 272 if self.timezone == 'lt':
272 273 thisUtcTime -= 5*3600
273 274
274 275 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
275 276
276 277 thisDate = thisDatetime.date()
277 278 thisTime = thisDatetime.time()
278 279
279 280 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
280 281 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
281 282
282 283 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
283 284
284 285 self.blockList = ind
285 286 self.blocksPerFile = len(ind)
286 287
287 288 return
288 289
289 290 def __readMetadata(self):
290 291 '''
291 292 Reads Metadata
292 293
293 294 self.pathMeta
294 295 self.listShapes
295 296 self.listMetaname
296 297 self.listMeta
297 298
298 299 '''
299 300
300 301 filename = self.filenameList[0]
301 302 fp = h5py.File(filename,'r')
302 303 gp = fp['Metadata']
303 304
304 305 listMetaname = []
305 306 listMetadata = []
306 307 for item in list(gp.items()):
307 308 name = item[0]
308 309
309 310 if name=='array dimensions':
310 311 table = gp[name][:]
311 312 listShapes = {}
312 313 for shapes in table:
313 314 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
314 315 else:
315 316 data = gp[name].value
316 317 listMetaname.append(name)
317 318 listMetadata.append(data)
318 319
319 320 self.listShapes = listShapes
320 321 self.listMetaname = listMetaname
321 322 self.listMeta = listMetadata
322 323
323 324 fp.close()
324 325 return
325 326
326 327 def __readData(self):
327 328 grp = self.fp['Data']
328 329 listdataname = []
329 330 listdata = []
330 331
331 332 for item in list(grp.items()):
332 333 name = item[0]
333 334 listdataname.append(name)
334 335
335 336 array = self.__setDataArray(grp[name],self.listShapes[name])
336 337 listdata.append(array)
337 338
338 339 self.listDataname = listdataname
339 340 self.listData = listdata
340 341 return
341 342
342 343 def __setDataArray(self, dataset, shapes):
343 344
344 345 nDims = shapes[0]
345 346 nDim2 = shapes[1] #Dimension 0
346 347 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
347 348 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
348 349 mode = shapes[4] #Mode of storing
349 350 blockList = self.blockList
350 351 blocksPerFile = self.blocksPerFile
351 352
352 353 #Depending on what mode the data was stored
353 354 if mode == 0: #Divided in channels
354 355 arrayData = dataset.value.astype(numpy.float)[0][blockList]
355 356 if mode == 1: #Divided in parameter
356 357 strds = 'table'
357 358 nDatas = nDim1
358 359 newShapes = (blocksPerFile,nDim2,nDim0)
359 360 elif mode==2: #Concatenated in a table
360 361 strds = 'table0'
361 362 arrayData = dataset[strds].value
362 363 #Selecting part of the dataset
363 364 utctime = arrayData[:,0]
364 365 u, indices = numpy.unique(utctime, return_index=True)
365 366
366 367 if blockList.size != indices.size:
367 368 indMin = indices[blockList[0]]
368 369 if blockList[1] + 1 >= indices.size:
369 370 arrayData = arrayData[indMin:,:]
370 371 else:
371 372 indMax = indices[blockList[1] + 1]
372 373 arrayData = arrayData[indMin:indMax,:]
373 374 return arrayData
374 375
375 376 # One dimension
376 377 if nDims == 0:
377 378 arrayData = dataset.value.astype(numpy.float)[0][blockList]
378 379
379 380 # Two dimensions
380 381 elif nDims == 2:
381 382 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
382 383 newShapes = (blocksPerFile,nDim0)
383 384 nDatas = nDim1
384 385
385 386 for i in range(nDatas):
386 387 data = dataset[strds + str(i)].value
387 388 arrayData[:,i,:] = data[blockList,:]
388 389
389 390 # Three dimensions
390 391 else:
391 392 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
392 393 for i in range(nDatas):
393 394
394 395 data = dataset[strds + str(i)].value
395 396
396 397 for b in range(blockList.size):
397 398 arrayData[b,:,i,:] = data[:,:,blockList[b]]
398 399
399 400 return arrayData
400 401
401 402 def __setDataOut(self):
402 403 listMeta = self.listMeta
403 404 listMetaname = self.listMetaname
404 405 listDataname = self.listDataname
405 406 listData = self.listData
406 407 listShapes = self.listShapes
407 408
408 409 blockIndex = self.blockIndex
409 410 # blockList = self.blockList
410 411
411 412 for i in range(len(listMeta)):
412 413 setattr(self.dataOut,listMetaname[i],listMeta[i])
413 414
414 415 for j in range(len(listData)):
415 416 nShapes = listShapes[listDataname[j]][0]
416 417 mode = listShapes[listDataname[j]][4]
417 418 if nShapes == 1:
418 419 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
419 420 elif nShapes > 1:
420 421 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
421 422 elif mode==0:
422 423 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
423 424 #Mode Meteors
424 425 elif mode ==2:
425 426 selectedData = self.__selectDataMode2(listData[j], blockIndex)
426 427 setattr(self.dataOut, listDataname[j], selectedData)
427 428 return
428 429
429 430 def __selectDataMode2(self, data, blockIndex):
430 431 utctime = data[:,0]
431 432 aux, indices = numpy.unique(utctime, return_inverse=True)
432 433 selInd = numpy.where(indices == blockIndex)[0]
433 434 selData = data[selInd,:]
434 435
435 436 return selData
436 437
437 438 def getData(self):
438 439
439 440 if self.blockIndex==self.blocksPerFile:
440 441 if not( self.__setNextFileOffline() ):
441 442 self.dataOut.flagNoData = True
442 443 return 0
443 444
444 445 self.__setDataOut()
445 446 self.dataOut.flagNoData = False
446 447
447 448 self.blockIndex += 1
448 449
449 450 return
450 451
451 452 def run(self, **kwargs):
452 453
453 454 if not(self.isConfig):
454 455 self.setup(**kwargs)
455 456 self.isConfig = True
456 457
457 458 self.getData()
458 459
459 460 return
460 461
461 462 @MPDecorator
462 463 class ParamWriter(Operation):
463 464 '''
464 465 HDF5 Writer, stores parameters data in HDF5 format files
465 466
466 467 path: path where the files will be stored
467 468 blocksPerFile: number of blocks that will be saved in per HDF5 format file
468 469 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
469 470 metadataList: list of attributes that will be stored as metadata
470 471 dataList: list of attributes that will be stores as data
471 472 '''
472 473
473 474 ext = ".hdf5"
474 475 optchar = "D"
475 476 metaoptchar = "M"
476 477 metaFile = None
477 478 filename = None
478 479 path = None
479 480 setFile = None
480 481 fp = None
481 482 grp = None
482 483 ds = None
483 484 firsttime = True
484 485 #Configurations
485 486 blocksPerFile = None
486 487 blockIndex = None
487 488 dataOut = None
488 489 #Data Arrays
489 490 dataList = None
490 491 metadataList = None
491 492 dsList = None #List of dictionaries with dataset properties
492 493 tableDim = None
493 494 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
494 495 currentDay = None
495 496 lastTime = None
496 497 setType = None
497 498
498 499 def __init__(self):
499 500
500 501 Operation.__init__(self)
501 502 return
502 503
503 504 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
504 505 self.path = path
505 506 self.blocksPerFile = blocksPerFile
506 507 self.metadataList = metadataList
507 508 self.dataList = dataList
508 509 self.dataOut = dataOut
509 510 self.mode = mode
510 511 if self.mode is not None:
511 512 self.mode = numpy.zeros(len(self.dataList)) + mode
512 513 else:
513 514 self.mode = numpy.ones(len(self.dataList))
514 515
515 516 self.setType = setType
516 517
517 518 arrayDim = numpy.zeros((len(self.dataList),5))
518 519
519 520 #Table dimensions
520 521 dtype0 = self.dtype
521 522 tableList = []
522 523
523 524 #Dictionary and list of tables
524 525 dsList = []
525 526
526 527 for i in range(len(self.dataList)):
527 528 dsDict = {}
528 529 dataAux = getattr(self.dataOut, self.dataList[i])
529 530 dsDict['variable'] = self.dataList[i]
530 531 #--------------------- Conditionals ------------------------
531 532 #There is no data
532 533
533 534 if dataAux is None:
534 535
535 536 return 0
536 537
537 538 if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
538 539 dsDict['mode'] = 0
539 540 dsDict['nDim'] = 0
540 541 arrayDim[i,0] = 0
541 542 dsList.append(dsDict)
542 543
543 544 #Mode 2: meteors
544 545 elif self.mode[i] == 2:
545 546 dsDict['dsName'] = 'table0'
546 547 dsDict['mode'] = 2 # Mode meteors
547 548 dsDict['shape'] = dataAux.shape[-1]
548 549 dsDict['nDim'] = 0
549 550 dsDict['dsNumber'] = 1
550 551 arrayDim[i,3] = dataAux.shape[-1]
551 552 arrayDim[i,4] = self.mode[i] #Mode the data was stored
552 553 dsList.append(dsDict)
553 554
554 555 #Mode 1
555 556 else:
556 557 arrayDim0 = dataAux.shape #Data dimensions
557 558 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
558 559 arrayDim[i,4] = self.mode[i] #Mode the data was stored
559 560 strtable = 'table'
560 561 dsDict['mode'] = 1 # Mode parameters
561 562
562 563 # Three-dimension arrays
563 564 if len(arrayDim0) == 3:
564 565 arrayDim[i,1:-1] = numpy.array(arrayDim0)
565 566 nTables = int(arrayDim[i,2])
566 567 dsDict['dsNumber'] = nTables
567 568 dsDict['shape'] = arrayDim[i,2:4]
568 569 dsDict['nDim'] = 3
569 570
570 571 for j in range(nTables):
571 572 dsDict = dsDict.copy()
572 573 dsDict['dsName'] = strtable + str(j)
573 574 dsList.append(dsDict)
574 575
575 576 # Two-dimension arrays
576 577 elif len(arrayDim0) == 2:
577 578 arrayDim[i,2:-1] = numpy.array(arrayDim0)
578 579 nTables = int(arrayDim[i,2])
579 580 dsDict['dsNumber'] = nTables
580 581 dsDict['shape'] = arrayDim[i,3]
581 582 dsDict['nDim'] = 2
582 583
583 584 for j in range(nTables):
584 585 dsDict = dsDict.copy()
585 586 dsDict['dsName'] = strtable + str(j)
586 587 dsList.append(dsDict)
587 588
588 589 # One-dimension arrays
589 590 elif len(arrayDim0) == 1:
590 591 arrayDim[i,3] = arrayDim0[0]
591 592 dsDict['shape'] = arrayDim0[0]
592 593 dsDict['dsNumber'] = 1
593 594 dsDict['dsName'] = strtable + str(0)
594 595 dsDict['nDim'] = 1
595 596 dsList.append(dsDict)
596 597
597 598 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
598 599 tableList.append(table)
599 600
600 601 self.dsList = dsList
601 602 self.tableDim = numpy.array(tableList, dtype = dtype0)
602 603 self.blockIndex = 0
603 604 timeTuple = time.localtime(dataOut.utctime)
604 605 self.currentDay = timeTuple.tm_yday
605 606
606 607 def putMetadata(self):
607 608
608 609 fp = self.createMetadataFile()
609 610 self.writeMetadata(fp)
610 611 fp.close()
611 612 return
612 613
613 614 def createMetadataFile(self):
614 615 ext = self.ext
615 616 path = self.path
616 617 setFile = self.setFile
617 618
618 619 timeTuple = time.localtime(self.dataOut.utctime)
619 620
620 621 subfolder = ''
621 622 fullpath = os.path.join( path, subfolder )
622 623
623 624 if not( os.path.exists(fullpath) ):
624 625 os.mkdir(fullpath)
625 626 setFile = -1 #inicializo mi contador de seteo
626 627
627 628 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
628 629 fullpath = os.path.join( path, subfolder )
629 630
630 631 if not( os.path.exists(fullpath) ):
631 632 os.mkdir(fullpath)
632 633 setFile = -1 #inicializo mi contador de seteo
633 634
634 635 else:
635 636 filesList = os.listdir( fullpath )
636 637 filesList = sorted( filesList, key=str.lower )
637 638 if len( filesList ) > 0:
638 639 filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
639 640 filen = filesList[-1]
640 641 # el filename debera tener el siguiente formato
641 642 # 0 1234 567 89A BCDE (hex)
642 643 # x YYYY DDD SSS .ext
643 644 if isNumber( filen[8:11] ):
644 645 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
645 646 else:
646 647 setFile = -1
647 648 else:
648 649 setFile = -1 #inicializo mi contador de seteo
649 650
650 651 if self.setType is None:
651 652 setFile += 1
652 653 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
653 654 timeTuple.tm_year,
654 655 timeTuple.tm_yday,
655 656 setFile,
656 657 ext )
657 658 else:
658 659 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
659 660 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
660 661 timeTuple.tm_year,
661 662 timeTuple.tm_yday,
662 663 setFile,
663 664 ext )
664 665
665 666 filename = os.path.join( path, subfolder, file )
666 667 self.metaFile = file
667 668 #Setting HDF5 File
668 669 fp = h5py.File(filename,'w')
669 670
670 671 return fp
671 672
672 673 def writeMetadata(self, fp):
673 674
674 675 grp = fp.create_group("Metadata")
675 676 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
676 677
677 678 for i in range(len(self.metadataList)):
678 679 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
679 680 return
680 681
681 682 def timeFlag(self):
682 683 currentTime = self.dataOut.utctime
683 684
684 685 if self.lastTime is None:
685 686 self.lastTime = currentTime
686 687
687 688 #Day
688 689 timeTuple = time.localtime(currentTime)
689 690 dataDay = timeTuple.tm_yday
690 691
691 692 #Time
692 693 timeDiff = currentTime - self.lastTime
693 694
694 695 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
695 696 if dataDay != self.currentDay:
696 697 self.currentDay = dataDay
697 698 return True
698 699 elif timeDiff > 3*60*60:
699 700 self.lastTime = currentTime
700 701 return True
701 702 else:
702 703 self.lastTime = currentTime
703 704 return False
704 705
705 706 def setNextFile(self):
706 707
707 708 ext = self.ext
708 709 path = self.path
709 710 setFile = self.setFile
710 711 mode = self.mode
711 712
712 713 timeTuple = time.localtime(self.dataOut.utctime)
713 714 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
714 715
715 716 fullpath = os.path.join( path, subfolder )
716 717
717 718 if os.path.exists(fullpath):
718 719 filesList = os.listdir( fullpath )
719 720 filesList = [k for k in filesList if 'M' in k]
720 721 if len( filesList ) > 0:
721 722 filesList = sorted( filesList, key=str.lower )
722 723 filen = filesList[-1]
723 724 # el filename debera tener el siguiente formato
724 725 # 0 1234 567 89A BCDE (hex)
725 726 # x YYYY DDD SSS .ext
726 727 if isNumber( filen[8:11] ):
727 728 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
728 729 else:
729 730 setFile = -1
730 731 else:
731 732 setFile = -1 #inicializo mi contador de seteo
732 733 else:
733 734 os.makedirs(fullpath)
734 735 setFile = -1 #inicializo mi contador de seteo
735 736
736 737 if self.setType is None:
737 738 setFile += 1
738 739 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
739 740 timeTuple.tm_year,
740 741 timeTuple.tm_yday,
741 742 setFile,
742 743 ext )
743 744 else:
744 745 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
745 746 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
746 747 timeTuple.tm_year,
747 748 timeTuple.tm_yday,
748 749 setFile,
749 750 ext )
750 751
751 752 filename = os.path.join( path, subfolder, file )
752 753
753 754 #Setting HDF5 File
754 755 fp = h5py.File(filename,'w')
755 756 #write metadata
756 757 self.writeMetadata(fp)
757 758 #Write data
758 759 grp = fp.create_group("Data")
759 760 ds = []
760 761 data = []
761 762 dsList = self.dsList
762 763 i = 0
763 764 while i < len(dsList):
764 765 dsInfo = dsList[i]
765 766 #One-dimension data
766 767 if dsInfo['mode'] == 0:
767 768 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
768 769 ds.append(ds0)
769 770 data.append([])
770 771 i += 1
771 772 continue
772 773
773 774 elif dsInfo['mode'] == 2:
774 775 grp0 = grp.create_group(dsInfo['variable'])
775 776 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
776 777 ds.append(ds0)
777 778 data.append([])
778 779 i += 1
779 780 continue
780 781
781 782 elif dsInfo['mode'] == 1:
782 783 grp0 = grp.create_group(dsInfo['variable'])
783 784
784 785 for j in range(dsInfo['dsNumber']):
785 786 dsInfo = dsList[i]
786 787 tableName = dsInfo['dsName']
787 788
788 789
789 790 if dsInfo['nDim'] == 3:
790 791 shape = dsInfo['shape'].astype(int)
791 792 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
792 793 else:
793 794 shape = int(dsInfo['shape'])
794 795 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
795 796
796 797 ds.append(ds0)
797 798 data.append([])
798 799 i += 1
799 800
800 801 fp.flush()
801 802 fp.close()
802 803
803 804 log.log('creating file: {}'.format(filename), 'Writing')
804 805 self.filename = filename
805 806 self.ds = ds
806 807 self.data = data
807 808 self.firsttime = True
808 809 self.blockIndex = 0
809 810 return
810 811
811 812 def putData(self):
812 813
813 814 if self.blockIndex == self.blocksPerFile or self.timeFlag():
814 815 self.setNextFile()
815 816
816 817 self.readBlock()
817 818 self.setBlock() #Prepare data to be written
818 819 self.writeBlock() #Write data
819 820
820 821 return
821 822
822 823 def readBlock(self):
823 824
824 825 '''
825 826 data Array configured
826 827
827 828
828 829 self.data
829 830 '''
830 831 dsList = self.dsList
831 832 ds = self.ds
832 833 #Setting HDF5 File
833 834 fp = h5py.File(self.filename,'r+')
834 835 grp = fp["Data"]
835 836 ind = 0
836 837
837 838 while ind < len(dsList):
838 839 dsInfo = dsList[ind]
839 840
840 841 if dsInfo['mode'] == 0:
841 842 ds0 = grp[dsInfo['variable']]
842 843 ds[ind] = ds0
843 844 ind += 1
844 845 else:
845 846
846 847 grp0 = grp[dsInfo['variable']]
847 848
848 849 for j in range(dsInfo['dsNumber']):
849 850 dsInfo = dsList[ind]
850 851 ds0 = grp0[dsInfo['dsName']]
851 852 ds[ind] = ds0
852 853 ind += 1
853 854
854 855 self.fp = fp
855 856 self.grp = grp
856 857 self.ds = ds
857 858
858 859 return
859 860
860 861 def setBlock(self):
861 862 '''
862 863 data Array configured
863 864
864 865
865 866 self.data
866 867 '''
867 868 #Creating Arrays
868 869 dsList = self.dsList
869 870 data = self.data
870 871 ind = 0
871 872
872 873 while ind < len(dsList):
873 874 dsInfo = dsList[ind]
874 875 dataAux = getattr(self.dataOut, dsInfo['variable'])
875 876
876 877 mode = dsInfo['mode']
877 878 nDim = dsInfo['nDim']
878 879
879 880 if mode == 0 or mode == 2 or nDim == 1:
880 881 data[ind] = dataAux
881 882 ind += 1
882 883 # elif nDim == 1:
883 884 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
884 885 # ind += 1
885 886 elif nDim == 2:
886 887 for j in range(dsInfo['dsNumber']):
887 888 data[ind] = dataAux[j,:]
888 889 ind += 1
889 890 elif nDim == 3:
890 891 for j in range(dsInfo['dsNumber']):
891 892 data[ind] = dataAux[:,j,:]
892 893 ind += 1
893 894
894 895 self.data = data
895 896 return
896 897
897 898 def writeBlock(self):
898 899 '''
899 900 Saves the block in the HDF5 file
900 901 '''
901 902 dsList = self.dsList
902 903
903 904 for i in range(len(self.ds)):
904 905 dsInfo = dsList[i]
905 906 nDim = dsInfo['nDim']
906 907 mode = dsInfo['mode']
907 908
908 909 # First time
909 910 if self.firsttime:
910 911 if type(self.data[i]) == numpy.ndarray:
911 912
912 913 if nDim == 3:
913 914 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
914 915 self.ds[i].resize(self.data[i].shape)
915 916 if mode == 2:
916 917 self.ds[i].resize(self.data[i].shape)
917 918 self.ds[i][:] = self.data[i]
918 919 else:
919 920
920 921 # From second time
921 922 # Meteors!
922 923 if mode == 2:
923 924 dataShape = self.data[i].shape
924 925 dsShape = self.ds[i].shape
925 926 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
926 927 self.ds[i][dsShape[0]:,:] = self.data[i]
927 928 # No dimension
928 929 elif mode == 0:
929 930 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
930 931 self.ds[i][0,-1] = self.data[i]
931 932 # One dimension
932 933 elif nDim == 1:
933 934 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
934 935 self.ds[i][-1,:] = self.data[i]
935 936 # Two dimension
936 937 elif nDim == 2:
937 938 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
938 939 self.ds[i][self.blockIndex,:] = self.data[i]
939 940 # Three dimensions
940 941 elif nDim == 3:
941 942 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
942 943 self.ds[i][:,:,-1] = self.data[i]
943 944
944 945 self.firsttime = False
945 946 self.blockIndex += 1
946 947
947 948 #Close to save changes
948 949 self.fp.flush()
949 950 self.fp.close()
950 951 return
951 952
952 953 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
953 954
954 955 self.dataOut = dataOut
955 956 if not(self.isConfig):
956 957 self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
957 958 metadataList=metadataList, dataList=dataList, mode=mode,
958 959 setType=setType)
959 960
960 961 self.isConfig = True
961 962 self.setNextFile()
962 963
963 964 self.putData()
964 965 return
965 966
966 967
967 968 @MPDecorator
968 969 class ParameterReader(JRODataReader,ProcessingUnit):
969 970 '''
970 971 Reads HDF5 format files
971 972 '''
972 973
973 974 ext = ".hdf5"
974 975 optchar = "D"
975 976 timezone = None
976 977 startTime = None
977 978 endTime = None
978 979 fileIndex = None
979 980 blockList = None #List to blocks to be read from the file
980 981 blocksPerFile = None #Number of blocks to be read
981 982 blockIndex = None
982 983 path = None
983 984 #List of Files
984 985 filenameList = None
985 986 datetimeList = None
986 987 #Hdf5 File
987 988 listMetaname = None
988 989 listMeta = None
989 990 listDataname = None
990 991 listData = None
991 992 listShapes = None
992 993 fp = None
993 994 #dataOut reconstruction
994 995 dataOut = None
995 996
996 997 def __init__(self):
997 998 ProcessingUnit.__init__(self)
998 999 self.dataOut = Parameters()
999 1000 return
1000 1001
1001 1002 def setup(self, **kwargs):
1002 1003
1003 1004 path = kwargs['path']
1004 1005 startDate = kwargs['startDate']
1005 1006 endDate = kwargs['endDate']
1006 1007 startTime = kwargs['startTime']
1007 1008 endTime = kwargs['endTime']
1008 1009 walk = kwargs['walk']
1009 1010 if 'ext' in kwargs:
1010 1011 ext = kwargs['ext']
1011 1012 else:
1012 1013 ext = '.hdf5'
1013 1014 if 'timezone' in kwargs:
1014 1015 self.timezone = kwargs['timezone']
1015 1016 else:
1016 1017 self.timezone = 'lt'
1017 1018
1018 1019 print("[Reading] Searching files in offline mode ...")
1019 1020 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1020 1021 startTime=startTime, endTime=endTime,
1021 1022 ext=ext, walk=walk)
1022 1023
1023 1024 if not(filenameList):
1024 1025 print("There is no files into the folder: %s"%(path))
1025 1026 sys.exit(-1)
1026 1027
1027 1028 self.fileIndex = -1
1028 1029 self.startTime = startTime
1029 1030 self.endTime = endTime
1030 1031 self.__readMetadata()
1031 1032 self.__setNextFileOffline()
1032 1033
1033 1034 return
1034 1035
1035 1036 def searchFilesOffLine(self, path, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), ext='.hdf5', walk=True):
1036 1037
1037 1038 expLabel = ''
1038 1039 self.filenameList = []
1039 1040 self.datetimeList = []
1040 1041 pathList = []
1041 1042 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
1042 1043
1043 1044 if dateList == []:
1044 1045 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
1045 1046 datetime.datetime.combine(startDate,startTime).ctime(),
1046 1047 datetime.datetime.combine(endDate,endTime).ctime()))
1047 1048
1048 1049 return None, None
1049 1050
1050 1051 if len(dateList) > 1:
1051 1052 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
1052 1053 else:
1053 1054 print("[Reading] data was found for the date %s" %(dateList[0]))
1054 1055
1055 1056 filenameList = []
1056 1057 datetimeList = []
1057 1058
1058 1059 for thisPath in pathList:
1059 1060
1060 1061 fileList = glob.glob1(thisPath, "*%s" %ext)
1061 1062 fileList.sort()
1062 1063
1063 1064 for file in fileList:
1064 1065
1065 1066 filename = os.path.join(thisPath,file)
1066 1067
1067 1068 if not isFileInDateRange(filename, startDate, endDate):
1068 1069 continue
1069 1070
1070 1071 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
1071 1072
1072 1073 if not(thisDatetime):
1073 1074 continue
1074 1075
1075 1076 filenameList.append(filename)
1076 1077 datetimeList.append(thisDatetime)
1077 1078
1078 1079 if not(filenameList):
1079 1080 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
1080 1081 return None, None
1081 1082
1082 1083 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
1083 1084 print()
1084 1085
1085 1086 self.filenameList = filenameList
1086 1087 self.datetimeList = datetimeList
1087 1088
1088 1089 return pathList, filenameList
1089 1090
1090 1091 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
1091 1092
1092 1093 """
1093 1094 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
1094 1095
1095 1096 Inputs:
1096 1097 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
1097 1098 startDate : fecha inicial del rango seleccionado en formato datetime.date
1098 1099 endDate : fecha final del rango seleccionado en formato datetime.date
1099 1100 startTime : tiempo inicial del rango seleccionado en formato datetime.time
1100 1101 endTime : tiempo final del rango seleccionado en formato datetime.time
1101 1102
1102 1103 Return:
1103 1104 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
1104 1105 fecha especificado, de lo contrario retorna False.
1105 1106
1106 1107 Excepciones:
1107 1108 Si el archivo no existe o no puede ser abierto
1108 1109 Si la cabecera no puede ser leida.
1109 1110
1110 1111 """
1111 1112
1112 1113 try:
1113 1114 fp = h5py.File(filename, 'r')
1114 1115 grp1 = fp['Data']
1115 1116
1116 1117 except IOError:
1117 1118 traceback.print_exc()
1118 1119 raise IOError("The file %s can't be opened" %(filename))
1119 1120 #In case has utctime attribute
1120 1121 grp2 = grp1['utctime']
1121 1122 thisUtcTime = grp2.value[0]
1122 1123
1123 1124 fp.close()
1124 1125
1125 1126 if self.timezone == 'lt':
1126 1127 thisUtcTime -= 5*3600
1127 1128
1128 1129 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600)
1129 1130 thisDate = thisDatetime.date()
1130 1131 thisTime = thisDatetime.time()
1131 1132
1132 1133 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
1133 1134 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
1134 1135
1135 1136 #General case
1136 1137 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
1137 1138 #-----------o----------------------------o-----------
1138 1139 # startTime endTime
1139 1140
1140 1141 if endTime >= startTime:
1141 1142 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
1142 1143 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
1143 1144 return thisDatetime
1144 1145 return None
1145 1146
1146 1147 #If endTime < startTime then endTime belongs to the next day
1147 1148 #<<<<<<<<<<<o o>>>>>>>>>>>
1148 1149 #-----------o----------------------------o-----------
1149 1150 # endTime startTime
1150 1151
1151 1152 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
1152 1153 return None
1153 1154
1154 1155 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
1155 1156 return None
1156 1157
1157 1158 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
1158 1159 return None
1159 1160
1160 1161 return thisDatetime
1161 1162
1162 1163 def __setNextFileOffline(self):
1163 1164
1164 1165 self.fileIndex += 1
1165 1166 idFile = self.fileIndex
1166 1167
1167 1168 if not(idFile < len(self.filenameList)):
1168 self.dataOut.error = 'No more files'
1169 return 0
1169 raise schainpy.admin.SchainError('No more files')
1170 1170
1171 1171 filename = self.filenameList[idFile]
1172 1172 self.fp = h5py.File(filename, 'r')
1173 1173 self.filename = filename
1174 1174
1175 1175 print("Setting the file: %s"%self.filename)
1176 1176
1177 1177 self.__setBlockList()
1178 1178 self.__readData()
1179 1179 self.blockIndex = 0
1180 1180 return 1
1181 1181
1182 1182 def __setBlockList(self):
1183 1183 '''
1184 1184 Selects the data within the times defined
1185 1185
1186 1186 self.fp
1187 1187 self.startTime
1188 1188 self.endTime
1189 1189 self.blockList
1190 1190 self.blocksPerFile
1191 1191
1192 1192 '''
1193 1193 fp = self.fp
1194 1194 startTime = self.startTime
1195 1195 endTime = self.endTime
1196 1196
1197 1197 grp = fp['Data']
1198 1198 thisUtcTime = grp['utctime'].value
1199 1199
1200 1200 if self.timezone == 'lt':
1201 1201 thisUtcTime -= 5*3600
1202 1202
1203 1203 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
1204 1204
1205 1205 thisDate = thisDatetime.date()
1206 1206 thisTime = thisDatetime.time()
1207 1207
1208 1208 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
1209 1209 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
1210 1210
1211 1211 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
1212 1212
1213 1213 self.blockList = ind
1214 1214 self.blocksPerFile = len(ind)
1215 1215 return
1216 1216
1217 1217 def __readMetadata(self):
1218 1218 '''
1219 1219 Reads Metadata
1220 1220 '''
1221 1221
1222 1222 filename = self.filenameList[0]
1223 1223 fp = h5py.File(filename, 'r')
1224 1224 gp = fp['Metadata']
1225 1225 listMetaname = []
1226 1226 listMetadata = []
1227 1227
1228 1228 for item in list(gp.items()):
1229 1229 name = item[0]
1230 1230
1231 1231 if name=='variables':
1232 1232 table = gp[name][:]
1233 1233 listShapes = {}
1234 1234 for shapes in table:
1235 1235 listShapes[shapes[0].decode()] = numpy.array([shapes[1]])
1236 1236 else:
1237 1237 data = gp[name].value
1238 1238 listMetaname.append(name)
1239 1239 listMetadata.append(data)
1240 1240
1241 1241 self.listShapes = listShapes
1242 1242 self.listMetaname = listMetaname
1243 1243 self.listMeta = listMetadata
1244 1244
1245 1245 fp.close()
1246 1246 return
1247 1247
1248 1248 def __readData(self):
1249 1249
1250 1250 grp = self.fp['Data']
1251 1251 listdataname = []
1252 1252 listdata = []
1253 1253
1254 1254 for item in list(grp.items()):
1255 1255 name = item[0]
1256 1256 listdataname.append(name)
1257 1257 dim = self.listShapes[name][0]
1258 1258 if dim == 0:
1259 1259 array = grp[name].value
1260 1260 else:
1261 1261 array = []
1262 1262 for i in range(dim):
1263 1263 array.append(grp[name]['table{:02d}'.format(i)].value)
1264 1264 array = numpy.array(array)
1265 1265
1266 1266 listdata.append(array)
1267 1267
1268 1268 self.listDataname = listdataname
1269 1269 self.listData = listdata
1270 1270 return
1271 1271
1272 1272 def getData(self):
1273 1273
1274 1274 for i in range(len(self.listMeta)):
1275 1275 setattr(self.dataOut, self.listMetaname[i], self.listMeta[i])
1276 1276
1277 1277 for j in range(len(self.listData)):
1278 1278 dim = self.listShapes[self.listDataname[j]][0]
1279 1279 if dim == 0:
1280 1280 setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex])
1281 1281 else:
1282 1282 setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex])
1283 1283
1284 1284 self.dataOut.flagNoData = False
1285 1285 self.blockIndex += 1
1286 1286
1287 1287 return
1288 1288
1289 1289 def run(self, **kwargs):
1290 1290
1291 1291 if not(self.isConfig):
1292 1292 self.setup(**kwargs)
1293 1293 self.isConfig = True
1294 1294
1295 1295 if self.blockIndex == self.blocksPerFile:
1296 1296 if not(self.__setNextFileOffline()):
1297 1297 self.dataOut.flagNoData = True
1298 1298 return 0
1299 1299
1300 1300 self.getData()
1301 1301
1302 1302 return
1303 1303
1304 1304 @MPDecorator
1305 1305 class ParameterWriter(Operation):
1306 1306 '''
1307 1307 HDF5 Writer, stores parameters data in HDF5 format files
1308 1308
1309 1309 path: path where the files will be stored
1310 1310 blocksPerFile: number of blocks that will be saved in per HDF5 format file
1311 1311 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
1312 1312 metadataList: list of attributes that will be stored as metadata
1313 1313 dataList: list of attributes that will be stores as data
1314 1314 '''
1315 1315
1316 1316
1317 1317 ext = ".hdf5"
1318 1318 optchar = "D"
1319 1319 metaoptchar = "M"
1320 1320 metaFile = None
1321 1321 filename = None
1322 1322 path = None
1323 1323 setFile = None
1324 1324 fp = None
1325 1325 grp = None
1326 1326 ds = None
1327 1327 firsttime = True
1328 1328 #Configurations
1329 1329 blocksPerFile = None
1330 1330 blockIndex = None
1331 1331 dataOut = None
1332 1332 #Data Arrays
1333 1333 dataList = None
1334 1334 metadataList = None
1335 1335 dsList = None #List of dictionaries with dataset properties
1336 1336 tableDim = None
1337 1337 dtype = [('name', 'S20'),('nDim', 'i')]
1338 1338 currentDay = None
1339 1339 lastTime = None
1340 1340
1341 1341 def __init__(self):
1342 1342
1343 1343 Operation.__init__(self)
1344 1344 return
1345 1345
1346 1346 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
1347 1347 self.path = path
1348 1348 self.blocksPerFile = blocksPerFile
1349 1349 self.metadataList = metadataList
1350 1350 self.dataList = dataList
1351 1351 self.setType = setType
1352 1352
1353 1353 tableList = []
1354 1354 dsList = []
1355 1355
1356 1356 for i in range(len(self.dataList)):
1357 1357 dsDict = {}
1358 1358 dataAux = getattr(self.dataOut, self.dataList[i])
1359 1359 dsDict['variable'] = self.dataList[i]
1360 1360
1361 1361 if dataAux is None:
1362 1362 continue
1363 1363 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
1364 1364 dsDict['nDim'] = 0
1365 1365 else:
1366 1366 dsDict['nDim'] = len(dataAux.shape)
1367 1367 dsDict['shape'] = dataAux.shape
1368 1368 dsDict['dsNumber'] = dataAux.shape[0]
1369 1369
1370 1370 dsList.append(dsDict)
1371 1371 tableList.append((self.dataList[i], dsDict['nDim']))
1372 1372
1373 1373 self.dsList = dsList
1374 1374 self.tableDim = numpy.array(tableList, dtype=self.dtype)
1375 1375 self.currentDay = self.dataOut.datatime.date()
1376 1376
1377 1377 def timeFlag(self):
1378 1378 currentTime = self.dataOut.utctime
1379 1379 timeTuple = time.localtime(currentTime)
1380 1380 dataDay = timeTuple.tm_yday
1381 1381
1382 1382 if self.lastTime is None:
1383 1383 self.lastTime = currentTime
1384 1384 self.currentDay = dataDay
1385 1385 return False
1386 1386
1387 1387 timeDiff = currentTime - self.lastTime
1388 1388
1389 1389 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
1390 1390 if dataDay != self.currentDay:
1391 1391 self.currentDay = dataDay
1392 1392 return True
1393 1393 elif timeDiff > 3*60*60:
1394 1394 self.lastTime = currentTime
1395 1395 return True
1396 1396 else:
1397 1397 self.lastTime = currentTime
1398 1398 return False
1399 1399
1400 1400 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None):
1401 1401
1402 1402 self.dataOut = dataOut
1403 1403 if not(self.isConfig):
1404 1404 self.setup(path=path, blocksPerFile=blocksPerFile,
1405 1405 metadataList=metadataList, dataList=dataList,
1406 1406 setType=setType)
1407 1407
1408 1408 self.isConfig = True
1409 1409 self.setNextFile()
1410 1410
1411 1411 self.putData()
1412 1412 return
1413 1413
1414 1414 def setNextFile(self):
1415 1415
1416 1416 ext = self.ext
1417 1417 path = self.path
1418 1418 setFile = self.setFile
1419 1419
1420 1420 timeTuple = time.localtime(self.dataOut.utctime)
1421 1421 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1422 1422 fullpath = os.path.join(path, subfolder)
1423 1423
1424 1424 if os.path.exists(fullpath):
1425 1425 filesList = os.listdir(fullpath)
1426 1426 filesList = [k for k in filesList if k.startswith(self.optchar)]
1427 1427 if len( filesList ) > 0:
1428 1428 filesList = sorted(filesList, key=str.lower)
1429 1429 filen = filesList[-1]
1430 1430 # el filename debera tener el siguiente formato
1431 1431 # 0 1234 567 89A BCDE (hex)
1432 1432 # x YYYY DDD SSS .ext
1433 1433 if isNumber(filen[8:11]):
1434 1434 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
1435 1435 else:
1436 1436 setFile = -1
1437 1437 else:
1438 1438 setFile = -1 #inicializo mi contador de seteo
1439 1439 else:
1440 1440 os.makedirs(fullpath)
1441 1441 setFile = -1 #inicializo mi contador de seteo
1442 1442
1443 1443 if self.setType is None:
1444 1444 setFile += 1
1445 1445 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
1446 1446 timeTuple.tm_year,
1447 1447 timeTuple.tm_yday,
1448 1448 setFile,
1449 1449 ext )
1450 1450 else:
1451 1451 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
1452 1452 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
1453 1453 timeTuple.tm_year,
1454 1454 timeTuple.tm_yday,
1455 1455 setFile,
1456 1456 ext )
1457 1457
1458 1458 self.filename = os.path.join( path, subfolder, file )
1459 1459
1460 1460 #Setting HDF5 File
1461 1461 self.fp = h5py.File(self.filename, 'w')
1462 1462 #write metadata
1463 1463 self.writeMetadata(self.fp)
1464 1464 #Write data
1465 1465 self.writeData(self.fp)
1466 1466
1467 1467 def writeMetadata(self, fp):
1468 1468
1469 1469 grp = fp.create_group("Metadata")
1470 1470 grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype)
1471 1471
1472 1472 for i in range(len(self.metadataList)):
1473 1473 if not hasattr(self.dataOut, self.metadataList[i]):
1474 1474 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
1475 1475 continue
1476 1476 value = getattr(self.dataOut, self.metadataList[i])
1477 1477 grp.create_dataset(self.metadataList[i], data=value)
1478 1478 return
1479 1479
1480 1480 def writeData(self, fp):
1481 1481
1482 1482 grp = fp.create_group("Data")
1483 1483 dtsets = []
1484 1484 data = []
1485 1485
1486 1486 for dsInfo in self.dsList:
1487 1487 if dsInfo['nDim'] == 0:
1488 1488 ds = grp.create_dataset(
1489 1489 dsInfo['variable'],
1490 1490 (self.blocksPerFile, ),
1491 1491 chunks=True,
1492 1492 dtype=numpy.float64)
1493 1493 dtsets.append(ds)
1494 1494 data.append((dsInfo['variable'], -1))
1495 1495 else:
1496 1496 sgrp = grp.create_group(dsInfo['variable'])
1497 1497 for i in range(dsInfo['dsNumber']):
1498 1498 ds = sgrp.create_dataset(
1499 1499 'table{:02d}'.format(i),
1500 1500 (self.blocksPerFile, ) + dsInfo['shape'][1:],
1501 1501 chunks=True)
1502 1502 dtsets.append(ds)
1503 1503 data.append((dsInfo['variable'], i))
1504 1504 fp.flush()
1505 1505
1506 1506 log.log('Creating file: {}'.format(fp.filename), self.name)
1507 1507
1508 1508 self.ds = dtsets
1509 1509 self.data = data
1510 1510 self.firsttime = True
1511 1511 self.blockIndex = 0
1512 1512 return
1513 1513
1514 1514 def putData(self):
1515 1515
1516 1516 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
1517 1517 self.closeFile()
1518 1518 self.setNextFile()
1519 1519
1520 1520 for i, ds in enumerate(self.ds):
1521 1521 attr, ch = self.data[i]
1522 1522 if ch == -1:
1523 1523 ds[self.blockIndex] = getattr(self.dataOut, attr)
1524 1524 else:
1525 1525 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
1526 1526
1527 1527 self.fp.flush()
1528 1528 self.blockIndex += 1
1529 1529 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
1530 1530
1531 1531 return
1532 1532
1533 1533 def closeFile(self):
1534 1534
1535 1535 if self.blockIndex != self.blocksPerFile:
1536 1536 for ds in self.ds:
1537 1537 ds.resize(self.blockIndex, axis=0)
1538 1538
1539 1539 self.fp.flush()
1540 1540 self.fp.close()
1541 1541
1542 1542 def close(self):
1543 1543
1544 1544 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,678 +1,675
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import numpy
7 7
8 8 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 11 from schainpy.model.data.jrodata import Spectra
12 12 from schainpy.utils import log
13 13
14 14 @MPDecorator
15 15 class SpectraReader(JRODataReader, ProcessingUnit):
16 16 """
17 17 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
18 18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
19 19 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
20 20
21 21 paresCanalesIguales * alturas * perfiles (Self Spectra)
22 22 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
23 23 canales * alturas (DC Channels)
24 24
25 25 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
26 26 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
27 27 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
28 28 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
29 29
30 30 Example:
31 31 dpath = "/home/myuser/data"
32 32
33 33 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
34 34
35 35 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
36 36
37 37 readerObj = SpectraReader()
38 38
39 39 readerObj.setup(dpath, startTime, endTime)
40 40
41 41 while(True):
42 42
43 43 readerObj.getData()
44 44
45 45 print readerObj.data_spc
46 46
47 47 print readerObj.data_cspc
48 48
49 49 print readerObj.data_dc
50 50
51 51 if readerObj.flagNoMoreFiles:
52 52 break
53 53
54 54 """
55 55
56 56 pts2read_SelfSpectra = 0
57 57
58 58 pts2read_CrossSpectra = 0
59 59
60 60 pts2read_DCchannels = 0
61 61
62 62 ext = ".pdata"
63 63
64 64 optchar = "P"
65 65
66 66 dataOut = None
67 67
68 68 nRdChannels = None
69 69
70 70 nRdPairs = None
71 71
72 72 rdPairList = []
73 73
74 74 def __init__(self):#, **kwargs):
75 75 """
76 76 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
77 77
78 78 Inputs:
79 79 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
80 80 almacenar un perfil de datos cada vez que se haga un requerimiento
81 81 (getData). El perfil sera obtenido a partir del buffer de datos,
82 82 si el buffer esta vacio se hara un nuevo proceso de lectura de un
83 83 bloque de datos.
84 84 Si este parametro no es pasado se creara uno internamente.
85 85
86 86 Affected:
87 87 self.dataOut
88 88
89 89 Return : None
90 90 """
91 91
92 92 #Eliminar de la base la herencia
93 93 ProcessingUnit.__init__(self)#, **kwargs)
94 94
95 # self.isConfig = False
96 95
97 96 self.pts2read_SelfSpectra = 0
98 97
99 98 self.pts2read_CrossSpectra = 0
100 99
101 100 self.pts2read_DCchannels = 0
102 101
103 102 self.datablock = None
104 103
105 104 self.utc = None
106 105
107 106 self.ext = ".pdata"
108 107
109 108 self.optchar = "P"
110 109
111 110 self.basicHeaderObj = BasicHeader(LOCALTIME)
112 111
113 112 self.systemHeaderObj = SystemHeader()
114 113
115 114 self.radarControllerHeaderObj = RadarControllerHeader()
116 115
117 116 self.processingHeaderObj = ProcessingHeader()
118 117
119 118 self.online = 0
120 119
121 120 self.fp = None
122 121
123 122 self.idFile = None
124 123
125 124 self.dtype = None
126 125
127 126 self.fileSizeByHeader = None
128 127
129 128 self.filenameList = []
130 129
131 130 self.filename = None
132 131
133 132 self.fileSize = None
134 133
135 134 self.firstHeaderSize = 0
136 135
137 136 self.basicHeaderSize = 24
138 137
139 138 self.pathList = []
140 139
141 140 self.lastUTTime = 0
142 141
143 142 self.maxTimeStep = 30
144 143
145 144 self.flagNoMoreFiles = 0
146 145
147 146 self.set = 0
148 147
149 148 self.path = None
150 149
151 150 self.delay = 60 #seconds
152 151
153 152 self.nTries = 3 #quantity tries
154 153
155 154 self.nFiles = 3 #number of files for searching
156 155
157 156 self.nReadBlocks = 0
158 157
159 158 self.flagIsNewFile = 1
160 159
161 160 self.__isFirstTimeOnline = 1
162 161
163 # self.ippSeconds = 0
164 162
165 163 self.flagDiscontinuousBlock = 0
166 164
167 165 self.flagIsNewBlock = 0
168 166
169 167 self.nTotalBlocks = 0
170 168
171 169 self.blocksize = 0
172 170
173 171 self.dataOut = self.createObjByDefault()
174 172
175 173 self.profileIndex = 1 #Always
176 174
177 175
178 176 def createObjByDefault(self):
179 177
180 178 dataObj = Spectra()
181 179
182 180 return dataObj
183 181
184 182 def __hasNotDataInBuffer(self):
185 183 return 1
186 184
187 185
188 186 def getBlockDimension(self):
189 187 """
190 188 Obtiene la cantidad de puntos a leer por cada bloque de datos
191 189
192 190 Affected:
193 191 self.nRdChannels
194 192 self.nRdPairs
195 193 self.pts2read_SelfSpectra
196 194 self.pts2read_CrossSpectra
197 195 self.pts2read_DCchannels
198 196 self.blocksize
199 197 self.dataOut.nChannels
200 198 self.dataOut.nPairs
201 199
202 200 Return:
203 201 None
204 202 """
205 203 self.nRdChannels = 0
206 204 self.nRdPairs = 0
207 205 self.rdPairList = []
208 206
209 207 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
210 208 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
211 209 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
212 210 else:
213 211 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
214 212 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
215 213
216 214 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
217 215
218 216 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
219 217 self.blocksize = self.pts2read_SelfSpectra
220 218
221 219 if self.processingHeaderObj.flag_cspc:
222 220 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
223 221 self.blocksize += self.pts2read_CrossSpectra
224 222
225 223 if self.processingHeaderObj.flag_dc:
226 224 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
227 225 self.blocksize += self.pts2read_DCchannels
228 226
229 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
227 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
230 228
231 229
232 230 def readBlock(self):
233 231 """
234 232 Lee el bloque de datos desde la posicion actual del puntero del archivo
235 233 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
236 234 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
237 235 es seteado a 0
238 236
239 237 Return: None
240 238
241 239 Variables afectadas:
242 240
243 241 self.flagIsNewFile
244 242 self.flagIsNewBlock
245 243 self.nTotalBlocks
246 244 self.data_spc
247 245 self.data_cspc
248 246 self.data_dc
249 247
250 248 Exceptions:
251 249 Si un bloque leido no es un bloque valido
252 250 """
253 251 blockOk_flag = False
254 252 fpointer = self.fp.tell()
255 253
256 254 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
257 255 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
258 256
259 257 if self.processingHeaderObj.flag_cspc:
260 258 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
261 259 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
262 260
263 261 if self.processingHeaderObj.flag_dc:
264 262 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
265 263 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
266 264
267 265
268 266 if not self.processingHeaderObj.shif_fft:
269 267 #desplaza a la derecha en el eje 2 determinadas posiciones
270 268 shift = int(self.processingHeaderObj.profilesPerBlock/2)
271 269 spc = numpy.roll( spc, shift , axis=2 )
272 270
273 271 if self.processingHeaderObj.flag_cspc:
274 272 #desplaza a la derecha en el eje 2 determinadas posiciones
275 273 cspc = numpy.roll( cspc, shift, axis=2 )
276 274
277 275 #Dimensions : nChannels, nProfiles, nSamples
278 276 spc = numpy.transpose( spc, (0,2,1) )
279 277 self.data_spc = spc
280 278
281 279 if self.processingHeaderObj.flag_cspc:
282 280 cspc = numpy.transpose( cspc, (0,2,1) )
283 281 self.data_cspc = cspc['real'] + cspc['imag']*1j
284 282 else:
285 283 self.data_cspc = None
286 284
287 285 if self.processingHeaderObj.flag_dc:
288 286 self.data_dc = dc['real'] + dc['imag']*1j
289 287 else:
290 288 self.data_dc = None
291 289
292 290 self.flagIsNewFile = 0
293 291 self.flagIsNewBlock = 1
294 292
295 293 self.nTotalBlocks += 1
296 294 self.nReadBlocks += 1
297 295
298 296 return 1
299 297
300 298 def getFirstHeader(self):
301 299
302 300 self.getBasicHeader()
303 301
304 302 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
305 303
306 304 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
307 305
308 306 # self.dataOut.ippSeconds = self.ippSeconds
309 307
310 308 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
311 309
312 310 self.dataOut.dtype = self.dtype
313 311
314 312 # self.dataOut.nPairs = self.nPairs
315 313
316 314 self.dataOut.pairsList = self.rdPairList
317 315
318 316 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
319 317
320 318 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
321 319
322 320 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
323 321
324 322 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
325 323
326 324 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
327 325
328 326 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
329 327
330 328 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
331 329
332 330 self.dataOut.flagShiftFFT = True #Data is always shifted
333 331
334 332 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
335 333
336 334 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
337 335
338 336 def getData(self):
339 337 """
340 338 First method to execute before "RUN" is called.
341 339
342 340 Copia el buffer de lectura a la clase "Spectra",
343 341 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
344 342 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
345 343
346 344 Return:
347 345 0 : Si no hay mas archivos disponibles
348 346 1 : Si hizo una buena copia del buffer
349 347
350 348 Affected:
351 349 self.dataOut
352 350
353 351 self.flagDiscontinuousBlock
354 352 self.flagIsNewBlock
355 353 """
356 354
357 355 if self.flagNoMoreFiles:
358 356 self.dataOut.flagNoData = True
359 print('Process finished')
360 357 return 0
361 358
362 359 self.flagDiscontinuousBlock = 0
363 360 self.flagIsNewBlock = 0
364 361
365 362 if self.__hasNotDataInBuffer():
366 363
367 364 if not( self.readNextBlock() ):
368 365 self.dataOut.flagNoData = True
369 366 return 0
370 367
371 368 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
372 369
373 370 if self.data_spc is None:
374 371 self.dataOut.flagNoData = True
375 372 return 0
376 373
377 374 self.getBasicHeader()
378 375
379 376 self.getFirstHeader()
380 377
381 378 self.dataOut.data_spc = self.data_spc
382 379
383 380 self.dataOut.data_cspc = self.data_cspc
384 381
385 382 self.dataOut.data_dc = self.data_dc
386 383
387 384 self.dataOut.flagNoData = False
388 385
389 386 self.dataOut.realtime = self.online
390 387
391 388 return self.dataOut.data_spc
392 389 @MPDecorator
393 390 class SpectraWriter(JRODataWriter, Operation):
394 391
395 392 """
396 393 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
397 394 de los datos siempre se realiza por bloques.
398 395 """
399 396
400 397 ext = ".pdata"
401 398
402 399 optchar = "P"
403 400
404 401 shape_spc_Buffer = None
405 402
406 403 shape_cspc_Buffer = None
407 404
408 405 shape_dc_Buffer = None
409 406
410 407 data_spc = None
411 408
412 409 data_cspc = None
413 410
414 411 data_dc = None
415 412
416 413 def __init__(self):
417 414 """
418 415 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
419 416
420 417 Affected:
421 418 self.dataOut
422 419 self.basicHeaderObj
423 420 self.systemHeaderObj
424 421 self.radarControllerHeaderObj
425 422 self.processingHeaderObj
426 423
427 424 Return: None
428 425 """
429 426
430 427 Operation.__init__(self)
431 428
432 429 self.nTotalBlocks = 0
433 430
434 431 self.data_spc = None
435 432
436 433 self.data_cspc = None
437 434
438 435 self.data_dc = None
439 436
440 437 self.fp = None
441 438
442 439 self.flagIsNewFile = 1
443 440
444 441 self.nTotalBlocks = 0
445 442
446 443 self.flagIsNewBlock = 0
447 444
448 445 self.setFile = None
449 446
450 447 self.dtype = None
451 448
452 449 self.path = None
453 450
454 451 self.noMoreFiles = 0
455 452
456 453 self.filename = None
457 454
458 455 self.basicHeaderObj = BasicHeader(LOCALTIME)
459 456
460 457 self.systemHeaderObj = SystemHeader()
461 458
462 459 self.radarControllerHeaderObj = RadarControllerHeader()
463 460
464 461 self.processingHeaderObj = ProcessingHeader()
465 462
466 463
467 464 def hasAllDataInBuffer(self):
468 465 return 1
469 466
470 467
471 468 def setBlockDimension(self):
472 469 """
473 470 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
474 471
475 472 Affected:
476 473 self.shape_spc_Buffer
477 474 self.shape_cspc_Buffer
478 475 self.shape_dc_Buffer
479 476
480 477 Return: None
481 478 """
482 479 self.shape_spc_Buffer = (self.dataOut.nChannels,
483 480 self.processingHeaderObj.nHeights,
484 481 self.processingHeaderObj.profilesPerBlock)
485 482
486 483 self.shape_cspc_Buffer = (self.dataOut.nPairs,
487 484 self.processingHeaderObj.nHeights,
488 485 self.processingHeaderObj.profilesPerBlock)
489 486
490 487 self.shape_dc_Buffer = (self.dataOut.nChannels,
491 488 self.processingHeaderObj.nHeights)
492 489
493 490
494 491 def writeBlock(self):
495 492 """processingHeaderObj
496 493 Escribe el buffer en el file designado
497 494
498 495 Affected:
499 496 self.data_spc
500 497 self.data_cspc
501 498 self.data_dc
502 499 self.flagIsNewFile
503 500 self.flagIsNewBlock
504 501 self.nTotalBlocks
505 502 self.nWriteBlocks
506 503
507 504 Return: None
508 505 """
509 506
510 507 spc = numpy.transpose( self.data_spc, (0,2,1) )
511 508 if not self.processingHeaderObj.shif_fft:
512 509 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
513 510 data = spc.reshape((-1))
514 511 data = data.astype(self.dtype[0])
515 512 data.tofile(self.fp)
516 513
517 514 if self.data_cspc is not None:
518 515
519 516 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
520 517 data = numpy.zeros( numpy.shape(cspc), self.dtype )
521 518 #print 'data.shape', self.shape_cspc_Buffer
522 519 if not self.processingHeaderObj.shif_fft:
523 520 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
524 521 data['real'] = cspc.real
525 522 data['imag'] = cspc.imag
526 523 data = data.reshape((-1))
527 524 data.tofile(self.fp)
528 525
529 526 if self.data_dc is not None:
530 527
531 528 dc = self.data_dc
532 529 data = numpy.zeros( numpy.shape(dc), self.dtype )
533 530 data['real'] = dc.real
534 531 data['imag'] = dc.imag
535 532 data = data.reshape((-1))
536 533 data.tofile(self.fp)
537 534
538 535 # self.data_spc.fill(0)
539 536 #
540 537 # if self.data_dc is not None:
541 538 # self.data_dc.fill(0)
542 539 #
543 540 # if self.data_cspc is not None:
544 541 # self.data_cspc.fill(0)
545 542
546 543 self.flagIsNewFile = 0
547 544 self.flagIsNewBlock = 1
548 545 self.nTotalBlocks += 1
549 546 self.nWriteBlocks += 1
550 547 self.blockIndex += 1
551 548
552 549 # print "[Writing] Block = %d04" %self.blockIndex
553 550
554 551 def putData(self):
555 552 """
556 553 Setea un bloque de datos y luego los escribe en un file
557 554
558 555 Affected:
559 556 self.data_spc
560 557 self.data_cspc
561 558 self.data_dc
562 559
563 560 Return:
564 561 0 : Si no hay data o no hay mas files que puedan escribirse
565 562 1 : Si se escribio la data de un bloque en un file
566 563 """
567 564
568 565 if self.dataOut.flagNoData:
569 566 return 0
570 567
571 568 self.flagIsNewBlock = 0
572 569
573 570 if self.dataOut.flagDiscontinuousBlock:
574 571 self.data_spc.fill(0)
575 572 if self.dataOut.data_cspc is not None:
576 573 self.data_cspc.fill(0)
577 574 if self.dataOut.data_dc is not None:
578 575 self.data_dc.fill(0)
579 576 self.setNextFile()
580 577
581 578 if self.flagIsNewFile == 0:
582 579 self.setBasicHeader()
583 580
584 581 self.data_spc = self.dataOut.data_spc.copy()
585 582
586 583 if self.dataOut.data_cspc is not None:
587 584 self.data_cspc = self.dataOut.data_cspc.copy()
588 585
589 586 if self.dataOut.data_dc is not None:
590 587 self.data_dc = self.dataOut.data_dc.copy()
591 588
592 589 # #self.processingHeaderObj.dataBlocksPerFile)
593 590 if self.hasAllDataInBuffer():
594 591 # self.setFirstHeader()
595 592 self.writeNextBlock()
596 593
597 594 def __getBlockSize(self):
598 595 '''
599 596 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
600 597 '''
601 598
602 599 dtype_width = self.getDtypeWidth()
603 600
604 601 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
605 602
606 603 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
607 604 blocksize = (pts2write_SelfSpectra*dtype_width)
608 605
609 606 if self.dataOut.data_cspc is not None:
610 607 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
611 608 blocksize += (pts2write_CrossSpectra*dtype_width*2)
612 609
613 610 if self.dataOut.data_dc is not None:
614 611 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
615 612 blocksize += (pts2write_DCchannels*dtype_width*2)
616 613
617 614 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
618 615
619 616 return blocksize
620 617
621 618 def setFirstHeader(self):
622 619
623 620 """
624 621 Obtiene una copia del First Header
625 622
626 623 Affected:
627 624 self.systemHeaderObj
628 625 self.radarControllerHeaderObj
629 626 self.dtype
630 627
631 628 Return:
632 629 None
633 630 """
634 631
635 632 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
636 633 self.systemHeaderObj.nChannels = self.dataOut.nChannels
637 634 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
638 635
639 636 self.processingHeaderObj.dtype = 1 # Spectra
640 637 self.processingHeaderObj.blockSize = self.__getBlockSize()
641 638 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
642 639 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
643 640 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
644 641 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
645 642 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
646 643 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
647 644 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
648 645
649 646 if self.processingHeaderObj.totalSpectra > 0:
650 647 channelList = []
651 648 for channel in range(self.dataOut.nChannels):
652 649 channelList.append(channel)
653 650 channelList.append(channel)
654 651
655 652 pairsList = []
656 653 if self.dataOut.nPairs > 0:
657 654 for pair in self.dataOut.pairsList:
658 655 pairsList.append(pair[0])
659 656 pairsList.append(pair[1])
660 657
661 658 spectraComb = channelList + pairsList
662 659 spectraComb = numpy.array(spectraComb, dtype="u1")
663 660 self.processingHeaderObj.spectraComb = spectraComb
664 661
665 662 if self.dataOut.code is not None:
666 663 self.processingHeaderObj.code = self.dataOut.code
667 664 self.processingHeaderObj.nCode = self.dataOut.nCode
668 665 self.processingHeaderObj.nBaud = self.dataOut.nBaud
669 666
670 667 if self.processingHeaderObj.nWindows != 0:
671 668 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
672 669 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
673 670 self.processingHeaderObj.nHeights = self.dataOut.nHeights
674 671 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
675 672
676 673 self.processingHeaderObj.processFlags = self.getProcessFlags()
677 674
678 675 self.setBasicHeader() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,424 +1,399
1 1 '''
2 2 Updated for multiprocessing
3 3 Author : Sergio Cortez
4 4 Jan 2018
5 5 Abstract:
6 6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7 7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8 8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9 9
10 10 Based on:
11 11 $Author: murco $
12 12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
13 13 '''
14 14
15 15 import os
16 16 import inspect
17 17 import zmq
18 18 import time
19 19 import pickle
20 import traceback
20 21 from queue import Queue
21 22 from threading import Thread
22 23 from multiprocessing import Process
23 from zmq.utils.monitor import recv_monitor_message
24 24
25 25 from schainpy.utils import log
26 26
27 27
28 28 class ProcessingUnit(object):
29 29
30 30 """
31 31 Update - Jan 2018 - MULTIPROCESSING
32 32 All the "call" methods present in the previous base were removed.
33 33 The majority of operations are independant processes, thus
34 34 the decorator is in charge of communicate the operation processes
35 35 with the proccessing unit via IPC.
36 36
37 37 The constructor does not receive any argument. The remaining methods
38 38 are related with the operations to execute.
39 39
40 40
41 41 """
42 42
43 43 def __init__(self):
44 44
45 45 self.dataIn = None
46 46 self.dataOut = None
47 47 self.isConfig = False
48 48 self.operations = []
49 49 self.plots = []
50 50
51 51 def getAllowedArgs(self):
52 52 if hasattr(self, '__attrs__'):
53 53 return self.__attrs__
54 54 else:
55 55 return inspect.getargspec(self.run).args
56 56
57 57 def addOperation(self, conf, operation):
58 58 """
59 59 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
60 60 posses the id of the operation process (IPC purposes)
61 61
62 62 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
63 63 identificador asociado a este objeto.
64 64
65 65 Input:
66 66
67 67 object : objeto de la clase "Operation"
68 68
69 69 Return:
70 70
71 71 objId : identificador del objeto, necesario para comunicar con master(procUnit)
72 72 """
73 73
74 74 self.operations.append(
75 75 (operation, conf.type, conf.id, conf.getKwargs()))
76 76
77 77 if 'plot' in self.name.lower():
78 78 self.plots.append(operation.CODE)
79 79
80 80 def getOperationObj(self, objId):
81 81
82 82 if objId not in list(self.operations.keys()):
83 83 return None
84 84
85 85 return self.operations[objId]
86 86
87 87 def operation(self, **kwargs):
88 88 """
89 89 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
90 90 atributos del objeto dataOut
91 91
92 92 Input:
93 93
94 94 **kwargs : Diccionario de argumentos de la funcion a ejecutar
95 95 """
96 96
97 97 raise NotImplementedError
98 98
99 99 def setup(self):
100 100
101 101 raise NotImplementedError
102 102
103 103 def run(self):
104 104
105 105 raise NotImplementedError
106 106
107 107 def close(self):
108 108
109 109 return
110 110
111 111
112 112 class Operation(object):
113 113
114 114 """
115 115 Update - Jan 2018 - MULTIPROCESSING
116 116
117 117 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
118 118 The constructor doe snot receive any argument, neither the baseclass.
119 119
120 120
121 121 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
122 122 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
123 123 acumulacion dentro de esta clase
124 124
125 125 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
126 126
127 127 """
128 128
129 129 def __init__(self):
130 130
131 131 self.id = None
132 132 self.isConfig = False
133 133
134 134 if not hasattr(self, 'name'):
135 135 self.name = self.__class__.__name__
136 136
137 137 def getAllowedArgs(self):
138 138 if hasattr(self, '__attrs__'):
139 139 return self.__attrs__
140 140 else:
141 141 return inspect.getargspec(self.run).args
142 142
143 143 def setup(self):
144 144
145 145 self.isConfig = True
146 146
147 147 raise NotImplementedError
148 148
149 149 def run(self, dataIn, **kwargs):
150 150 """
151 151 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
152 152 atributos del objeto dataIn.
153 153
154 154 Input:
155 155
156 156 dataIn : objeto del tipo JROData
157 157
158 158 Return:
159 159
160 160 None
161 161
162 162 Affected:
163 163 __buffer : buffer de recepcion de datos.
164 164
165 165 """
166 166 if not self.isConfig:
167 167 self.setup(**kwargs)
168 168
169 169 raise NotImplementedError
170 170
171 171 def close(self):
172 172
173 173 return
174 174
175 175 class InputQueue(Thread):
176 176 '''
177 177 Class to hold input data for Proccessing Units and external Operations,
178 178 '''
179 179
180 180 def __init__(self, project_id, inputId):
181 181 Thread.__init__(self)
182 182 self.queue = Queue()
183 183 self.project_id = project_id
184 184 self.inputId = inputId
185 185
186 186 def run(self):
187 187
188 188 c = zmq.Context()
189 189 self.receiver = c.socket(zmq.SUB)
190 190 self.receiver.connect(
191 191 'ipc:///tmp/schain/{}_pub'.format(self.project_id))
192 192 self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
193 193
194 194 while True:
195 195 self.queue.put(self.receiver.recv_multipart()[1])
196 196
197 197 def get(self):
198 198
199 199 return pickle.loads(self.queue.get())
200 200
201 201
202 202 def MPDecorator(BaseClass):
203 203 """
204 204 Multiprocessing class decorator
205 205
206 206 This function add multiprocessing features to a BaseClass. Also, it handle
207 207 the communication beetween processes (readers, procUnits and operations).
208 208 """
209 209
210 210 class MPClass(BaseClass, Process):
211 211
212 212 def __init__(self, *args, **kwargs):
213 213 super(MPClass, self).__init__()
214 214 Process.__init__(self)
215 215 self.operationKwargs = {}
216 216 self.args = args
217 217 self.kwargs = kwargs
218 218 self.sender = None
219 219 self.receiver = None
220 220 self.i = 0
221 221 self.name = BaseClass.__name__
222
222 223 if 'plot' in self.name.lower() and not self.name.endswith('_'):
223 224 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
224 self.start_time = time.time()
225
226 if len(self.args) is 3:
227 self.typeProc = "ProcUnit"
228 self.id = args[0]
229 self.inputId = args[1]
230 self.project_id = args[2]
231 elif len(self.args) is 2:
232 self.id = args[0]
233 self.inputId = args[0]
234 self.project_id = args[1]
235 self.typeProc = "Operation"
236
225
226 self.start_time = time.time()
227 self.id = args[0]
228 self.inputId = args[1]
229 self.project_id = args[2]
230 self.err_queue = args[3]
231 self.typeProc = args[4]
237 232 self.queue = InputQueue(self.project_id, self.inputId)
233 self.err_queue.put('#_start_#')
238 234
239 235 def subscribe(self):
240 236 '''
241 237 This function start the input queue.
242 238 '''
243 239
244 240 self.queue.start()
245 241
246 242
247 243 def listen(self):
248 244 '''
249 245 This function waits for objects
250 246 '''
251 247
252 248 return self.queue.get()
253 249
254 250 def set_publisher(self):
255 251 '''
256 252 This function create a socket for publishing purposes.
257 253 '''
258 254
259 255 time.sleep(1)
260 256 c = zmq.Context()
261 257 self.sender = c.socket(zmq.PUB)
262 258 self.sender.connect(
263 259 'ipc:///tmp/schain/{}_sub'.format(self.project_id))
264 260
265 261 def publish(self, data, id):
266 262 '''
267 263 This function publish an object, to a specific topic.
268 264 For Read Units (inputId == None) adds a little delay
269 265 to avoid data loss
270 266 '''
271 267
272 268 if self.inputId is None:
273 269 self.i += 1
274 if self.i % 100 == 0:
270 if self.i % 80 == 0:
275 271 self.i = 0
276 272 time.sleep(0.01)
277 273
278 274 self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
279 275
280 276 def runReader(self):
281 277 '''
282 278 Run fuction for read units
283 279 '''
284 280 while True:
285 281
286 BaseClass.run(self, **self.kwargs)
282 try:
283 BaseClass.run(self, **self.kwargs)
284 except:
285 err = traceback.format_exc()
286 if 'No more files' in err:
287 log.warning('No more files to read', self.name)
288 else:
289 self.err_queue.put('{}|{}'.format(self.name, err))
290 self.dataOut.error = True
287 291
288 292 for op, optype, opId, kwargs in self.operations:
289 293 if optype == 'self' and not self.dataOut.flagNoData:
290 294 op(**kwargs)
291 295 elif optype == 'other' and not self.dataOut.flagNoData:
292 296 self.dataOut = op.run(self.dataOut, **self.kwargs)
293 297 elif optype == 'external':
294 298 self.publish(self.dataOut, opId)
295 299
296 300 if self.dataOut.flagNoData and not self.dataOut.error:
297 301 continue
298 302
299 303 self.publish(self.dataOut, self.id)
300 304
301 if self.dataOut.error:
302 log.error(self.dataOut.error, self.name)
303 # self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
305 if self.dataOut.error:
304 306 break
305 307
306 time.sleep(1)
308 time.sleep(0.5)
307 309
308 310 def runProc(self):
309 311 '''
310 312 Run function for proccessing units
311 313 '''
312 314
313 315 while True:
314 316 self.dataIn = self.listen()
315 317
316 318 if self.dataIn.flagNoData and self.dataIn.error is None:
317 319 continue
318
319 BaseClass.run(self, **self.kwargs)
320
321 if self.dataIn.error:
320 elif not self.dataIn.error:
321 try:
322 BaseClass.run(self, **self.kwargs)
323 except:
324 self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
325 self.dataOut.error = True
326 elif self.dataIn.error:
322 327 self.dataOut.error = self.dataIn.error
323 328 self.dataOut.flagNoData = True
324 329
325 330 for op, optype, opId, kwargs in self.operations:
326 331 if optype == 'self' and not self.dataOut.flagNoData:
327 332 op(**kwargs)
328 333 elif optype == 'other' and not self.dataOut.flagNoData:
329 334 self.dataOut = op.run(self.dataOut, **kwargs)
330 335 elif optype == 'external' and not self.dataOut.flagNoData:
331 336 self.publish(self.dataOut, opId)
332
333 if not self.dataOut.flagNoData or self.dataOut.error:
334 self.publish(self.dataOut, self.id)
335 for op, optype, opId, kwargs in self.operations:
336 if optype == 'self' and self.dataOut.error:
337 op(**kwargs)
338 elif optype == 'other' and self.dataOut.error:
339 self.dataOut = op.run(self.dataOut, **kwargs)
340 elif optype == 'external' and self.dataOut.error:
341 self.publish(self.dataOut, opId)
342 337
343 if self.dataIn.error:
338 self.publish(self.dataOut, self.id)
339 for op, optype, opId, kwargs in self.operations:
340 if optype == 'external' and self.dataOut.error:
341 self.publish(self.dataOut, opId)
342
343 if self.dataOut.error:
344 344 break
345 345
346 time.sleep(1)
346 time.sleep(0.5)
347 347
348 348 def runOp(self):
349 349 '''
350 350 Run function for external operations (this operations just receive data
351 351 ex: plots, writers, publishers)
352 352 '''
353 353
354 354 while True:
355 355
356 356 dataOut = self.listen()
357 357
358 BaseClass.run(self, dataOut, **self.kwargs)
359
360 if dataOut.error:
361 break
362
363 time.sleep(1)
358 if not dataOut.error:
359 BaseClass.run(self, dataOut, **self.kwargs)
360 else:
361 break
364 362
365 363 def run(self):
366 364 if self.typeProc is "ProcUnit":
367 365
368 366 if self.inputId is not None:
369
370 367 self.subscribe()
371 368
372 369 self.set_publisher()
373 370
374 371 if 'Reader' not in BaseClass.__name__:
375 372 self.runProc()
376 373 else:
377 374 self.runReader()
378 375
379 376 elif self.typeProc is "Operation":
380 377
381 378 self.subscribe()
382 379 self.runOp()
383 380
384 381 else:
385 382 raise ValueError("Unknown type")
386 383
387 384 self.close()
388 385
389 def event_monitor(self, monitor):
390
391 events = {}
392
393 for name in dir(zmq):
394 if name.startswith('EVENT_'):
395 value = getattr(zmq, name)
396 events[value] = name
397
398 while monitor.poll():
399 evt = recv_monitor_message(monitor)
400 if evt['event'] == 32:
401 self.connections += 1
402 if evt['event'] == 512:
403 pass
404
405 evt.update({'description': events[evt['event']]})
406
407 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
408 break
409 monitor.close()
410 print('event monitor thread done!')
411
412 386 def close(self):
413 387
414 388 BaseClass.close(self)
389 self.err_queue.put('#_end_#')
415 390
416 391 if self.sender:
417 392 self.sender.close()
418 393
419 394 if self.receiver:
420 395 self.receiver.close()
421 396
422 397 log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
423 398
424 399 return MPClass
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now