##// END OF EJS Templates
schain
RSS
Remove numpy.float, numpy.int, numpy.complex
Juan C. Espinoza -
r1594:16c6f41ffb3e
parent child
Show More
Side by Side Unified
Context file:
r1594:16c6f41ffb3e Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,793 +1,793
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 # SUBCHANNELS EN VEZ DE CHANNELS
7 7 # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
8 8 # ACTUALIZACION DE VERSION
9 9 # HEADERS
10 10 # MODULO DE ESCRITURA
11 11 # METADATA
12 12
13 13 import os
14 14 import time
15 15 import datetime
16 16 import numpy
17 17 import timeit
18 18 from fractions import Fraction
19 19 from time import time
20 20 from time import sleep
21 21
22 22 import schainpy.admin
23 23 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 24 from schainpy.model.data.jrodata import Voltage
25 25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
26 26
27 27 import pickle
28 28 try:
29 29 import digital_rf
30 30 except:
31 31 pass
32 32
33 33
34 34 class DigitalRFReader(ProcessingUnit):
35 35 '''
36 36 classdocs
37 37 '''
38 38
39 39 def __init__(self):
40 40 '''
41 41 Constructor
42 42 '''
43 43
44 44 ProcessingUnit.__init__(self)
45 45
46 46 self.dataOut = Voltage()
47 47 self.__printInfo = True
48 48 self.__flagDiscontinuousBlock = False
49 49 self.__bufferIndex = 9999999
50 50 self.__codeType = 0
51 51 self.__ippKm = None
52 52 self.__nCode = None
53 53 self.__nBaud = None
54 54 self.__code = None
55 55 self.dtype = None
56 56 self.oldAverage = None
57 57 self.path = None
58 58
59 59 def close(self):
60 60 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
61 61 return
62 62
63 63 def __getCurrentSecond(self):
64 64
65 65 return self.__thisUnixSample / self.__sample_rate
66 66
67 67 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
68 68
69 69 def __setFileHeader(self):
70 70 '''
71 71 In this method will be initialized every parameter of dataOut object (header, no data)
72 72 '''
73 73 ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
74 74
75 75 nProfiles = 1.0 / ippSeconds # Number of profiles in one second
76 76
77 77 try:
78 78 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
79 79 self.__radarControllerHeader)
80 80 except:
81 81 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
82 82 txA=0,
83 83 txB=0,
84 84 nWindows=1,
85 85 nHeights=self.__nSamples,
86 86 firstHeight=self.__firstHeigth,
87 87 deltaHeight=self.__deltaHeigth,
88 88 codeType=self.__codeType,
89 89 nCode=self.__nCode, nBaud=self.__nBaud,
90 90 code=self.__code)
91 91
92 92 try:
93 93 self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
94 94 except:
95 95 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
96 96 nProfiles=nProfiles,
97 97 nChannels=len(
98 98 self.__channelList),
99 99 adcResolution=14)
100 100 self.dataOut.type = "Voltage"
101 101
102 102 self.dataOut.data = None
103 103
104 104 self.dataOut.dtype = self.dtype
105 105
106 106 # self.dataOut.nChannels = 0
107 107
108 108 # self.dataOut.nHeights = 0
109 109
110 110 self.dataOut.nProfiles = int(nProfiles)
111 111
112 112 self.dataOut.heightList = self.__firstHeigth + \
113 numpy.arange(self.__nSamples, dtype=numpy.float) * \
113 numpy.arange(self.__nSamples, dtype=numpy.float32) * \
114 114 self.__deltaHeigth
115 115
116 116 self.dataOut.channelList = list(range(self.__num_subchannels))
117 117
118 118 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
119 119
120 120 # self.dataOut.channelIndexList = None
121 121
122 122 self.dataOut.flagNoData = True
123 123
124 124 self.dataOut.flagDataAsBlock = False
125 125 # Set to TRUE if the data is discontinuous
126 126 self.dataOut.flagDiscontinuousBlock = False
127 127
128 128 self.dataOut.utctime = None
129 129
130 130 # timezone like jroheader, difference in minutes between UTC and localtime
131 131 self.dataOut.timeZone = self.__timezone / 60
132 132
133 133 self.dataOut.dstFlag = 0
134 134
135 135 self.dataOut.errorCount = 0
136 136
137 137 try:
138 138 self.dataOut.nCohInt = self.fixed_metadata_dict.get(
139 139 'nCohInt', self.nCohInt)
140 140
141 141 # asumo que la data esta decodificada
142 142 self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
143 143 'flagDecodeData', self.flagDecodeData)
144 144
145 145 # asumo que la data esta sin flip
146 146 self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
147 147
148 148 self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
149 149
150 150 self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
151 151 except:
152 152 pass
153 153
154 154 self.dataOut.ippSeconds = ippSeconds
155 155
156 156 # Time interval between profiles
157 157 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
158 158
159 159 self.dataOut.frequency = self.__frequency
160 160
161 161 self.dataOut.realtime = self.__online
162 162
163 163 def findDatafiles(self, path, startDate=None, endDate=None):
164 164
165 165 if not os.path.isdir(path):
166 166 return []
167 167
168 168 try:
169 169 digitalReadObj = digital_rf.DigitalRFReader(
170 170 path, load_all_metadata=True)
171 171 except:
172 172 digitalReadObj = digital_rf.DigitalRFReader(path)
173 173
174 174 channelNameList = digitalReadObj.get_channels()
175 175
176 176 if not channelNameList:
177 177 return []
178 178
179 179 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
180 180
181 181 sample_rate = metadata_dict['sample_rate'][0]
182 182
183 183 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
184 184
185 185 try:
186 186 timezone = this_metadata_file['timezone'].value
187 187 except:
188 188 timezone = 0
189 189
190 190 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
191 191 channelNameList[0]) / sample_rate - timezone
192 192
193 193 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
194 194 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
195 195
196 196 if not startDate:
197 197 startDate = startDatetime.date()
198 198
199 199 if not endDate:
200 200 endDate = endDatatime.date()
201 201
202 202 dateList = []
203 203
204 204 thisDatetime = startDatetime
205 205
206 206 while(thisDatetime <= endDatatime):
207 207
208 208 thisDate = thisDatetime.date()
209 209
210 210 if thisDate < startDate:
211 211 continue
212 212
213 213 if thisDate > endDate:
214 214 break
215 215
216 216 dateList.append(thisDate)
217 217 thisDatetime += datetime.timedelta(1)
218 218
219 219 return dateList
220 220
221 221 def setup(self, path=None,
222 222 startDate=None,
223 223 endDate=None,
224 224 startTime=datetime.time(0, 0, 0),
225 225 endTime=datetime.time(23, 59, 59),
226 226 channelList=None,
227 227 nSamples=None,
228 228 online=False,
229 229 delay=60,
230 230 buffer_size=1024,
231 231 ippKm=None,
232 232 nCohInt=1,
233 233 nCode=1,
234 234 nBaud=1,
235 235 flagDecodeData=False,
236 code=numpy.ones((1, 1), dtype=numpy.int),
236 code=numpy.ones((1, 1), dtype=numpy.int32),
237 237 **kwargs):
238 238 '''
239 239 In this method we should set all initial parameters.
240 240
241 241 Inputs:
242 242 path
243 243 startDate
244 244 endDate
245 245 startTime
246 246 endTime
247 247 set
248 248 expLabel
249 249 ext
250 250 online
251 251 delay
252 252 '''
253 253 self.path = path
254 254 self.nCohInt = nCohInt
255 255 self.flagDecodeData = flagDecodeData
256 256 self.i = 0
257 257 if not os.path.isdir(path):
258 258 raise ValueError("[Reading] Directory %s does not exist" % path)
259 259
260 260 try:
261 261 self.digitalReadObj = digital_rf.DigitalRFReader(
262 262 path, load_all_metadata=True)
263 263 except:
264 264 self.digitalReadObj = digital_rf.DigitalRFReader(path)
265 265
266 266 channelNameList = self.digitalReadObj.get_channels()
267 267
268 268 if not channelNameList:
269 269 raise ValueError("[Reading] Directory %s does not have any files" % path)
270 270
271 271 if not channelList:
272 272 channelList = list(range(len(channelNameList)))
273 273
274 274 ########## Reading metadata ######################
275 275
276 276 top_properties = self.digitalReadObj.get_properties(
277 277 channelNameList[channelList[0]])
278 278
279 279 self.__num_subchannels = top_properties['num_subchannels']
280 280 self.__sample_rate = 1.0 * \
281 281 top_properties['sample_rate_numerator'] / \
282 282 top_properties['sample_rate_denominator']
283 283 # self.__samples_per_file = top_properties['samples_per_file'][0]
284 284 self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate # why 0.15?
285 285
286 286 this_metadata_file = self.digitalReadObj.get_digital_metadata(
287 287 channelNameList[channelList[0]])
288 288 metadata_bounds = this_metadata_file.get_bounds()
289 289 self.fixed_metadata_dict = this_metadata_file.read(
290 290 metadata_bounds[0])[metadata_bounds[0]] # GET FIRST HEADER
291 291
292 292 try:
293 293 self.__processingHeader = self.fixed_metadata_dict['processingHeader']
294 294 self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
295 295 self.__systemHeader = self.fixed_metadata_dict['systemHeader']
296 296 self.dtype = pickle.loads(self.fixed_metadata_dict['dtype'])
297 297 except:
298 298 pass
299 299
300 300 self.__frequency = None
301 301
302 302 self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
303 303
304 304 self.__timezone = self.fixed_metadata_dict.get('timezone', 18000)
305 305
306 306 try:
307 307 nSamples = self.fixed_metadata_dict['nSamples']
308 308 except:
309 309 nSamples = None
310 310
311 311 self.__firstHeigth = 0
312 312
313 313 try:
314 314 codeType = self.__radarControllerHeader['codeType']
315 315 except:
316 316 codeType = 0
317 317
318 318 try:
319 319 if codeType:
320 320 nCode = self.__radarControllerHeader['nCode']
321 321 nBaud = self.__radarControllerHeader['nBaud']
322 322 code = self.__radarControllerHeader['code']
323 323 except:
324 324 pass
325 325
326 326 if not ippKm:
327 327 try:
328 328 # seconds to km
329 329 ippKm = self.__radarControllerHeader['ipp']
330 330 except:
331 331 ippKm = None
332 332 ####################################################
333 333 self.__ippKm = ippKm
334 334 startUTCSecond = None
335 335 endUTCSecond = None
336 336
337 337 if startDate:
338 338 startDatetime = datetime.datetime.combine(startDate, startTime)
339 339 startUTCSecond = (
340 340 startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
341 341
342 342 if endDate:
343 343 endDatetime = datetime.datetime.combine(endDate, endTime)
344 344 endUTCSecond = (endDatetime - datetime.datetime(1970,
345 345 1, 1)).total_seconds() + self.__timezone
346 346
347 347 start_index, end_index = self.digitalReadObj.get_bounds(
348 348 channelNameList[channelList[0]])
349 349
350 350 if not startUTCSecond:
351 351 startUTCSecond = start_index / self.__sample_rate
352 352
353 353 if start_index > startUTCSecond * self.__sample_rate:
354 354 startUTCSecond = start_index / self.__sample_rate
355 355
356 356 if not endUTCSecond:
357 357 endUTCSecond = end_index / self.__sample_rate
358 358
359 359 if end_index < endUTCSecond * self.__sample_rate:
360 360 endUTCSecond = end_index / self.__sample_rate
361 361 if not nSamples:
362 362 if not ippKm:
363 363 raise ValueError("[Reading] nSamples or ippKm should be defined")
364 364 nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
365 365 channelBoundList = []
366 366 channelNameListFiltered = []
367 367
368 368 for thisIndexChannel in channelList:
369 369 thisChannelName = channelNameList[thisIndexChannel]
370 370 start_index, end_index = self.digitalReadObj.get_bounds(
371 371 thisChannelName)
372 372 channelBoundList.append((start_index, end_index))
373 373 channelNameListFiltered.append(thisChannelName)
374 374
375 375 self.profileIndex = 0
376 376 self.i = 0
377 377 self.__delay = delay
378 378
379 379 self.__codeType = codeType
380 380 self.__nCode = nCode
381 381 self.__nBaud = nBaud
382 382 self.__code = code
383 383
384 384 self.__datapath = path
385 385 self.__online = online
386 386 self.__channelList = channelList
387 387 self.__channelNameList = channelNameListFiltered
388 388 self.__channelBoundList = channelBoundList
389 389 self.__nSamples = nSamples
390 390 self.__samples_to_read = int(nSamples) # FIJO: AHORA 40
391 391 self.__nChannels = len(self.__channelList)
392 392
393 393 self.__startUTCSecond = startUTCSecond
394 394 self.__endUTCSecond = endUTCSecond
395 395
396 396 self.__timeInterval = 1.0 * self.__samples_to_read / \
397 397 self.__sample_rate # Time interval
398 398
399 399 if online:
400 400 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
401 401 startUTCSecond = numpy.floor(endUTCSecond)
402 402
403 403 # por que en el otro metodo lo primero q se hace es sumar samplestoread
404 404 self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
405 405
406 406 self.__data_buffer = numpy.zeros(
407 (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
407 (self.__num_subchannels, self.__samples_to_read), dtype=complex)
408 408
409 409 self.__setFileHeader()
410 410 self.isConfig = True
411 411
412 412 print("[Reading] Digital RF Data was found from %s to %s " % (
413 413 datetime.datetime.utcfromtimestamp(
414 414 self.__startUTCSecond - self.__timezone),
415 415 datetime.datetime.utcfromtimestamp(
416 416 self.__endUTCSecond - self.__timezone)
417 417 ))
418 418
419 419 print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
420 420 datetime.datetime.utcfromtimestamp(
421 421 endUTCSecond - self.__timezone)
422 422 ))
423 423 self.oldAverage = None
424 424 self.count = 0
425 425 self.executionTime = 0
426 426
427 427 def __reload(self):
428 428 # print
429 429 # print "%s not in range [%s, %s]" %(
430 430 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
431 431 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
432 432 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
433 433 # )
434 434 print("[Reading] reloading metadata ...")
435 435
436 436 try:
437 437 self.digitalReadObj.reload(complete_update=True)
438 438 except:
439 439 self.digitalReadObj = digital_rf.DigitalRFReader(self.path)
440 440
441 441 start_index, end_index = self.digitalReadObj.get_bounds(
442 442 self.__channelNameList[self.__channelList[0]])
443 443
444 444 if start_index > self.__startUTCSecond * self.__sample_rate:
445 445 self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
446 446
447 447 if end_index > self.__endUTCSecond * self.__sample_rate:
448 448 self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
449 449 print()
450 450 print("[Reading] New timerange found [%s, %s] " % (
451 451 datetime.datetime.utcfromtimestamp(
452 452 self.__startUTCSecond - self.__timezone),
453 453 datetime.datetime.utcfromtimestamp(
454 454 self.__endUTCSecond - self.__timezone)
455 455 ))
456 456
457 457 return True
458 458
459 459 return False
460 460
461 461 def timeit(self, toExecute):
462 462 t0 = time.time()
463 463 toExecute()
464 464 self.executionTime = time.time() - t0
465 465 if self.oldAverage is None:
466 466 self.oldAverage = self.executionTime
467 467 self.oldAverage = (self.executionTime + self.count *
468 468 self.oldAverage) / (self.count + 1.0)
469 469 self.count = self.count + 1.0
470 470 return
471 471
472 472 def __readNextBlock(self, seconds=30, volt_scale=1):
473 473 '''
474 474 '''
475 475
476 476 # Set the next data
477 477 self.__flagDiscontinuousBlock = False
478 478 self.__thisUnixSample += self.__samples_to_read
479 479
480 480 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
481 481 print ("[Reading] There are no more data into selected time-range")
482 482 if self.__online:
483 483 sleep(3)
484 484 self.__reload()
485 485 else:
486 486 return False
487 487
488 488 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
489 489 return False
490 490 self.__thisUnixSample -= self.__samples_to_read
491 491
492 492 indexChannel = 0
493 493
494 494 dataOk = False
495 495
496 496 for thisChannelName in self.__channelNameList: # TODO VARIOS CHANNELS?
497 497 for indexSubchannel in range(self.__num_subchannels):
498 498 try:
499 499 t0 = time()
500 500 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
501 501 self.__samples_to_read,
502 502 thisChannelName, sub_channel=indexSubchannel)
503 503 self.executionTime = time() - t0
504 504 if self.oldAverage is None:
505 505 self.oldAverage = self.executionTime
506 506 self.oldAverage = (
507 507 self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
508 508 self.count = self.count + 1.0
509 509
510 510 except IOError as e:
511 511 # read next profile
512 512 self.__flagDiscontinuousBlock = True
513 513 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
514 514 break
515 515
516 516 if result.shape[0] != self.__samples_to_read:
517 517 self.__flagDiscontinuousBlock = True
518 518 print("[Reading] %s: Too few samples were found, just %d/%d samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
519 519 result.shape[0],
520 520 self.__samples_to_read))
521 521 break
522 522
523 523 self.__data_buffer[indexSubchannel, :] = result * volt_scale
524 524 indexChannel += 1
525 525
526 526 dataOk = True
527 527
528 528 self.__utctime = self.__thisUnixSample / self.__sample_rate
529 529
530 530 if not dataOk:
531 531 return False
532 532
533 533 print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
534 534 self.__samples_to_read,
535 535 self.__timeInterval))
536 536
537 537 self.__bufferIndex = 0
538 538
539 539 return True
540 540
541 541 def __isBufferEmpty(self):
542 542 return self.__bufferIndex > self.__samples_to_read - self.__nSamples # 40960 - 40
543 543
544 544 def getData(self, seconds=30, nTries=5):
545 545 '''
546 546 This method gets the data from files and put the data into the dataOut object
547 547
548 548 In addition, increase el the buffer counter in one.
549 549
550 550 Return:
551 551 data : retorna un perfil de voltages (alturas * canales) copiados desde el
552 552 buffer. Si no hay mas archivos a leer retorna None.
553 553
554 554 Affected:
555 555 self.dataOut
556 556 self.profileIndex
557 557 self.flagDiscontinuousBlock
558 558 self.flagIsNewBlock
559 559 '''
560 560 # print("getdata")
561 561 err_counter = 0
562 562 self.dataOut.flagNoData = True
563 563
564 564 if self.__isBufferEmpty():
565 565 # print("hi")
566 566 self.__flagDiscontinuousBlock = False
567 567
568 568 while True:
569 569 # print ("q ha pasado")
570 570 if self.__readNextBlock():
571 571 break
572 572 if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
573 573 raise schainpy.admin.SchainError('Error')
574 574 return
575 575
576 576 if self.__flagDiscontinuousBlock:
577 577 raise schainpy.admin.SchainError('discontinuous block found')
578 578 return
579 579
580 580 if not self.__online:
581 581 raise schainpy.admin.SchainError('Online?')
582 582 return
583 583
584 584 err_counter += 1
585 585 if err_counter > nTries:
586 586 raise schainpy.admin.SchainError('Max retrys reach')
587 587 return
588 588
589 589 print('[Reading] waiting %d seconds to read a new block' % seconds)
590 590 time.sleep(seconds)
591 591
592 592 self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
593 593 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
594 594 self.dataOut.flagNoData = False
595 595 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
596 596 self.dataOut.profileIndex = self.profileIndex
597 597
598 598 self.__bufferIndex += self.__nSamples
599 599 self.profileIndex += 1
600 600
601 601 if self.profileIndex == self.dataOut.nProfiles:
602 602 self.profileIndex = 0
603 603
604 604 return True
605 605
606 606 def printInfo(self):
607 607 '''
608 608 '''
609 609 if self.__printInfo == False:
610 610 return
611 611
612 612 # self.systemHeaderObj.printInfo()
613 613 # self.radarControllerHeaderObj.printInfo()
614 614
615 615 self.__printInfo = False
616 616
617 617 def printNumberOfBlock(self):
618 618 '''
619 619 '''
620 620 return
621 621 # print self.profileIndex
622 622
623 623 def run(self, **kwargs):
624 624 '''
625 625 This method will be called many times so here you should put all your code
626 626 '''
627 627
628 628 if not self.isConfig:
629 629 self.setup(**kwargs)
630 630 # self.i = self.i+1
631 631 self.getData(seconds=self.__delay)
632 632
633 633 return
634 634
635 635 @MPDecorator
636 636 class DigitalRFWriter(Operation):
637 637 '''
638 638 classdocs
639 639 '''
640 640
641 641 def __init__(self, **kwargs):
642 642 '''
643 643 Constructor
644 644 '''
645 645 Operation.__init__(self, **kwargs)
646 646 self.metadata_dict = {}
647 647 self.dataOut = None
648 648 self.dtype = None
649 649 self.oldAverage = 0
650 650
651 651 def setHeader(self):
652 652
653 653 self.metadata_dict['frequency'] = self.dataOut.frequency
654 654 self.metadata_dict['timezone'] = self.dataOut.timeZone
655 655 self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype)
656 656 self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
657 657 self.metadata_dict['heightList'] = self.dataOut.heightList
658 658 self.metadata_dict['channelList'] = self.dataOut.channelList
659 659 self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
660 660 self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
661 661 self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
662 662 self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
663 663 self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
664 664 self.metadata_dict['type'] = self.dataOut.type
665 665 self.metadata_dict['flagDataAsBlock'] = getattr(
666 666 self.dataOut, 'flagDataAsBlock', None) # chequear
667 667
668 668 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
669 669 '''
670 670 In this method we should set all initial parameters.
671 671 Input:
672 672 dataOut: Input data will also be outputa data
673 673 '''
674 674 self.setHeader()
675 675 self.__ippSeconds = dataOut.ippSeconds
676 676 self.__deltaH = dataOut.getDeltaH()
677 677 self.__sample_rate = 1e6 * 0.15 / self.__deltaH
678 678 self.__dtype = dataOut.dtype
679 679 if len(dataOut.dtype) == 2:
680 680 self.__dtype = dataOut.dtype[0]
681 681 self.__nSamples = dataOut.systemHeaderObj.nSamples
682 682 self.__nProfiles = dataOut.nProfiles
683 683
684 684 if self.dataOut.type != 'Voltage':
685 685 raise 'Digital RF cannot be used with this data type'
686 686 self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
687 687 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
688 688 else:
689 689 self.arr_data = numpy.ones((self.__nSamples, len(
690 690 self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
691 691
692 692 file_cadence_millisecs = 1000
693 693
694 694 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
695 695 sample_rate_numerator = int(sample_rate_fraction.numerator)
696 696 sample_rate_denominator = int(sample_rate_fraction.denominator)
697 697 start_global_index = dataOut.utctime * self.__sample_rate
698 698
699 699 uuid = 'prueba'
700 700 compression_level = 0
701 701 checksum = False
702 702 is_complex = True
703 703 num_subchannels = len(dataOut.channelList)
704 704 is_continuous = True
705 705 marching_periods = False
706 706
707 707 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
708 708 fileCadence, start_global_index,
709 709 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
710 710 is_complex, num_subchannels, is_continuous, marching_periods)
711 711 metadata_dir = os.path.join(path, 'metadata')
712 712 os.system('mkdir %s' % (metadata_dir))
713 713 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, # 236, file_cadence_millisecs / 1000
714 714 sample_rate_numerator, sample_rate_denominator,
715 715 metadataFile)
716 716 self.isConfig = True
717 717 self.currentSample = 0
718 718 self.oldAverage = 0
719 719 self.count = 0
720 720 return
721 721
722 722 def writeMetadata(self):
723 723 start_idx = self.__sample_rate * self.dataOut.utctime
724 724
725 725 self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict(
726 726 )
727 727 self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
728 728 )
729 729 self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict(
730 730 )
731 731 self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
732 732 return
733 733
734 734 def timeit(self, toExecute):
735 735 t0 = time()
736 736 toExecute()
737 737 self.executionTime = time() - t0
738 738 if self.oldAverage is None:
739 739 self.oldAverage = self.executionTime
740 740 self.oldAverage = (self.executionTime + self.count *
741 741 self.oldAverage) / (self.count + 1.0)
742 742 self.count = self.count + 1.0
743 743 return
744 744
745 745 def writeData(self):
746 746 if self.dataOut.type != 'Voltage':
747 747 raise 'Digital RF cannot be used with this data type'
748 748 for channel in self.dataOut.channelList:
749 749 for i in range(self.dataOut.nFFTPoints):
750 750 self.arr_data[1][channel * self.dataOut.nFFTPoints +
751 751 i]['r'] = self.dataOut.data[channel][i].real
752 752 self.arr_data[1][channel * self.dataOut.nFFTPoints +
753 753 i]['i'] = self.dataOut.data[channel][i].imag
754 754 else:
755 755 for i in range(self.dataOut.systemHeaderObj.nSamples):
756 756 for channel in self.dataOut.channelList:
757 757 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
758 758 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
759 759
760 760 def f(): return self.digitalWriteObj.rf_write(self.arr_data)
761 761 self.timeit(f)
762 762
763 763 return
764 764
765 765 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
766 766 '''
767 767 This method will be called many times so here you should put all your code
768 768 Inputs:
769 769 dataOut: object with the data
770 770 '''
771 771 # print dataOut.__dict__
772 772 self.dataOut = dataOut
773 773 if not self.isConfig:
774 774 self.setup(dataOut, path, frequency, fileCadence,
775 775 dirCadence, metadataCadence, **kwargs)
776 776 self.writeMetadata()
777 777
778 778 self.writeData()
779 779
780 780 # # self.currentSample += 1
781 781 # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
782 782 # self.writeMetadata()
783 783 # # if self.currentSample == self.__nProfiles: self.currentSample = 0
784 784
785 785 return dataOut # en la version 2.7 no aparece este return
786 786
787 787 def close(self):
788 788 print('[Writing] - Closing files ')
789 789 print('Average of writing to digital rf format is ', self.oldAverage * 1000)
790 790 try:
791 791 self.digitalWriteObj.close()
792 792 except:
793 793 pass
Show file before | Show file after | Hide comments
@@ -1,862 +1,862
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-com0419
5 5 '''
6 6
7 7 import os, sys
8 8 import time, datetime
9 9 import h5py
10 10 import numpy
11 11 import fnmatch
12 12 import re
13 13
14 14 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 15 from schainpy.model.data.jrodata import Voltage
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
17 17
18 18
19 19 def isNumber(str):
20 20 """
21 21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22 22
23 23 Excepciones:
24 24 Si un determinado string no puede ser convertido a numero
25 25 Input:
26 26 str, string al cual se le analiza para determinar si convertible a un numero o no
27 27
28 28 Return:
29 29 True : si el string es uno numerico
30 30 False : no es un string numerico
31 31 """
32 32 try:
33 33 float(str)
34 34 return True
35 35 except:
36 36 return False
37 37
38 38 def getFileFromSet(path, ext, set=None):
39 39 validFilelist = []
40 40 fileList = os.listdir(path)
41 41
42 42
43 43 if len(fileList) < 1:
44 44 return None
45 45
46 46 # 0 1234 567 89A BCDE
47 47 # H YYYY DDD SSS .ext
48 48
49 49 for thisFile in fileList:
50 50 try:
51 51 number = int(thisFile[6:16])
52 52
53 53 # year = int(thisFile[1:5])
54 54 # doy = int(thisFile[5:8])
55 55 except:
56 56 continue
57 57
58 58 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
59 59 continue
60 60
61 61 validFilelist.append(thisFile)
62 62
63 63 if len(validFilelist) < 1:
64 64 return None
65 65
66 66 validFilelist = sorted(validFilelist, key=str.lower)
67 67
68 68 if set == None:
69 69 return validFilelist[-1]
70 70
71 71 print("set =" , set)
72 72 for thisFile in validFilelist:
73 73 if set <= int(thisFile[6:16]):
74 74 print(thisFile, int(thisFile[6:16]))
75 75 return thisFile
76 76
77 77 return validFilelist[-1]
78 78
79 79 myfile = fnmatch.filter(validFilelist, '*%10d*' % (set))
80 80 # myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
81 81
82 82 if len(myfile) != 0:
83 83 return myfile[0]
84 84 else:
85 85 filename = '*%10.10d%s' % (set, ext.lower())
86 86 print('the filename %s does not exist' % filename)
87 87 print('...going to the last file: ')
88 88
89 89 if validFilelist:
90 90 validFilelist = sorted(validFilelist, key=str.lower)
91 91 return validFilelist[-1]
92 92
93 93 return None
94 94
95 95 def getlastFileFromPath(path, ext):
96 96 """
97 97 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
98 98 al final de la depuracion devuelve el ultimo file de la lista que quedo.
99 99
100 100 Input:
101 101 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
102 102 ext : extension de los files contenidos en una carpeta
103 103
104 104 Return:
105 105 El ultimo file de una determinada carpeta, no se considera el path.
106 106 """
107 107 validFilelist = []
108 108 fileList = os.listdir(path)
109 109
110 110 # 0 1234 567 89A BCDE
111 111 # H YYYY DDD SSS .ext
112 112
113 113 for thisFile in fileList:
114 114
115 115 try:
116 116 number = int(thisFile[6:16])
117 117 except:
118 118 print("There is a file or folder with different format")
119 119 if not isNumber(number):
120 120 continue
121 121
122 122 # year = thisFile[1:5]
123 123 # if not isNumber(year):
124 124 # continue
125 125
126 126 # doy = thisFile[5:8]
127 127 # if not isNumber(doy):
128 128 # continue
129 129
130 130 number = int(number)
131 131 # year = int(year)
132 132 # doy = int(doy)
133 133
134 134 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
135 135 continue
136 136
137 137
138 138 validFilelist.append(thisFile)
139 139
140 140
141 141 if validFilelist:
142 142 validFilelist = sorted(validFilelist, key=str.lower)
143 143 return validFilelist[-1]
144 144
145 145 return None
146 146
147 147
148 148
149 149 class HFReader(ProcessingUnit):
150 150 '''
151 151 classdocs
152 152 '''
153 153 path = None
154 154 startDate = None
155 155 endDate = None
156 156 startTime = None
157 157 endTime = None
158 158 walk = None
159 159 isConfig = False
160 160 dataOut = None
161 161 nTries = 3
162 162 ext = ".hdf5"
163 163
164 164 def __init__(self, **kwargs):
165 165 '''
166 166 Constructor
167 167 '''
168 168 ProcessingUnit.__init__(self, **kwargs)
169 169
170 170 self.isConfig = False
171 171
172 172 self.datablock = None
173 173
174 174 self.filename_current = None
175 175
176 176 self.utc = 0
177 177
178 178 self.ext = '.hdf5'
179 179
180 180 self.flagIsNewFile = 1
181 181
182 182 #-------------------------------------------------
183 183 self.fileIndex = None
184 184
185 185 self.profileIndex_offset = None
186 186
187 187 self.filenameList = []
188 188
189 189 self.hfFilePointer = None
190 190
191 191 self.filename_online = None
192 192
193 193 self.status = True
194 194
195 195 self.flagNoMoreFiles = False
196 196
197 197 self.__waitForNewFile = 20
198 198
199 199
200 200 #--------------------------------------------------
201 201
202 202 self.dataOut = self.createObjByDefault()
203 203
204 204
205 205 def createObjByDefault(self):
206 206
207 207 dataObj = Voltage()
208 208
209 209 return dataObj
210 210
211 211 def setObjProperties(self):
212 212
213 213 pass
214 214
215 215 def getBlockDimension(self):
216 216 """
217 217 Obtiene la cantidad de puntos a leer por cada bloque de datos
218 218
219 219 Affected:
220 220 self.blocksize
221 221
222 222 Return:
223 223 None
224 224 """
225 225 pts2read = self.nChannels * self.nHeights * self.nProfiles
226 226 self.blocksize = pts2read
227 227
228 228 def __readHeader(self):
229 229
230 230 self.nProfiles = 100
231 231 self.nHeights = 1000
232 232 self.nChannels = 2
233 233 self.__firstHeigth = 0
234 234 self.__nSamples = 1000
235 235 self.__deltaHeigth = 1.5
236 236 self.__sample_rate = 1e5
237 237 # self.__frequency=2.72e6
238 238 # self.__frequency=3.64e6
239 239 self.__frequency = None
240 240 self.__online = False
241 241 self.filename_next_set = None
242 242
243 243 # print "Frequency of Operation:", self.__frequency
244 244
245 245
246 246 def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''):
247 247 self.path = path
248 248 self.startDate = startDate
249 249 self.endDate = endDate
250 250 self.startTime = startTime
251 251 self.endTime = endTime
252 252 self.walk = walk
253 253
254 254 def __checkPath(self):
255 255 if os.path.exists(self.path):
256 256 self.status = 1
257 257 else:
258 258 self.status = 0
259 259 print('Path %s does not exits' % self.path)
260 260 return
261 261 return
262 262
263 263 def __selDates(self, hf_dirname_format):
264 264 try:
265 265 dir_hf_filename = self.path + "/" + hf_dirname_format
266 266 fp = h5py.File(dir_hf_filename, 'r')
267 267 hipoc = fp['t'].value
268 268 fp.close()
269 269 date_time = datetime.datetime.utcfromtimestamp(hipoc)
270 270 year = int(date_time[0:4])
271 271 month = int(date_time[5:7])
272 272 dom = int(date_time[8:10])
273 273 thisDate = datetime.date(year, month, dom)
274 274 if (thisDate >= self.startDate and thisDate <= self.endDate):
275 275 return hf_dirname_format
276 276 except:
277 277 return None
278 278
279 279 def __findDataForDates(self, online=False):
280 280 if not(self.status):
281 281 return None
282 282
283 283 pat = '\d+.\d+'
284 284 dirnameList = [re.search(pat, x) for x in os.listdir(self.path)]
285 285 dirnameList = [x for x in dirnameList if x != None]
286 286 dirnameList = [x.string for x in dirnameList]
287 287 if not(online):
288 288
289 289 dirnameList = [self.__selDates(x) for x in dirnameList]
290 290 dirnameList = [x for x in dirnameList if x != None]
291 291
292 292 if len(dirnameList) > 0:
293 293 self.status = 1
294 294 self.dirnameList = dirnameList
295 295 self.dirnameList.sort()
296 296
297 297 else:
298 298 self.status = 0
299 299 return None
300 300
301 301 def __getTimeFromData(self):
302 302 startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime)
303 303 endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime)
304 304 print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader))
305 305 print('........................................')
306 306 filter_filenameList = []
307 307 self.filenameList.sort()
308 308 for i in range(len(self.filenameList) - 1):
309 309 filename = self.filenameList[i]
310 310 dir_hf_filename = filename
311 311 fp = h5py.File(dir_hf_filename, 'r')
312 312 hipoc = fp['t'].value
313 313 hipoc = hipoc + self.timezone
314 314 date_time = datetime.datetime.utcfromtimestamp(hipoc)
315 315 fp.close()
316 316 year = int(date_time[0:4])
317 317 month = int(date_time[5:7])
318 318 dom = int(date_time[8:10])
319 319 hour = int(date_time[11:13])
320 320 min = int(date_time[14:16])
321 321 sec = int(date_time[17:19])
322 322 this_time = datetime.datetime(year, month, dom, hour, min, sec)
323 323 if (this_time >= startDateTime_Reader and this_time <= endDateTime_Reader):
324 324 filter_filenameList.append(filename)
325 325 filter_filenameList.sort()
326 326 self.filenameList = filter_filenameList
327 327 return 1
328 328
329 329 def __getFilenameList(self):
330 330 # print "hola"
331 331 # print self.dirnameList
332 332 dirList = [os.path.join(self.path, x) for x in self.dirnameList]
333 333 self.filenameList = dirList
334 334 # print self.filenameList
335 335 # print "pase",len(self.filenameList)
336 336
337 337 def __selectDataForTimes(self, online=False):
338 338
339 339 if not(self.status):
340 340 return None
341 341 #----------------
342 342 self.__getFilenameList()
343 343 #----------------
344 344 if not(online):
345 345 if not(self.all):
346 346 self.__getTimeFromData()
347 347 if len(self.filenameList) > 0:
348 348 self.status = 1
349 349 self.filenameList.sort()
350 350 else:
351 351 self.status = 0
352 352 return None
353 353 else:
354 354 if self.set != None:
355 355
356 356 filename = getFileFromSet(self.path, self.ext, self.set)
357 357
358 358 if self.flag_nextfile == True:
359 359 self.dirnameList = [filename]
360 360 fullfilename = self.path + "/" + filename
361 361 self.filenameList = [fullfilename]
362 362 self.filename_next_set = int(filename[6:16]) + 10
363 363
364 364 self.flag_nextfile = False
365 365 else:
366 366 print(filename)
367 367 print("PRIMERA CONDICION")
368 368 # if self.filename_next_set== int(filename[6:16]):
369 369 print("TODO BIEN")
370 370
371 371 if filename == None:
372 372 raise ValueError("corregir")
373 373
374 374 self.dirnameList = [filename]
375 375 fullfilename = self.path + "/" + filename
376 376 self.filenameList = [fullfilename]
377 377 self.filename_next_set = int(filename[6:16]) + 10
378 378 print("Setting next file", self.filename_next_set)
379 379 self.set = int(filename[6:16])
380 380 if True:
381 381 pass
382 382 else:
383 383 print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
384 384
385 385 else:
386 386 filename = getlastFileFromPath(self.path, self.ext)
387 387
388 388 if self.flag_nextfile == True:
389 389 self.dirnameList = [filename]
390 390 fullfilename = self.path + "/" + filename
391 391 self.filenameList = [self.filenameList[-1]]
392 392 self.filename_next_set = int(filename[6:16]) + 10
393 393
394 394 self.flag_nextfile = False
395 395 else:
396 396 filename = getFileFromSet(self.path, self.ext, self.set)
397 397 print(filename)
398 398 print("PRIMERA CONDICION")
399 399 # if self.filename_next_set== int(filename[6:16]):
400 400 print("TODO BIEN")
401 401
402 402 if filename == None:
403 403 raise ValueError("corregir")
404 404
405 405 self.dirnameList = [filename]
406 406 fullfilename = self.path + "/" + filename
407 407 self.filenameList = [fullfilename]
408 408 self.filename_next_set = int(filename[6:16]) + 10
409 409 print("Setting next file", self.filename_next_set)
410 410 self.set = int(filename[6:16])
411 411 if True:
412 412 pass
413 413 else:
414 414 print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
415 415
416 416
417 417
418 418 def searchFilesOffLine(self,
419 419 path,
420 420 startDate,
421 421 endDate,
422 422 ext,
423 423 startTime=datetime.time(0, 0, 0),
424 424 endTime=datetime.time(23, 59, 59),
425 425 walk=True):
426 426
427 427 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
428 428
429 429 self.__checkPath()
430 430
431 431 self.__findDataForDates()
432 432 # print self.dirnameList
433 433
434 434 self.__selectDataForTimes()
435 435
436 436 for i in range(len(self.filenameList)):
437 437 print("%s" % (self.filenameList[i]))
438 438
439 439 return
440 440
441 441 def searchFilesOnLine(self,
442 442 path,
443 443 expLabel="",
444 444 ext=None,
445 445 startDate=None,
446 446 endDate=None,
447 447 walk=True,
448 448 set=None):
449 449
450 450
451 451 startDate = datetime.datetime.utcnow().date()
452 452 endDate = datetime.datetime.utcnow().date()
453 453
454 454 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
455 455
456 456 self.__checkPath()
457 457
458 458 fullpath = path
459 459 print("%s folder was found: " % (fullpath))
460 460
461 461 if set == None:
462 462 self.set = None
463 463 filename = getlastFileFromPath(fullpath, ext)
464 464 startDate = datetime.datetime.utcnow().date
465 465 endDate = datetime.datetime.utcnow().date()
466 466 #
467 467 else:
468 468 filename = getFileFromSet(fullpath, ext, set)
469 469 startDate = None
470 470 endDate = None
471 471 #
472 472 if not (filename):
473 473 return None, None, None, None, None
474 474 # print "%s file was found" %(filename)
475 475
476 476 #
477 477 # dir_hf_filename= self.path+"/"+filename
478 478 # fp= h5py.File(dir_hf_filename,'r')
479 479 # hipoc=fp['t'].value
480 480 # fp.close()
481 481 # date_time=datetime.datetime.utcfromtimestamp(hipoc)
482 482 #
483 483 # year =int(date_time[0:4])
484 484 # month=int(date_time[5:7])
485 485 # dom =int(date_time[8:10])
486 486 # set= int(filename[4:10])
487 487 # self.set=set-1
488 488 # self.dirnameList=[filename]
489 489 filenameList = fullpath + "/" + filename
490 490 self.dirnameList = [filename]
491 491 self.filenameList = [filenameList]
492 492 self.flag_nextfile = True
493 493
494 494 # self.__findDataForDates(online=True)
495 495 # self.dirnameList=[self.dirnameList[-1]]
496 496 # print self.dirnameList
497 497 # self.__selectDataForTimes(online=True)
498 498 # return fullpath,filename,year,month,dom,set
499 499 return
500 500
501 501 def __setNextFile(self, online=False):
502 502 """
503 503 """
504 504 if not(online):
505 505 newFile = self.__setNextFileOffline()
506 506 else:
507 507 newFile = self.__setNextFileOnline()
508 508
509 509 if not(newFile):
510 510 return 0
511 511 return 1
512 512
513 513 def __setNextFileOffline(self):
514 514 """
515 515 """
516 516 idFile = self.fileIndex
517 517 while(True):
518 518 idFile += 1
519 519 if not (idFile < len(self.filenameList)):
520 520 self.flagNoMoreFiles = 1
521 521 print("No more Files")
522 522 return 0
523 523 filename = self.filenameList[idFile]
524 524 hfFilePointer = h5py.File(filename, 'r')
525 525
526 526 epoc = hfFilePointer['t'].value
527 527 # this_time=datetime.datetime(year,month,dom,hour,min,sec)
528 528 break
529 529
530 530 self.flagIsNewFile = 1
531 531 self.fileIndex = idFile
532 532 self.filename = filename
533 533
534 534 self.hfFilePointer = hfFilePointer
535 535 hfFilePointer.close()
536 536 self.__t0 = epoc
537 537 print("Setting the file: %s" % self.filename)
538 538
539 539 return 1
540 540
541 541 def __setNextFileOnline(self):
542 542 """
543 543 """
544 544 print("SOY NONE", self.set)
545 545 if self.set == None:
546 546 pass
547 547 else:
548 548 self.set += 10
549 549
550 550 filename = self.filenameList[0] # fullfilename
551 551 if self.filename_online != None:
552 552 self.__selectDataForTimes(online=True)
553 553 filename = self.filenameList[0]
554 554 while self.filename_online == filename:
555 555 print('waiting %d seconds to get a new file...' % (self.__waitForNewFile))
556 556 time.sleep(self.__waitForNewFile)
557 557 # self.__findDataForDates(online=True)
558 558 self.set = self.filename_next_set
559 559 self.__selectDataForTimes(online=True)
560 560 filename = self.filenameList[0]
561 561 sizeoffile = os.path.getsize(filename)
562 562
563 563 # print filename
564 564 sizeoffile = os.path.getsize(filename)
565 565 if sizeoffile < 1670240:
566 566 print("%s is not the rigth size" % filename)
567 567 delay = 50
568 568 print('waiting %d seconds for delay...' % (delay))
569 569 time.sleep(delay)
570 570 sizeoffile = os.path.getsize(filename)
571 571 if sizeoffile < 1670240:
572 572 delay = 50
573 573 print('waiting %d more seconds for delay...' % (delay))
574 574 time.sleep(delay)
575 575
576 576 sizeoffile = os.path.getsize(filename)
577 577 if sizeoffile < 1670240:
578 578 delay = 50
579 579 print('waiting %d more seconds for delay...' % (delay))
580 580 time.sleep(delay)
581 581
582 582 try:
583 583 hfFilePointer = h5py.File(filename, 'r')
584 584
585 585 except:
586 586 print("Error reading file %s" % filename)
587 587
588 588 self.filename_online = filename
589 589 epoc = hfFilePointer['t'].value
590 590
591 591 self.hfFilePointer = hfFilePointer
592 592 hfFilePointer.close()
593 593 self.__t0 = epoc
594 594
595 595
596 596 self.flagIsNewFile = 1
597 597 self.filename = filename
598 598
599 599 print("Setting the file: %s" % self.filename)
600 600 return 1
601 601
602 602 def __getExpParameters(self):
603 603 if not(self.status):
604 604 return None
605 605
606 606 def setup(self,
607 607 path=None,
608 608 startDate=None,
609 609 endDate=None,
610 610 startTime=datetime.time(0, 0, 0),
611 611 endTime=datetime.time(23, 59, 59),
612 612 set=None,
613 613 expLabel="",
614 614 ext=None,
615 615 all=0,
616 616 timezone=0,
617 617 online=False,
618 618 delay=60,
619 619 walk=True):
620 620 '''
621 621 In this method we should set all initial parameters.
622 622
623 623 '''
624 624 if path == None:
625 625 raise ValueError("The path is not valid")
626 626
627 627 if ext == None:
628 628 ext = self.ext
629 629
630 630 self.timezone = timezone
631 631 self.online = online
632 632 self.all = all
633 633 # if set==None:
634 634
635 635 # print set
636 636 if not(online):
637 637 print("Searching files in offline mode...")
638 638
639 639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 640 else:
641 641 print("Searching files in online mode...")
642 642 self.searchFilesOnLine(path, walk, ext, set=set)
643 643 if set == None:
644 644 pass
645 645 else:
646 646 self.set = set - 10
647 647
648 648 # for nTries in range(self.nTries):
649 649 #
650 650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 651 #
652 652 # if fullpath:
653 653 # break
654 654 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
655 655 # time.sleep(self.delay)
656 656 # if not(fullpath):
657 657 # print "There ins't valid files in %s" % path
658 658 # return None
659 659
660 660
661 661 if not(self.filenameList):
662 662 print("There is no files into the folder: %s" % (path))
663 663 sys.exit(-1)
664 664
665 665 self.__getExpParameters()
666 666
667 667
668 668 self.fileIndex = -1
669 669
670 670 self.__setNextFile(online)
671 671
672 672 self.__readMetadata()
673 673
674 674 self.__setLocalVariables()
675 675
676 676 self.__setHeaderDO()
677 677 # self.profileIndex_offset= 0
678 678
679 679 # self.profileIndex = self.profileIndex_offset
680 680
681 681 self.isConfig = True
682 682
683 683 def __readMetadata(self):
684 684 self.__readHeader()
685 685
686 686
687 687 def __setLocalVariables(self):
688 688
689 self.datablock = numpy.zeros((self.nChannels, self.nHeights, self.nProfiles), dtype=numpy.complex)
689 self.datablock = numpy.zeros((self.nChannels, self.nHeights, self.nProfiles), dtype=complex)
690 690 #
691 691
692 692
693 693
694 694 self.profileIndex = 9999
695 695
696 696
697 697 def __setHeaderDO(self):
698 698
699 699
700 700 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
701 701
702 702 self.dataOut.systemHeaderObj = SystemHeader()
703 703
704 704
705 705 #---------------------------------------------------------
706 706 self.dataOut.systemHeaderObj.nProfiles = 100
707 707 self.dataOut.systemHeaderObj.nSamples = 1000
708 708
709 709
710 710 SAMPLING_STRUCTURE = [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
711 711 self.dataOut.radarControllerHeaderObj.samplingWindow = numpy.zeros((1,), SAMPLING_STRUCTURE)
712 712 self.dataOut.radarControllerHeaderObj.samplingWindow['h0'] = 0
713 713 self.dataOut.radarControllerHeaderObj.samplingWindow['dh'] = 1.5
714 714 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'] = 1000
715 715 self.dataOut.radarControllerHeaderObj.nHeights = int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
716 716 self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
717 717 self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
718 718 self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
719 719
720 720 self.dataOut.radarControllerHeaderObj.nWindows = 1
721 721 self.dataOut.radarControllerHeaderObj.codetype = 0
722 722 self.dataOut.radarControllerHeaderObj.numTaus = 0
723 723 # self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
724 724
725 725
726 726 # self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
727 727 # self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
728 728 # self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
729 729
730 730 self.dataOut.radarControllerHeaderObj.code_size = 0
731 731 self.dataOut.nBaud = 0
732 732 self.dataOut.nCode = 0
733 733 self.dataOut.nPairs = 0
734 734
735 735
736 736 #---------------------------------------------------------
737 737
738 738 self.dataOut.type = "Voltage"
739 739
740 740 self.dataOut.data = None
741 741
742 742 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
743 743
744 744 self.dataOut.nProfiles = 1
745 745
746 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float) * self.__deltaHeigth
746 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float32) * self.__deltaHeigth
747 747
748 748 self.dataOut.channelList = list(range(self.nChannels))
749 749
750 750 # self.dataOut.channelIndexList = None
751 751
752 752 self.dataOut.flagNoData = True
753 753
754 754 # Set to TRUE if the data is discontinuous
755 755 self.dataOut.flagDiscontinuousBlock = False
756 756
757 757 self.dataOut.utctime = None
758 758
759 759 self.dataOut.timeZone = self.timezone
760 760
761 761 self.dataOut.dstFlag = 0
762 762
763 763 self.dataOut.errorCount = 0
764 764
765 765 self.dataOut.nCohInt = 1
766 766
767 767 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
768 768
769 769 self.dataOut.flagDecodeData = False # asumo que la data esta decodificada
770 770
771 771 self.dataOut.flagDeflipData = False # asumo que la data esta sin flip
772 772
773 773 self.dataOut.flagShiftFFT = False
774 774
775 775 self.dataOut.ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
776 776
777 777 # Time interval between profiles
778 778 # self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
779 779
780 780
781 781 self.dataOut.frequency = self.__frequency
782 782
783 783 self.dataOut.realtime = self.__online
784 784
785 785 def __hasNotDataInBuffer(self):
786 786
787 787 if self.profileIndex >= self.nProfiles:
788 788 return 1
789 789
790 790 return 0
791 791
792 792 def readNextBlock(self):
793 793 if not(self.__setNewBlock()):
794 794 return 0
795 795
796 796 if not(self.readBlock()):
797 797 return 0
798 798
799 799 return 1
800 800
801 801 def __setNewBlock(self):
802 802
803 803 if self.hfFilePointer == None:
804 804 return 0
805 805
806 806 if self.flagIsNewFile:
807 807 return 1
808 808
809 809 if self.profileIndex < self.nProfiles:
810 810 return 1
811 811
812 812 self.__setNextFile(self.online)
813 813
814 814 return 1
815 815
816 816
817 817
818 818 def readBlock(self):
819 819 fp = h5py.File(self.filename, 'r')
820 820 # Puntero que apunta al archivo hdf5
821 821 ch0 = (fp['ch0']).value # Primer canal (100,1000)--(perfiles,alturas)
822 822 ch1 = (fp['ch1']).value # Segundo canal (100,1000)--(perfiles,alturas)
823 823 fp.close()
824 824 ch0 = ch0.swapaxes(0, 1) # Primer canal (100,1000)--(alturas,perfiles)
825 825 ch1 = ch1.swapaxes(0, 1) # Segundo canal (100,1000)--(alturas,perfiles)
826 826 self.datablock = numpy.array([ch0, ch1])
827 827 self.flagIsNewFile = 0
828 828
829 829 self.profileIndex = 0
830 830
831 831 return 1
832 832
833 833 def getData(self):
834 834 if self.flagNoMoreFiles:
835 835 self.dataOut.flagNoData = True
836 836 return 0
837 837
838 838 if self.__hasNotDataInBuffer():
839 839 if not(self.readNextBlock()):
840 840 self.dataOut.flagNodata = True
841 841 return 0
842 842
843 843 ##############################
844 844 ##############################
845 845 self.dataOut.data = self.datablock[:, :, self.profileIndex]
846 846 self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds * self.profileIndex
847 847 self.dataOut.profileIndex = self.profileIndex
848 848 self.dataOut.flagNoData = False
849 849 self.profileIndex += 1
850 850
851 851 return self.dataOut.data
852 852
853 853
854 854 def run(self, **kwargs):
855 855 '''
856 856 This method will be called many times so here you should put all your code
857 857 '''
858 858
859 859 if not self.isConfig:
860 860 self.setup(**kwargs)
861 861 self.isConfig = True
862 862 self.getData()
Show file before | Show file after | Hide comments
@@ -1,873 +1,887
1 1 import os
2 2 import time
3 3 import datetime
4 4
5 5 import numpy
6 6 import h5py
7 7
8 8 import schainpy.admin
9 9 from schainpy.model.data.jrodata import *
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.io.jroIO_base import *
12 12 from schainpy.utils import log
13 13
14 14
15 15 class HDFReader(Reader, ProcessingUnit):
16 16 """Processing unit to read HDF5 format files
17 17
18 18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 20 attributes.
21 21 It is possible to read any HDF5 file by given the structure in the `description`
22 22 parameter, also you can add extra values to metadata with the parameter `extras`.
23 23
24 24 Parameters:
25 25 -----------
26 26 path : str
27 27 Path where files are located.
28 28 startDate : date
29 29 Start date of the files
30 30 endDate : list
31 31 End date of the files
32 32 startTime : time
33 33 Start time of the files
34 34 endTime : time
35 35 End time of the files
36 36 description : dict, optional
37 37 Dictionary with the description of the HDF5 file
38 38 extras : dict, optional
39 39 Dictionary with extra metadata to be be added to `dataOut`
40 40
41 41 Examples
42 42 --------
43 43
44 44 desc = {
45 45 'Data': {
46 46 'data_output': ['u', 'v', 'w'],
47 47 'utctime': 'timestamps',
48 48 } ,
49 49 'Metadata': {
50 50 'heightList': 'heights'
51 51 }
52 52 }
53 53
54 54 desc = {
55 55 'Data': {
56 56 'data_output': 'winds',
57 57 'utctime': 'timestamps'
58 58 },
59 59 'Metadata': {
60 60 'heightList': 'heights'
61 61 }
62 62 }
63 63
64 64 extras = {
65 65 'timeZone': 300
66 66 }
67 67
68 68 reader = project.addReadUnit(
69 69 name='HDFReader',
70 70 path='/path/to/files',
71 71 startDate='2019/01/01',
72 72 endDate='2019/01/31',
73 73 startTime='00:00:00',
74 74 endTime='23:59:59',
75 75 # description=json.dumps(desc),
76 76 # extras=json.dumps(extras),
77 77 )
78 78
79 79 """
80 80
81 81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 82
83 83 def __init__(self):
84 84 ProcessingUnit.__init__(self)
85 85 self.dataOut = Parameters()
86 86 self.ext = ".hdf5"
87 87 self.optchar = "D"
88 88 self.meta = {}
89 89 self.data = {}
90 90 self.open_file = h5py.File
91 91 self.open_mode = 'r'
92 92 self.description = {}
93 93 self.extras = {}
94 94 self.filefmt = "*%Y%j***"
95 95 self.folderfmt = "*%Y%j"
96 96 self.utcoffset = 0
97 97
98 98 def setup(self, **kwargs):
99 99
100 100 self.set_kwargs(**kwargs)
101 101 if not self.ext.startswith('.'):
102 102 self.ext = '.{}'.format(self.ext)
103 103
104 104 if self.online:
105 105 log.log("Searching files in online mode...", self.name)
106 106
107 107 for nTries in range(self.nTries):
108 108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 109 self.endDate, self.expLabel, self.ext, self.walk,
110 110 self.filefmt, self.folderfmt)
111 111 try:
112 112 fullpath = next(fullpath)
113 113 except:
114 114 fullpath = None
115 115
116 116 if fullpath:
117 117 break
118 118
119 119 log.warning(
120 120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 121 self.delay, self.path, nTries + 1),
122 122 self.name)
123 123 time.sleep(self.delay)
124 124
125 125 if not(fullpath):
126 126 raise schainpy.admin.SchainError(
127 127 'There isn\'t any valid file in {}'.format(self.path))
128 128
129 129 pathname, filename = os.path.split(fullpath)
130 130 self.year = int(filename[1:5])
131 131 self.doy = int(filename[5:8])
132 132 self.set = int(filename[8:11]) - 1
133 133 else:
134 134 log.log("Searching files in {}".format(self.path), self.name)
135 135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137 137
138 138 self.setNextFile()
139 139
140 140 return
141 141
142 142 def readFirstHeader(self):
143 143 '''Read metadata and data'''
144 144
145 145 self.__readMetadata()
146 146 self.__readData()
147 147 self.__setBlockList()
148 148
149 149 for attr in self.meta:
150 150 setattr(self.dataOut, attr, self.meta[attr])
151 151
152 152 self.blockIndex = 0
153 153
154 154 return
155 155
156 156 def __setBlockList(self):
157 157 '''
158 158 Selects the data within the times defined
159 159
160 160 self.fp
161 161 self.startTime
162 162 self.endTime
163 163 self.blockList
164 164 self.blocksPerFile
165 165
166 166 '''
167 167
168 168 startTime = self.startTime
169 169 endTime = self.endTime
170 170 thisUtcTime = self.data['utctime'] + self.utcoffset
171 171 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
172 172 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
173 173
174 174 thisDate = thisDatetime.date()
175 175 thisTime = thisDatetime.time()
176 176
177 177 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
178 178 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
179 179
180 180 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
181 181
182 182 self.blockList = ind
183 183 self.blocksPerFile = len(ind)
184 184 return
185 185
186 186 def __readMetadata(self):
187 187 '''
188 188 Reads Metadata
189 189 '''
190 190
191 191 meta = {}
192 192
193 193 if self.description:
194 194 for key, value in self.description['Metadata'].items():
195 195 meta[key] = self.fp[value][()]
196 196 else:
197 197 grp = self.fp['Metadata']
198 198 for name in grp:
199 199 meta[name] = grp[name][()]
200 200
201 201 if self.extras:
202 202 for key, value in self.extras.items():
203 203 meta[key] = value
204 204 self.meta = meta
205 205
206 206 return
207 207
208 208 def __readData(self):
209 209
210 210 data = {}
211 211
212 212 if self.description:
213 213 for key, value in self.description['Data'].items():
214 214 if isinstance(value, str):
215 215 if isinstance(self.fp[value], h5py.Dataset):
216 216 data[key] = self.fp[value][()]
217 217 elif isinstance(self.fp[value], h5py.Group):
218 218 array = []
219 219 for ch in self.fp[value]:
220 220 array.append(self.fp[value][ch][()])
221 221 data[key] = numpy.array(array)
222 222 elif isinstance(value, list):
223 223 array = []
224 224 for ch in value:
225 225 array.append(self.fp[ch][()])
226 226 data[key] = numpy.array(array)
227 227 else:
228 228 grp = self.fp['Data']
229 229 for name in grp:
230 230 if isinstance(grp[name], h5py.Dataset):
231 231 array = grp[name][()]
232 232 elif isinstance(grp[name], h5py.Group):
233 233 array = []
234 234 for ch in grp[name]:
235 235 array.append(grp[name][ch][()])
236 236 array = numpy.array(array)
237 237 else:
238 238 log.warning('Unknown type: {}'.format(name))
239 239
240 240 if name in self.description:
241 241 key = self.description[name]
242 242 else:
243 243 key = name
244 244 data[key] = array
245 245
246 246 self.data = data
247 247 return
248 248
249 249 def getData(self):
250 250
251 251 for attr in self.data:
252 252 if self.data[attr].ndim == 1:
253 253 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
254 254 else:
255 255 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
256 256
257 257 self.dataOut.flagNoData = False
258 258 self.blockIndex += 1
259 259
260 260 log.log("Block No. {}/{} -> {}".format(
261 261 self.blockIndex,
262 262 self.blocksPerFile,
263 263 self.dataOut.datatime.ctime()), self.name)
264 264
265 265 return
266 266
267 267 def run(self, **kwargs):
268 268
269 269 if not(self.isConfig):
270 270 self.setup(**kwargs)
271 271 self.isConfig = True
272 272
273 273 if self.blockIndex == self.blocksPerFile:
274 274 self.setNextFile()
275 275
276 276 self.getData()
277 277
278 278 if 'type' in self.meta:
279 279 self.dataOut.type = self.meta['type'].decode('utf-8')
280 280
281 281 return
282 282
283 283 @MPDecorator
284 284 class HDFWriter(Operation):
285 285 """Operation to write HDF5 files.
286 286
287 287 The HDF5 file contains by default two groups Data and Metadata where
288 288 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
289 289 parameters, data attributes are normaly time dependent where the metadata
290 290 are not.
291 291 It is possible to customize the structure of the HDF5 file with the
292 292 optional description parameter see the examples.
293 293
294 294 Parameters:
295 295 -----------
296 296 path : str
297 297 Path where files will be saved.
298 298 blocksPerFile : int
299 299 Number of blocks per file
300 300 metadataList : list
301 301 List of the dataOut attributes that will be saved as metadata
302 302 dataList : int
303 303 List of the dataOut attributes that will be saved as data
304 304 setType : bool
305 305 If True the name of the files corresponds to the timestamp of the data
306 306 description : dict, optional
307 307 Dictionary with the desired description of the HDF5 file
308 308
309 309 Examples
310 310 --------
311 311
312 312 desc = {
313 313 'data_output': {'winds': ['z', 'w', 'v']},
314 314 'utctime': 'timestamps',
315 315 'heightList': 'heights'
316 316 }
317 317 desc = {
318 318 'data_output': ['z', 'w', 'v'],
319 319 'utctime': 'timestamps',
320 320 'heightList': 'heights'
321 321 }
322 322 desc = {
323 323 'Data': {
324 324 'data_output': 'winds',
325 325 'utctime': 'timestamps'
326 326 },
327 327 'Metadata': {
328 328 'heightList': 'heights'
329 329 }
330 330 }
331 331
332 332 writer = proc_unit.addOperation(name='HDFWriter')
333 333 writer.addParameter(name='path', value='/path/to/file')
334 334 writer.addParameter(name='blocksPerFile', value='32')
335 335 writer.addParameter(name='metadataList', value='heightList,timeZone')
336 336 writer.addParameter(name='dataList',value='data_output,utctime')
337 337 # writer.addParameter(name='description',value=json.dumps(desc))
338 338
339 339 """
340 340
341 341 ext = ".hdf5"
342 342 optchar = "D"
343 343 filename = None
344 344 path = None
345 345 setFile = None
346 346 fp = None
347 347 firsttime = True
348 348 # Configurations
349 349 blocksPerFile = None
350 350 blockIndex = None
351 351 dataOut = None
352 352 # Data Arrays
353 353 dataList = None
354 354 metadataList = None
355 355 currentDay = None
356 356 lastTime = None
357 357
358 358 def __init__(self):
359 359
360 360 Operation.__init__(self)
361 361 return
362 362
363 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
363 def set_kwargs(self, **kwargs):
364
365 for key, value in kwargs.items():
366 setattr(self, key, value)
367
368 def set_kwargs_obj(self, obj, **kwargs):
369
370 for key, value in kwargs.items():
371 setattr(obj, key, value)
372
373 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
364 374 self.path = path
365 375 self.blocksPerFile = blocksPerFile
366 376 self.metadataList = metadataList
367 377 self.dataList = [s.strip() for s in dataList]
368 378 self.setType = setType
369 379 self.description = description
380 self.set_kwargs(**kwargs)
370 381
371 382 if self.metadataList is None:
372 383 self.metadataList = self.dataOut.metadata_list
373 384
374 385 tableList = []
375 386 dsList = []
376 387
377 388 for i in range(len(self.dataList)):
378 389 dsDict = {}
379 390 if hasattr(self.dataOut, self.dataList[i]):
380 391 dataAux = getattr(self.dataOut, self.dataList[i])
381 392 dsDict['variable'] = self.dataList[i]
382 393 else:
383 394 log.warning('Attribute {} not found in dataOut', self.name)
384 395 continue
385 396
386 397 if dataAux is None:
387 398 continue
388 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
399 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
389 400 dsDict['nDim'] = 0
390 401 else:
391 402 dsDict['nDim'] = len(dataAux.shape)
392 403 dsDict['shape'] = dataAux.shape
393 404 dsDict['dsNumber'] = dataAux.shape[0]
394 405 dsDict['dtype'] = dataAux.dtype
395 406
396 407 dsList.append(dsDict)
397 408
398 409 self.dsList = dsList
399 410 self.currentDay = self.dataOut.datatime.date()
400 411
401 412 def timeFlag(self):
402 413 currentTime = self.dataOut.utctime
403 414 timeTuple = time.localtime(currentTime)
404 415 dataDay = timeTuple.tm_yday
405 416
406 417 if self.lastTime is None:
407 418 self.lastTime = currentTime
408 419 self.currentDay = dataDay
409 420 return False
410 421
411 422 timeDiff = currentTime - self.lastTime
412 423
413 424 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
414 425 if dataDay != self.currentDay:
415 426 self.currentDay = dataDay
416 427 return True
417 428 elif timeDiff > 3 * 60 * 60:
418 429 self.lastTime = currentTime
419 430 return True
420 431 else:
421 432 self.lastTime = currentTime
422 433 return False
423 434
424 435 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
425 dataList=[], setType=None, description={}):
436 dataList=[], setType=None, description={}, **kwargs):
426 437
427 438 self.dataOut = dataOut
439 self.set_kwargs_obj(self.dataOut, **kwargs)
428 440 if not(self.isConfig):
429 441 self.setup(path=path, blocksPerFile=blocksPerFile,
430 442 metadataList=metadataList, dataList=dataList,
431 setType=setType, description=description)
443 setType=setType, description=description, **kwargs)
432 444
433 445 self.isConfig = True
434 446 self.setNextFile()
435 447
436 448 self.putData()
437 449 return
438 450
439 451 def setNextFile(self):
440 452
441 453 ext = self.ext
442 454 path = self.path
443 455 setFile = self.setFile
444 456
445 457 timeTuple = time.localtime(self.dataOut.utctime)
446 458 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
447 459 fullpath = os.path.join(path, subfolder)
448 460
449 461 if os.path.exists(fullpath):
450 462 filesList = os.listdir(fullpath)
451 463 filesList = [k for k in filesList if k.startswith(self.optchar)]
452 464 if len(filesList) > 0:
453 465 filesList = sorted(filesList, key=str.lower)
454 466 filen = filesList[-1]
455 467 # el filename debera tener el siguiente formato
456 468 # 0 1234 567 89A BCDE (hex)
457 469 # x YYYY DDD SSS .ext
458 470 if isNumber(filen[8:11]):
459 471 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
460 472 else:
461 473 setFile = -1
462 474 else:
463 475 setFile = -1 # inicializo mi contador de seteo
464 476 else:
465 477 os.makedirs(fullpath)
466 478 setFile = -1 # inicializo mi contador de seteo
467 479
468 480 if self.setType is None:
469 481 setFile += 1
470 482 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
471 483 timeTuple.tm_year,
472 484 timeTuple.tm_yday,
473 485 setFile,
474 486 ext)
475 487 else:
476 488 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
477 489 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
478 490 timeTuple.tm_year,
479 491 timeTuple.tm_yday,
480 492 setFile,
481 493 ext)
482 494
483 495 self.filename = os.path.join(path, subfolder, file)
484 496
485 497 # Setting HDF5 File
486 498 self.fp = h5py.File(self.filename, 'w')
487 499 # write metadata
488 500 self.writeMetadata(self.fp)
489 501 # Write data
490 502 self.writeData(self.fp)
491 503
492 504 def getLabel(self, name, x=None):
493 505
494 506 if x is None:
495 507 if 'Data' in self.description:
496 508 data = self.description['Data']
497 509 if 'Metadata' in self.description:
498 510 data.update(self.description['Metadata'])
499 511 else:
500 512 data = self.description
501 513 if name in data:
502 514 if isinstance(data[name], str):
503 515 return data[name]
504 516 elif isinstance(data[name], list):
505 517 return None
506 518 elif isinstance(data[name], dict):
507 519 for key, value in data[name].items():
508 520 return key
509 521 return name
510 522 else:
511 if 'Metadata' in self.description:
512 meta = self.description['Metadata']
523 if 'Data' in self.description:
524 data = self.description['Data']
525 if 'Metadata' in self.description:
526 data.update(self.description['Metadata'])
513 527 else:
514 meta = self.description
515 if name in meta:
516 if isinstance(meta[name], list):
517 return meta[name][x]
518 elif isinstance(meta[name], dict):
519 for key, value in meta[name].items():
528 data = self.description
529 if name in data:
530 if isinstance(data[name], list):
531 return data[name][x]
532 elif isinstance(data[name], dict):
533 for key, value in data[name].items():
520 534 return value[x]
521 535 if 'cspc' in name:
522 536 return 'pair{:02d}'.format(x)
523 537 else:
524 538 return 'channel{:02d}'.format(x)
525 539
526 540 def writeMetadata(self, fp):
527
541
528 542 if self.description:
529 543 if 'Metadata' in self.description:
530 544 grp = fp.create_group('Metadata')
531 545 else:
532 546 grp = fp
533 547 else:
534 548 grp = fp.create_group('Metadata')
535 549
536 550 for i in range(len(self.metadataList)):
537 551 if not hasattr(self.dataOut, self.metadataList[i]):
538 552 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
539 553 continue
540 554 value = getattr(self.dataOut, self.metadataList[i])
541 555 if isinstance(value, bool):
542 556 if value is True:
543 557 value = 1
544 558 else:
545 559 value = 0
546 560 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
547 561 return
548 562
549 563 def writeData(self, fp):
550 564
551 565 if self.description:
552 566 if 'Data' in self.description:
553 567 grp = fp.create_group('Data')
554 568 else:
555 569 grp = fp
556 570 else:
557 571 grp = fp.create_group('Data')
558 572
559 573 dtsets = []
560 574 data = []
561 575
562 576 for dsInfo in self.dsList:
563 577 if dsInfo['nDim'] == 0:
564 578 ds = grp.create_dataset(
565 579 self.getLabel(dsInfo['variable']),
566 580 (self.blocksPerFile,),
567 581 chunks=True,
568 582 dtype=numpy.float64)
569 583 dtsets.append(ds)
570 584 data.append((dsInfo['variable'], -1))
571 585 else:
572 586 label = self.getLabel(dsInfo['variable'])
573 587 if label is not None:
574 588 sgrp = grp.create_group(label)
575 589 else:
576 590 sgrp = grp
577 591 for i in range(dsInfo['dsNumber']):
578 592 ds = sgrp.create_dataset(
579 593 self.getLabel(dsInfo['variable'], i),
580 594 (self.blocksPerFile,) + dsInfo['shape'][1:],
581 595 chunks=True,
582 596 dtype=dsInfo['dtype'])
583 597 dtsets.append(ds)
584 598 data.append((dsInfo['variable'], i))
585 599 fp.flush()
586 600
587 601 log.log('Creating file: {}'.format(fp.filename), self.name)
588 602
589 603 self.ds = dtsets
590 604 self.data = data
591 605 self.firsttime = True
592 606 self.blockIndex = 0
593 607 return
594 608
595 609 def putData(self):
596 610
597 611 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
598 612 self.closeFile()
599 613 self.setNextFile()
600 614
601 615 for i, ds in enumerate(self.ds):
602 616 attr, ch = self.data[i]
603 617 if ch == -1:
604 618 ds[self.blockIndex] = getattr(self.dataOut, attr)
605 619 else:
606 620 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
607 621
608 622 self.fp.flush()
609 623 self.blockIndex += 1
610 624 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
611 625
612 626 return
613 627
614 628 def closeFile(self):
615 629
616 630 if self.blockIndex != self.blocksPerFile:
617 631 for ds in self.ds:
618 632 ds.resize(self.blockIndex, axis=0)
619 633
620 634 if self.fp:
621 635 self.fp.flush()
622 636 self.fp.close()
623 637
624 638 def close(self):
625 639
626 640 self.closeFile()
627 641
628 642
629 643 @MPDecorator
630 644 class ASCIIWriter(Operation):
631 645 """Operation to write data in ascii files.
632 646
633 647 Parameters:
634 648 -----------
635 649 path : str
636 650 Path where files will be saved.
637 651 blocksPerFile : int
638 652 Number of blocks per file
639 653 metadataList : list
640 654 List of the dataOut attributes that will be saved as metadata
641 655 dataDict : dict
642 656 Dictionary with the varaibles to be saved
643 657 setType : bool
644 658 If True the name of the files corresponds to the timestamp of the data
645 659
646 660 Examples
647 661 --------
648 662
649 663 data = {
650 664 'data_output': ['z', 'w', 'v'],
651 665 'utctime': 'time',
652 666 'heightList': 'height'
653 667 }
654 668
655 669 writer = proc_unit.addOperation(name='ASCIIWriter')
656 670 writer.addParameter(name='path', value='/path/to/file')
657 671 writer.addParameter(name='blocksPerFile', value='32')
658 672 writer.addParameter(name='dataDict',value=json.dumps(data))
659 673
660 674 """
661 675
662 676 ext = ".txt"
663 677 optchar = "D"
664 678 filename = None
665 679 path = None
666 680 setFile = None
667 681 fp = None
668 682 firsttime = True
669 683 # Configurations
670 684 blocksPerFile = None
671 685 blockIndex = None
672 686 dataOut = None
673 687 # Data Arrays
674 688 dataDict = None
675 689 metadataList = None
676 690 currentDay = None
677 691 lastTime = None
678 692 localtime = True
679 693
680 694 def __init__(self):
681 695
682 696 Operation.__init__(self)
683 697 return
684 698
685 699 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataDict=None, setType=None, localtime=True):
686 700 self.path = path
687 701 self.blocksPerFile = blocksPerFile
688 702 self.metadataList = metadataList
689 703 self.dataDict = dataDict
690 704 self.setType = setType
691 705 self.localtime = localtime
692 706
693 707 if self.metadataList is None:
694 708 self.metadataList = self.dataOut.metadata_list
695 709
696 710 dsList = []
697 711
698 712 for key, value in self.dataDict.items():
699 713 dsDict = {}
700 714 if hasattr(self.dataOut, key):
701 715 dataAux = getattr(self.dataOut, key)
702 716 dsDict['variable'] = key
703 717 else:
704 718 log.warning('Attribute {} not found in dataOut', self.name)
705 719 continue
706 720
707 721 if dataAux is None:
708 722 continue
709 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
723 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
710 724 dsDict['nDim'] = 0
711 725 else:
712 726 dsDict['nDim'] = len(dataAux.shape)
713 727 dsDict['shape'] = dataAux.shape
714 728 dsDict['dsNumber'] = dataAux.shape[0]
715 729 dsDict['dtype'] = dataAux.dtype
716 730
717 731 dsList.append(dsDict)
718 732 self.dsList = dsList
719 733 self.currentDay = self.dataOut.datatime.date()
720 734
721 735 def timeFlag(self):
722 736 currentTime = self.dataOut.utctime
723 737 if self.localtime:
724 738 timeTuple = time.localtime(currentTime)
725 739 else:
726 740 timeTuple = time.gmtime(currentTime)
727 741
728 742 dataDay = timeTuple.tm_yday
729 743
730 744 if self.lastTime is None:
731 745 self.lastTime = currentTime
732 746 self.currentDay = dataDay
733 747 return False
734 748
735 749 timeDiff = currentTime - self.lastTime
736 750
737 751 # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
738 752 if dataDay != self.currentDay:
739 753 self.currentDay = dataDay
740 754 return True
741 755 elif timeDiff > 3 * 60 * 60:
742 756 self.lastTime = currentTime
743 757 return True
744 758 else:
745 759 self.lastTime = currentTime
746 760 return False
747 761
748 762 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
749 763 dataDict={}, setType=None, localtime=True):
750 764
751 765 self.dataOut = dataOut
752 766 if not(self.isConfig):
753 767 self.setup(path=path, blocksPerFile=blocksPerFile,
754 768 metadataList=metadataList, dataDict=dataDict,
755 769 setType=setType, localtime=localtime)
756 770
757 771 self.isConfig = True
758 772 self.setNextFile()
759 773
760 774 self.putData()
761 775 return
762 776
763 777 def setNextFile(self):
764 778
765 779 ext = self.ext
766 780 path = self.path
767 781 setFile = self.setFile
768 782 if self.localtime:
769 783 timeTuple = time.localtime(self.dataOut.utctime)
770 784 else:
771 785 timeTuple = time.gmtime(self.dataOut.utctime)
772 786 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
773 787 fullpath = os.path.join(path, subfolder)
774 788
775 789 if os.path.exists(fullpath):
776 790 filesList = os.listdir(fullpath)
777 791 filesList = [k for k in filesList if k.startswith(self.optchar)]
778 792 if len(filesList) > 0:
779 793 filesList = sorted(filesList, key=str.lower)
780 794 filen = filesList[-1]
781 795 # el filename debera tener el siguiente formato
782 796 # 0 1234 567 89A BCDE (hex)
783 797 # x YYYY DDD SSS .ext
784 798 if isNumber(filen[8:11]):
785 799 setFile = int(filen[8:11]) # inicializo mi contador de seteo al seteo del ultimo file
786 800 else:
787 801 setFile = -1
788 802 else:
789 803 setFile = -1 # inicializo mi contador de seteo
790 804 else:
791 805 os.makedirs(fullpath)
792 806 setFile = -1 # inicializo mi contador de seteo
793 807
794 808 if self.setType is None:
795 809 setFile += 1
796 810 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
797 811 timeTuple.tm_year,
798 812 timeTuple.tm_yday,
799 813 setFile,
800 814 ext)
801 815 else:
802 816 setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
803 817 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
804 818 timeTuple.tm_year,
805 819 timeTuple.tm_yday,
806 820 setFile,
807 821 ext)
808 822
809 823 self.filename = os.path.join(path, subfolder, file)
810 824
811 825 # Setting HDF5 File
812 826 self.fp = open(self.filename, 'w')
813 827 # write metadata
814 828 self.writeMetadata(self.fp)
815 829 # Write data
816 830 self.writeData(self.fp)
817 831
818 832 def writeMetadata(self, fp):
819 833
820 834 line = ''
821 835 for d in self.dsList:
822 836 par = self.dataDict[d['variable']]
823 837 if isinstance(par, (list,tuple)):
824 838 for p in par:
825 839 line += '{:>16}'.format(p)
826 840 else:
827 841 line += '{:>16}'.format(par)
828 842
829 843 line += '\n'
830 844 fp.write(line)
831 845
832 846 def writeData(self, fp):
833 847
834 848 log.log('Creating file: {}'.format(self.filename), self.name)
835 849
836 850 self.firsttime = True
837 851 self.blockIndex = 0
838 852 return
839 853
840 854 def putData(self):
841 855
842 856 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
843 857 self.closeFile()
844 858 self.setNextFile()
845 859
846 860 line = ''
847 861 for j in range(len(self.dataOut.heightList)):
848 862 for ds in self.dsList:
849 863 par = self.dataDict[ds['variable']]
850 864 if ds['nDim'] == 2:
851 865 for i in range(len(par)):
852 866 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[i][j])
853 867 elif ds['nDim'] == 1:
854 868 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[j])
855 869 else:
856 870 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable']))
857 871
858 872 line += '\n'
859 873 self.fp.write(line)
860 874
861 875 self.blockIndex += 1
862 876 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
863 877
864 878 return
865 879
866 880 def closeFile(self):
867 881
868 882 if self.fp:
869 883 self.fp.close()
870 884
871 885 def close(self):
872 886
873 887 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,519 +1,519
1 1 import numpy, math, random, time
2 2 #---------------1 Heredamos JRODatareader
3 3 from schainpy.model.io.jroIO_base import *
4 4 #---------------2 Heredamos las propiedades de ProcessingUnit
5 5 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
6 6 #---------------3 Importaremos las clases BascicHeader, SystemHeader, RadarControlHeader, ProcessingHeader
7 7 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
8 8 #---------------4 Importaremos el objeto Voltge
9 9 from schainpy.model.data.jrodata import Voltage
10 10
11 11 class SimulatorReader(JRODataReader, ProcessingUnit):
12 12 incIntFactor = 1
13 13 nFFTPoints = 0
14 14 FixPP_IncInt = 1
15 15 FixRCP_IPP = 1000
16 16 FixPP_CohInt = 1
17 17 Tau_0 = 250
18 18 AcqH0_0 = 70
19 19 H0 = AcqH0_0
20 20 AcqDH_0 = 1.25
21 21 DH0 = AcqDH_0
22 22 Bauds = 32
23 23 BaudWidth = None
24 24 FixRCP_TXA = 40
25 25 FixRCP_TXB = 70
26 26 fAngle = 2.0 * math.pi * (1 / 16)
27 27 DC_level = 500
28 28 stdev = 8
29 29 Num_Codes = 2
30 30 # code0 = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1])
31 31 # code1 = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,0,1,1,1,0,0,0,1,0])
32 32 # Dyn_snCode = numpy.array([Num_Codes,Bauds])
33 33 Dyn_snCode = None
34 34 Samples = 200
35 35 channels = 2
36 36 pulses = None
37 37 Reference = None
38 38 pulse_size = None
39 39 prof_gen = None
40 40 Fdoppler = 100
41 41 Hdoppler = 36
42 42 Adoppler = 300
43 43 frequency = 9345
44 44 nTotalReadFiles = 1000
45 45
46 46 def __init__(self):
47 47 """
48 48 Inicializador de la clases SimulatorReader para
49 49 generar datos de voltage simulados.
50 50 Input:
51 51 dataOut: Objeto de la clase Voltage.
52 52 Este Objeto sera utilizado apra almacenar
53 53 un perfil de datos cada vez qe se haga
54 54 un requerimiento (getData)
55 55 """
56 56 ProcessingUnit.__init__(self)
57 57 print(" [ START ] init - Metodo Simulator Reader")
58 58
59 59 self.isConfig = False
60 60 self.basicHeaderObj = BasicHeader(LOCALTIME)
61 61 self.systemHeaderObj = SystemHeader()
62 62 self.radarControllerHeaderObj = RadarControllerHeader()
63 63 self.processingHeaderObj = ProcessingHeader()
64 64 self.profileIndex = 2 ** 32 - 1
65 65 self.dataOut = Voltage()
66 66 # code0 = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1])
67 67 code0 = numpy.array([1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, 1])
68 68 # code1 = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,0,1,1,1,0,0,0,1,0])
69 69 code1 = numpy.array([1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1])
70 70 # self.Dyn_snCode = numpy.array([code0,code1])
71 71 self.Dyn_snCode = None
72 72
73 73 def set_kwargs(self, **kwargs):
74 74 for key, value in kwargs.items():
75 75 setattr(self, key, value)
76 76
77 77 def __hasNotDataInBuffer(self):
78 78
79 79 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
80 80 if self.nReadBlocks > 0:
81 81 tmp = self.dataOut.utctime
82 82 tmp_utc = int(self.dataOut.utctime)
83 83 tmp_milisecond = int((tmp - tmp_utc) * 1000)
84 84 self.basicHeaderObj.utc = tmp_utc
85 85 self.basicHeaderObj.miliSecond = tmp_milisecond
86 86 return 1
87 87 return 0
88 88
89 89 def setNextFile(self):
90 90 """Set the next file to be readed open it and parse de file header"""
91 91
92 92 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
93 93 self.nReadFiles = self.nReadFiles + 1
94 94 if self.nReadFiles > self.nTotalReadFiles:
95 95 self.flagNoMoreFiles = 1
96 96 raise schainpy.admin.SchainWarning('No more files to read')
97 97
98 98 print('------------------- [Opening file] ------------------------------', self.nReadFiles)
99 99 self.nReadBlocks = 0
100 100 # if self.nReadBlocks==0:
101 101 # self.readFirstHeader()
102 102
103 103 def __setNewBlock(self):
104 104 self.setNextFile()
105 105 if self.flagIsNewFile:
106 106 return 1
107 107
108 108 def readNextBlock(self):
109 109 while True:
110 110 self.__setNewBlock()
111 111 if not(self.readBlock()):
112 112 return 0
113 113 self.getBasicHeader()
114 114 break
115 115 if self.verbose:
116 116 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
117 117 self.processingHeaderObj.dataBlocksPerFile,
118 118 self.dataOut.datatime.ctime()))
119 119 return 1
120 120
121 121 def getFirstHeader(self):
122 122 self.getBasicHeader()
123 123 self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
124 124 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
125 125 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
126 126 self.dataOut.dtype = self.dtype
127 127
128 128 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
129 129 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
130 130 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
131 131 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
132 132 # asumo q la data no esta decodificada
133 133 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
134 134 # asumo q la data no esta sin flip
135 135 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
136 136 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
137 137 self.dataOut.frequency = self.frequency
138 138
139 139 def getBasicHeader(self):
140 140 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
141 141 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
142 142
143 143 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
144 144 self.dataOut.timeZone = self.basicHeaderObj.timeZone
145 145 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
146 146 self.dataOut.errorCount = self.basicHeaderObj.errorCount
147 147 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
148 148 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
149 149
150 150 def readFirstHeader(self):
151 151
152 152 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
153 153 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
154 154 if datatype == 0:
155 155 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
156 156 elif datatype == 1:
157 157 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
158 158 elif datatype == 2:
159 159 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
160 160 elif datatype == 3:
161 161 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
162 162 elif datatype == 4:
163 163 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
164 164 elif datatype == 5:
165 165 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
166 166 else:
167 167 raise ValueError('Data type was not defined')
168 168
169 169 self.dtype = datatype_str
170 170
171 171
172 172 def set_RCH(self, expType=2, nTx=1, ipp=None, txA=0, txB=0,
173 173 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
174 174 numTaus=0, line6Function=0, line5Function=0, fClock=None,
175 175 prePulseBefore=0, prePulseAfter=0,
176 176 codeType=0, nCode=0, nBaud=0, code=None,
177 177 flip1=0, flip2=0, Taus=0):
178 178 self.radarControllerHeaderObj.expType = expType
179 179 self.radarControllerHeaderObj.nTx = nTx
180 180 self.radarControllerHeaderObj.ipp = float(ipp)
181 181 self.radarControllerHeaderObj.txA = float(txA)
182 182 self.radarControllerHeaderObj.txB = float(txB)
183 183 self.radarControllerHeaderObj.rangeIpp = b'A\n' # ipp
184 184 self.radarControllerHeaderObj.rangeTxA = b''
185 185 self.radarControllerHeaderObj.rangeTxB = b''
186 186
187 187 self.radarControllerHeaderObj.nHeights = int(nHeights)
188 188 self.radarControllerHeaderObj.firstHeight = numpy.array([firstHeight])
189 189 self.radarControllerHeaderObj.deltaHeight = numpy.array([deltaHeight])
190 190 self.radarControllerHeaderObj.samplesWin = numpy.array([nHeights])
191 191
192 192
193 193 self.radarControllerHeaderObj.nWindows = nWindows
194 194 self.radarControllerHeaderObj.numTaus = numTaus
195 195 self.radarControllerHeaderObj.codeType = codeType
196 196 self.radarControllerHeaderObj.line6Function = line6Function
197 197 self.radarControllerHeaderObj.line5Function = line5Function
198 198 # self.radarControllerHeaderObj.fClock = fClock
199 199 self.radarControllerHeaderObj.prePulseBefore = prePulseBefore
200 200 self.radarControllerHeaderObj.prePulseAfter = prePulseAfter
201 201
202 202 self.radarControllerHeaderObj.flip1 = flip1
203 203 self.radarControllerHeaderObj.flip2 = flip2
204 204
205 205 self.radarControllerHeaderObj.code_size = 0
206 206 if self.radarControllerHeaderObj.codeType != 0:
207 207 self.radarControllerHeaderObj.nCode = nCode
208 208 self.radarControllerHeaderObj.nBaud = nBaud
209 209 self.radarControllerHeaderObj.code = code
210 210 self.radarControllerHeaderObj.code_size = int(numpy.ceil(nBaud / 32.)) * nCode * 4
211 211
212 212 if fClock is None and deltaHeight is not None:
213 213 self.fClock = 0.15 / (deltaHeight * 1e-6)
214 214 self.radarControllerHeaderObj.fClock = self.fClock
215 215 if numTaus == 0:
216 216 self.radarControllerHeaderObj.Taus = numpy.array(0, '<f4')
217 217 else:
218 218 self.radarControllerHeaderObj.Taus = numpy.array(Taus, '<f4')
219 219
220 220 def set_PH(self, dtype=0, blockSize=0, profilesPerBlock=0,
221 221 dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
222 222 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0,
223 223 deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
224 224 code=0, nBaud=None, shif_fft=False, flag_dc=False,
225 225 flag_cspc=False, flag_decode=False, flag_deflip=False):
226 226
227 227 self.processingHeaderObj.dtype = dtype
228 228 self.processingHeaderObj.profilesPerBlock = profilesPerBlock
229 229 self.processingHeaderObj.dataBlocksPerFile = dataBlocksPerFile
230 230 self.processingHeaderObj.nWindows = nWindows
231 231 self.processingHeaderObj.processFlags = processFlags
232 232 self.processingHeaderObj.nCohInt = nCohInt
233 233 self.processingHeaderObj.nIncohInt = nIncohInt
234 234 self.processingHeaderObj.totalSpectra = totalSpectra
235 235
236 236 self.processingHeaderObj.nHeights = int(nHeights)
237 237 self.processingHeaderObj.firstHeight = firstHeight # numpy.array([firstHeight])#firstHeight
238 238 self.processingHeaderObj.deltaHeight = deltaHeight # numpy.array([deltaHeight])#deltaHeight
239 239 self.processingHeaderObj.samplesWin = nHeights # numpy.array([nHeights])#nHeights
240 240
241 241 def set_BH(self, utc=0, miliSecond=0, timeZone=0):
242 242 self.basicHeaderObj.utc = utc
243 243 self.basicHeaderObj.miliSecond = miliSecond
244 244 self.basicHeaderObj.timeZone = timeZone
245 245
246 246 def set_SH(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=32):
247 247 # self.systemHeaderObj.size = size
248 248 self.systemHeaderObj.nSamples = nSamples
249 249 self.systemHeaderObj.nProfiles = nProfiles
250 250 self.systemHeaderObj.nChannels = nChannels
251 251 self.systemHeaderObj.adcResolution = adcResolution
252 252 self.systemHeaderObj.pciDioBusWidth = pciDioBusWidth
253 253
254 254 def init_acquisition(self):
255 255
256 256 if self.nFFTPoints != 0:
257 257 self.incIntFactor = m_nProfilesperBlock / self.nFFTPoints
258 258 if (self.FixPP_IncInt > self.incIntFactor):
259 259 self.incIntFactor = self.FixPP_IncInt / self.incIntFactor
260 260 elif(self.FixPP_IncInt < self.incIntFactor):
261 261 print("False alert...")
262 262
263 263 ProfilesperBlock = self.processingHeaderObj.profilesPerBlock
264 264
265 265 self.timeperblock = int(((self.FixRCP_IPP
266 266 * ProfilesperBlock
267 267 * self.FixPP_CohInt
268 268 * self.incIntFactor)
269 269 / 150.0)
270 270 * 0.9
271 271 + 0.5)
272 272 # para cada canal
273 273 self.profiles = ProfilesperBlock * self.FixPP_CohInt
274 274 self.profiles = ProfilesperBlock
275 275 self.Reference = int((self.Tau_0 - self.AcqH0_0) / (self.AcqDH_0) + 0.5)
276 276 self.BaudWidth = int((self.FixRCP_TXA / self.AcqDH_0) / self.Bauds + 0.5)
277 277
278 278 if (self.BaudWidth == 0):
279 279 self.BaudWidth = 1
280 280
281 281 def init_pulse(self, Num_Codes=Num_Codes, Bauds=Bauds, BaudWidth=BaudWidth, Dyn_snCode=Dyn_snCode):
282 282
283 283 Num_Codes = Num_Codes
284 284 Bauds = Bauds
285 285 BaudWidth = BaudWidth
286 286 Dyn_snCode = Dyn_snCode
287 287
288 288 if Dyn_snCode:
289 289 print("EXISTE")
290 290 else:
291 291 print("No existe")
292 292
293 293 if Dyn_snCode: # if Bauds:
294 294 pulses = list(range(0, Num_Codes))
295 295 num_codes = Num_Codes
296 296 for i in range(num_codes):
297 297 pulse_size = Bauds * BaudWidth
298 298 pulses[i] = numpy.zeros(pulse_size)
299 299 for j in range(Bauds):
300 300 for k in range(BaudWidth):
301 301 pulses[i][j * BaudWidth + k] = int(Dyn_snCode[i][j] * 600)
302 302 else:
303 303 print("sin code")
304 304 pulses = list(range(1))
305 305 if self.AcqDH_0 > 0.149:
306 306 pulse_size = int(self.FixRCP_TXB / 0.15 + 0.5)
307 307 else:
308 308 pulse_size = int((self.FixRCP_TXB / self.AcqDH_0) + 0.5) # 0.0375
309 309 pulses[0] = numpy.ones(pulse_size)
310 310 pulses = 600 * pulses[0]
311 311
312 312 return pulses, pulse_size
313 313
314 314 def jro_GenerateBlockOfData(self, Samples=Samples, DC_level=DC_level, stdev=stdev,
315 315 Reference=Reference, pulses=pulses,
316 316 Num_Codes=Num_Codes, pulse_size=pulse_size,
317 317 prof_gen=prof_gen, H0=H0, DH0=DH0,
318 318 Adoppler=Adoppler, Fdoppler=Fdoppler, Hdoppler=Hdoppler):
319 319 Samples = Samples
320 320 DC_level = DC_level
321 321 stdev = stdev
322 322 m_nR = Reference
323 323 pulses = pulses
324 324 num_codes = Num_Codes
325 325 ps = pulse_size
326 326 prof_gen = prof_gen
327 327 channels = self.channels
328 328 H0 = H0
329 329 DH0 = DH0
330 330 ippSec = self.radarControllerHeaderObj.ippSeconds
331 331 Fdoppler = self.Fdoppler
332 332 Hdoppler = self.Hdoppler
333 333 Adoppler = self.Adoppler
334 334
335 self.datablock = numpy.zeros([channels, prof_gen, Samples], dtype=numpy.complex64)
335 self.datablock = numpy.zeros([channels, prof_gen, Samples], dtype=complex)
336 336 for i in range(channels):
337 337 for k in range(prof_gen):
338 338 #-----------------------NOISE---------------
339 339 Noise_r = numpy.random.normal(DC_level, stdev, Samples)
340 340 Noise_i = numpy.random.normal(DC_level, stdev, Samples)
341 341 Noise = numpy.zeros(Samples, dtype=complex)
342 342 Noise.real = Noise_r
343 343 Noise.imag = Noise_i
344 344 #-----------------------PULSOS--------------
345 345 Pulso = numpy.zeros(pulse_size, dtype=complex)
346 346 Pulso.real = pulses[k % num_codes]
347 347 Pulso.imag = pulses[k % num_codes]
348 348 #--------------------- PULSES+NOISE----------
349 349 InBuffer = numpy.zeros(Samples, dtype=complex)
350 350 InBuffer[m_nR:m_nR + ps] = Pulso
351 351 InBuffer = InBuffer + Noise
352 352 #--------------------- ANGLE -------------------------------
353 353 InBuffer.real[m_nR:m_nR + ps] = InBuffer.real[m_nR:m_nR + ps] * (math.cos(self.fAngle) * 5)
354 354 InBuffer.imag[m_nR:m_nR + ps] = InBuffer.imag[m_nR:m_nR + ps] * (math.sin(self.fAngle) * 5)
355 355 InBuffer = InBuffer
356 356 self.datablock[i][k] = InBuffer
357 357
358 358 #----------------DOPPLER SIGNAL...............................................
359 359 time_vec = numpy.linspace(0, (prof_gen - 1) * ippSec, int(prof_gen)) + self.nReadBlocks * ippSec * prof_gen + (self.nReadFiles - 1) * ippSec * prof_gen
360 360 fd = Fdoppler # +(600.0/120)*self.nReadBlocks
361 d_signal = Adoppler * numpy.array(numpy.exp(1.0j * 2.0 * math.pi * fd * time_vec), dtype=numpy.complex64)
361 d_signal = Adoppler * numpy.array(numpy.exp(1.0j * 2.0 * math.pi * fd * time_vec), dtype=complex)
362 362 #-------------Senal con ancho espectral--------------------
363 363 if prof_gen % 2 == 0:
364 364 min = int(prof_gen / 2.0 - 1.0)
365 365 max = int(prof_gen / 2.0)
366 366 else:
367 367 min = int(prof_gen / 2.0)
368 368 max = int(prof_gen / 2.0)
369 369 specw_sig = numpy.linspace(-min, max, prof_gen)
370 370 w = 4
371 371 A = 20
372 372 specw_sig = specw_sig / w
373 373 specw_sig = numpy.sinc(specw_sig)
374 specw_sig = A * numpy.array(specw_sig, dtype=numpy.complex64)
374 specw_sig = A * numpy.array(specw_sig, dtype=complex)
375 375 #------------------ DATABLOCK + DOPPLER--------------------
376 376 HD = int(Hdoppler / self.AcqDH_0)
377 377 for i in range(12):
378 378 self.datablock[0, :, HD + i] = self.datablock[0, :, HD + i] + d_signal # RESULT
379 379 #------------------ DATABLOCK + DOPPLER*Sinc(x)--------------------
380 380 HD = int(Hdoppler / self.AcqDH_0)
381 381 HD = int(HD / 2)
382 382 for i in range(12):
383 383 self.datablock[0, :, HD + i] = self.datablock[0, :, HD + i] + specw_sig * d_signal # RESULT
384 384
385 385 def readBlock(self):
386 386
387 387 self.jro_GenerateBlockOfData(Samples=self.samples, DC_level=self.DC_level,
388 388 stdev=self.stdev, Reference=self.Reference,
389 389 pulses=self.pulses, Num_Codes=self.Num_Codes,
390 390 pulse_size=self.pulse_size, prof_gen=self.profiles,
391 391 H0=self.H0, DH0=self.DH0)
392 392
393 393 self.profileIndex = 0
394 394 self.flagIsNewFile = 0
395 395 self.flagIsNewBlock = 1
396 396 self.nTotalBlocks += 1
397 397 self.nReadBlocks += 1
398 398
399 399 return 1
400 400
401 401
402 402 def getData(self):
403 403 if self.flagNoMoreFiles:
404 404 self.dataOut.flagNodata = True
405 405 return 0
406 406 self.flagDiscontinuousBlock = 0
407 407 self.flagIsNewBlock = 0
408 408 if self.__hasNotDataInBuffer(): # aqui es verdad
409 409 if not(self.readNextBlock()): # return 1 y por eso el if not salta a getBasic Header
410 410 return 0
411 411 self.getFirstHeader() # atributo
412 412
413 413 if not self.getByBlock:
414 414 self.dataOut.flagDataAsBlock = False
415 415 self.dataOut.data = self.datablock[:, self.profileIndex, :]
416 416 self.dataOut.profileIndex = self.profileIndex
417 417 self.profileIndex += 1
418 418 else:
419 419 pass
420 420 self.dataOut.flagNoData = False
421 421 self.getBasicHeader()
422 422 self.dataOut.realtime = self.online
423 423 return self.dataOut.data
424 424
425 425
426 426 def setup(self, frequency=49.92e6, incIntFactor=1, nFFTPoints=0, FixPP_IncInt=1, FixRCP_IPP=1000,
427 427 FixPP_CohInt=1, Tau_0=250, AcqH0_0=70 , AcqDH_0=1.25, Bauds=32,
428 428 FixRCP_TXA=40, FixRCP_TXB=50, fAngle=2.0 * math.pi * (1 / 16), DC_level=50,
429 429 stdev=8, Num_Codes=1 , Dyn_snCode=None, samples=200,
430 430 channels=2, Fdoppler=20, Hdoppler=36, Adoppler=500,
431 431 profilesPerBlock=300, dataBlocksPerFile=120, nTotalReadFiles=10000,
432 432 **kwargs):
433 433
434 434 self.set_kwargs(**kwargs)
435 435 self.nReadBlocks = 0
436 436 self.nReadFiles = 1
437 437 print('------------------- [Opening file: ] ------------------------------', self.nReadFiles)
438 438
439 439 tmp = time.time()
440 440 tmp_utc = int(tmp)
441 441 tmp_milisecond = int((tmp - tmp_utc) * 1000)
442 442 print(" SETUP -basicHeaderObj.utc", datetime.datetime.utcfromtimestamp(tmp))
443 443 if Dyn_snCode is None:
444 444 Num_Codes = 1
445 445 Bauds = 1
446 446
447 447
448 448
449 449 self.set_BH(utc=tmp_utc, miliSecond=tmp_milisecond, timeZone=300)
450 450 self.set_RCH(expType=0, nTx=150, ipp=FixRCP_IPP, txA=FixRCP_TXA, txB=FixRCP_TXB,
451 451 nWindows=1 , nHeights=samples, firstHeight=AcqH0_0, deltaHeight=AcqDH_0,
452 452 numTaus=1, line6Function=0, line5Function=0, fClock=None,
453 453 prePulseBefore=0, prePulseAfter=0,
454 454 codeType=0, nCode=Num_Codes, nBaud=32, code=Dyn_snCode,
455 455 flip1=0, flip2=0, Taus=Tau_0)
456 456
457 457 self.set_PH(dtype=0, blockSize=0, profilesPerBlock=profilesPerBlock,
458 458 dataBlocksPerFile=dataBlocksPerFile, nWindows=1, processFlags=numpy.array([1024]), nCohInt=1,
459 459 nIncohInt=1, totalSpectra=0, nHeights=samples, firstHeight=AcqH0_0,
460 460 deltaHeight=AcqDH_0, samplesWin=samples, spectraComb=0, nCode=0,
461 461 code=0, nBaud=None, shif_fft=False, flag_dc=False,
462 462 flag_cspc=False, flag_decode=False, flag_deflip=False)
463 463
464 464 self.set_SH(nSamples=samples, nProfiles=profilesPerBlock, nChannels=channels)
465 465
466 466 self.readFirstHeader()
467 467
468 468 self.frequency = frequency
469 469 self.incIntFactor = incIntFactor
470 470 self.nFFTPoints = nFFTPoints
471 471 self.FixPP_IncInt = FixPP_IncInt
472 472 self.FixRCP_IPP = FixRCP_IPP
473 473 self.FixPP_CohInt = FixPP_CohInt
474 474 self.Tau_0 = Tau_0
475 475 self.AcqH0_0 = AcqH0_0
476 476 self.H0 = AcqH0_0
477 477 self.AcqDH_0 = AcqDH_0
478 478 self.DH0 = AcqDH_0
479 479 self.Bauds = Bauds
480 480 self.FixRCP_TXA = FixRCP_TXA
481 481 self.FixRCP_TXB = FixRCP_TXB
482 482 self.fAngle = fAngle
483 483 self.DC_level = DC_level
484 484 self.stdev = stdev
485 485 self.Num_Codes = Num_Codes
486 486 self.Dyn_snCode = Dyn_snCode
487 487 self.samples = samples
488 488 self.channels = channels
489 489 self.profiles = None
490 490 self.m_nReference = None
491 491 self.Baudwidth = None
492 492 self.Fdoppler = Fdoppler
493 493 self.Hdoppler = Hdoppler
494 494 self.Adoppler = Adoppler
495 495 self.nTotalReadFiles = int(nTotalReadFiles)
496 496
497 497 print("IPP ", self.FixRCP_IPP)
498 498 print("Tau_0 ", self.Tau_0)
499 499 print("AcqH0_0", self.AcqH0_0)
500 500 print("samples,window ", self.samples)
501 501 print("AcqDH_0", AcqDH_0)
502 502 print("FixRCP_TXA", self.FixRCP_TXA)
503 503 print("FixRCP_TXB", self.FixRCP_TXB)
504 504 print("Dyn_snCode", Dyn_snCode)
505 505 print("Fdoppler", Fdoppler)
506 506 print("Hdoppler", Hdoppler)
507 507 print("Vdopplermax", Fdoppler * (3.0e8 / self.frequency) / 2.0)
508 508 print("nTotalReadFiles", nTotalReadFiles)
509 509
510 510 self.init_acquisition()
511 511 self.pulses, self.pulse_size = self.init_pulse(Num_Codes=self.Num_Codes, Bauds=self.Bauds, BaudWidth=self.BaudWidth, Dyn_snCode=Dyn_snCode)
512 512 print(" [ END ] - SETUP metodo")
513 513 return
514 514
515 515 def run(self, **kwargs): # metodo propio
516 516 if not(self.isConfig):
517 517 self.setup(**kwargs)
518 518 self.isConfig = True
519 519 self.getData()
Show file before | Show file after | Hide comments
@@ -1,602 +1,602
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 try:
11 11 from gevent import sleep
12 12 except:
13 13 from time import sleep
14 14
15 15 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
16 16 from schainpy.model.data.jrodata import Voltage
17 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
18 18
19 19 try:
20 20 import digital_rf_hdf5
21 21 except:
22 22 pass
23 23
24 24 class USRPReader(ProcessingUnit):
25 25 '''
26 26 classdocs
27 27 '''
28 28
29 29 def __init__(self, **kwargs):
30 30 '''
31 31 Constructor
32 32 '''
33 33
34 34 ProcessingUnit.__init__(self, **kwargs)
35 35
36 36 self.dataOut = Voltage()
37 37 self.__printInfo = True
38 38 self.__flagDiscontinuousBlock = False
39 39 self.__bufferIndex = 9999999
40 40
41 41 self.__ippKm = None
42 42 self.__codeType = 0
43 43 self.__nCode = None
44 44 self.__nBaud = None
45 45 self.__code = None
46 46
47 47 def __getCurrentSecond(self):
48 48
49 49 return self.__thisUnixSample / self.__sample_rate
50 50
51 51 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
52 52
53 53 def __setFileHeader(self):
54 54 '''
55 55 In this method will be initialized every parameter of dataOut object (header, no data)
56 56 '''
57 57 ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
58 58
59 59 nProfiles = 1.0 / ippSeconds # Number of profiles in one second
60 60
61 61 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
62 62 txA=0,
63 63 txB=0,
64 64 nWindows=1,
65 65 nHeights=self.__nSamples,
66 66 firstHeight=self.__firstHeigth,
67 67 deltaHeight=self.__deltaHeigth,
68 68 codeType=self.__codeType,
69 69 nCode=self.__nCode, nBaud=self.__nBaud,
70 70 code=self.__code)
71 71
72 72 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
73 73 nProfiles=nProfiles,
74 74 nChannels=len(self.__channelList),
75 75 adcResolution=14)
76 76
77 77 self.dataOut.type = "Voltage"
78 78
79 79 self.dataOut.data = None
80 80
81 81 self.dataOut.dtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
82 82
83 83 # self.dataOut.nChannels = 0
84 84
85 85 # self.dataOut.nHeights = 0
86 86
87 87 self.dataOut.nProfiles = nProfiles
88 88
89 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float) * self.__deltaHeigth
89 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float32) * self.__deltaHeigth
90 90
91 91 self.dataOut.channelList = self.__channelList
92 92
93 93 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
94 94
95 95 # self.dataOut.channelIndexList = None
96 96
97 97 self.dataOut.flagNoData = True
98 98
99 99 # Set to TRUE if the data is discontinuous
100 100 self.dataOut.flagDiscontinuousBlock = False
101 101
102 102 self.dataOut.utctime = None
103 103
104 104 self.dataOut.timeZone = self.__timezone / 60 # timezone like jroheader, difference in minutes between UTC and localtime
105 105
106 106 self.dataOut.dstFlag = 0
107 107
108 108 self.dataOut.errorCount = 0
109 109
110 110 self.dataOut.nCohInt = 1
111 111
112 112 self.dataOut.flagDecodeData = False # asumo que la data esta decodificada
113 113
114 114 self.dataOut.flagDeflipData = False # asumo que la data esta sin flip
115 115
116 116 self.dataOut.flagShiftFFT = False
117 117
118 118 self.dataOut.ippSeconds = ippSeconds
119 119
120 120 # Time interval between profiles
121 121 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
122 122
123 123 self.dataOut.frequency = self.__frequency
124 124
125 125 self.dataOut.realtime = self.__online
126 126
127 127 def findDatafiles(self, path, startDate=None, endDate=None):
128 128
129 129 if not os.path.isdir(path):
130 130 return []
131 131
132 132 try:
133 133 digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
134 134 except:
135 135 digitalReadObj = digital_rf_hdf5.read_hdf5(path)
136 136
137 137 channelNameList = digitalReadObj.get_channels()
138 138
139 139 if not channelNameList:
140 140 return []
141 141
142 142 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
143 143
144 144 sample_rate = metadata_dict['sample_rate'][0]
145 145
146 146 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
147 147
148 148 try:
149 149 timezone = this_metadata_file['timezone'].value
150 150 except:
151 151 timezone = 0
152 152
153 153 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0]) / sample_rate - timezone
154 154
155 155 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
156 156 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
157 157
158 158 if not startDate:
159 159 startDate = startDatetime.date()
160 160
161 161 if not endDate:
162 162 endDate = endDatatime.date()
163 163
164 164 dateList = []
165 165
166 166 thisDatetime = startDatetime
167 167
168 168 while(thisDatetime <= endDatatime):
169 169
170 170 thisDate = thisDatetime.date()
171 171
172 172 if thisDate < startDate:
173 173 continue
174 174
175 175 if thisDate > endDate:
176 176 break
177 177
178 178 dateList.append(thisDate)
179 179 thisDatetime += datetime.timedelta(1)
180 180
181 181 return dateList
182 182
183 183 def setup(self, path=None,
184 184 startDate=None,
185 185 endDate=None,
186 186 startTime=datetime.time(0, 0, 0),
187 187 endTime=datetime.time(23, 59, 59),
188 188 channelList=None,
189 189 nSamples=None,
190 190 ippKm=60,
191 191 online=False,
192 192 delay=60,
193 193 buffer_size=1024,
194 194 **kwargs):
195 195 '''
196 196 In this method we should set all initial parameters.
197 197
198 198 Inputs:
199 199 path
200 200 startDate
201 201 endDate
202 202 startTime
203 203 endTime
204 204 set
205 205 expLabel
206 206 ext
207 207 online
208 208 delay
209 209 '''
210 210
211 211 if not os.path.isdir(path):
212 212 raise ValueError("[Reading] Directory %s does not exist" % path)
213 213
214 214 try:
215 215 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
216 216 except:
217 217 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
218 218
219 219 channelNameList = self.digitalReadObj.get_channels()
220 220
221 221 if not channelNameList:
222 222 raise ValueError("[Reading] Directory %s does not have any files" % path)
223 223
224 224 if not channelList:
225 225 channelList = list(range(len(channelNameList)))
226 226
227 227 ########## Reading metadata ######################
228 228
229 229 metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
230 230
231 231 self.__sample_rate = metadata_dict['sample_rate'][0]
232 232 # self.__samples_per_file = metadata_dict['samples_per_file'][0]
233 233 self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate
234 234
235 235 this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
236 236
237 237 self.__frequency = None
238 238 try:
239 239 self.__frequency = this_metadata_file['center_frequencies'].value
240 240 except:
241 241 self.__frequency = this_metadata_file['fc'].value
242 242
243 243 if not self.__frequency:
244 244 raise ValueError("Center Frequency is not defined in metadata file")
245 245
246 246 try:
247 247 self.__timezone = this_metadata_file['timezone'].value
248 248 except:
249 249 self.__timezone = 0
250 250
251 251 self.__firstHeigth = 0
252 252
253 253 try:
254 254 codeType = this_metadata_file['codeType'].value
255 255 except:
256 256 codeType = 0
257 257
258 258 nCode = 1
259 259 nBaud = 1
260 code = numpy.ones((nCode, nBaud), dtype=numpy.int)
260 code = numpy.ones((nCode, nBaud), dtype=numpy.int32)
261 261
262 262 if codeType:
263 263 nCode = this_metadata_file['nCode'].value
264 264 nBaud = this_metadata_file['nBaud'].value
265 265 code = this_metadata_file['code'].value
266 266
267 267 if not ippKm:
268 268 try:
269 269 # seconds to km
270 270 ippKm = 1e6 * 0.15 * this_metadata_file['ipp'].value
271 271 except:
272 272 ippKm = None
273 273
274 274 ####################################################
275 275 startUTCSecond = None
276 276 endUTCSecond = None
277 277
278 278 if startDate:
279 279 startDatetime = datetime.datetime.combine(startDate, startTime)
280 280 startUTCSecond = (startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
281 281
282 282 if endDate:
283 283 endDatetime = datetime.datetime.combine(endDate, endTime)
284 284 endUTCSecond = (endDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
285 285
286 286 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
287 287
288 288 if not startUTCSecond:
289 289 startUTCSecond = start_index / self.__sample_rate
290 290
291 291 if start_index > startUTCSecond * self.__sample_rate:
292 292 startUTCSecond = start_index / self.__sample_rate
293 293
294 294 if not endUTCSecond:
295 295 endUTCSecond = end_index / self.__sample_rate
296 296
297 297 if end_index < endUTCSecond * self.__sample_rate:
298 298 endUTCSecond = end_index / self.__sample_rate
299 299
300 300 if not nSamples:
301 301 if not ippKm:
302 302 raise ValueError("[Reading] nSamples or ippKm should be defined")
303 303
304 304 nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
305 305
306 306 channelBoundList = []
307 307 channelNameListFiltered = []
308 308
309 309 for thisIndexChannel in channelList:
310 310 thisChannelName = channelNameList[thisIndexChannel]
311 311 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
312 312 channelBoundList.append((start_index, end_index))
313 313 channelNameListFiltered.append(thisChannelName)
314 314
315 315 self.profileIndex = 0
316 316
317 317 self.__delay = delay
318 318 self.__ippKm = ippKm
319 319 self.__codeType = codeType
320 320 self.__nCode = nCode
321 321 self.__nBaud = nBaud
322 322 self.__code = code
323 323
324 324 self.__datapath = path
325 325 self.__online = online
326 326 self.__channelList = channelList
327 327 self.__channelNameList = channelNameListFiltered
328 328 self.__channelBoundList = channelBoundList
329 329 self.__nSamples = nSamples
330 330 self.__samples_to_read = int(buffer_size * nSamples)
331 331 self.__nChannels = len(self.__channelList)
332 332
333 333 self.__startUTCSecond = startUTCSecond
334 334 self.__endUTCSecond = endUTCSecond
335 335
336 336 self.__timeInterval = 1.0 * self.__samples_to_read / self.__sample_rate # Time interval
337 337
338 338 if online:
339 339 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
340 340 startUTCSecond = numpy.floor(endUTCSecond)
341 341
342 342 self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
343 343
344 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype=numpy.complex)
344 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype=complex)
345 345
346 346 self.__setFileHeader()
347 347 self.isConfig = True
348 348
349 349 print("[Reading] USRP Data was found from %s to %s " % (
350 350 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
351 351 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
352 352 ))
353 353
354 354 print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
355 355 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
356 356 ))
357 357
358 358 def __reload(self):
359 359
360 360 if not self.__online:
361 361 return
362 362
363 363 # print
364 364 # print "%s not in range [%s, %s]" %(
365 365 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
366 366 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
367 367 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
368 368 # )
369 369 print("[Reading] reloading metadata ...")
370 370
371 371 try:
372 372 self.digitalReadObj.reload(complete_update=True)
373 373 except:
374 374 self.digitalReadObj.reload()
375 375
376 376 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
377 377
378 378 if start_index > self.__startUTCSecond * self.__sample_rate:
379 379 self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
380 380
381 381 if end_index > self.__endUTCSecond * self.__sample_rate:
382 382 self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
383 383 print()
384 384 print("[Reading] New timerange found [%s, %s] " % (
385 385 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
386 386 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
387 387 ))
388 388
389 389 return True
390 390
391 391 return False
392 392
393 393 def __readNextBlock(self, seconds=30, volt_scale=218776):
394 394 '''
395 395 '''
396 396
397 397 # Set the next data
398 398 self.__flagDiscontinuousBlock = False
399 399 self.__thisUnixSample += self.__samples_to_read
400 400
401 401 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
402 402 print("[Reading] There are no more data into selected time-range")
403 403
404 404 self.__reload()
405 405
406 406 if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
407 407 self.__thisUnixSample -= self.__samples_to_read
408 408 return False
409 409
410 410 indexChannel = 0
411 411
412 412 dataOk = False
413 413
414 414 for thisChannelName in self.__channelNameList:
415 415
416 416 try:
417 417 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
418 418 self.__samples_to_read,
419 419 thisChannelName)
420 420
421 421 except IOError as e:
422 422 # read next profile
423 423 self.__flagDiscontinuousBlock = True
424 424 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
425 425 break
426 426
427 427 if result.shape[0] != self.__samples_to_read:
428 428 self.__flagDiscontinuousBlock = True
429 429 print("[Reading] %s: Too few samples were found, just %d/%d samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
430 430 result.shape[0],
431 431 self.__samples_to_read))
432 432 break
433 433
434 434 self.__data_buffer[indexChannel, :] = result * volt_scale
435 435
436 436 indexChannel += 1
437 437
438 438 dataOk = True
439 439
440 440 self.__utctime = self.__thisUnixSample / self.__sample_rate
441 441
442 442 if not dataOk:
443 443 return False
444 444
445 445 print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
446 446 self.__samples_to_read,
447 447 self.__timeInterval))
448 448
449 449 self.__bufferIndex = 0
450 450
451 451 return True
452 452
453 453 def __isBufferEmpty(self):
454 454
455 455 if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
456 456 return False
457 457
458 458 return True
459 459
460 460 def getData(self, seconds=30, nTries=5):
461 461
462 462 '''
463 463 This method gets the data from files and put the data into the dataOut object
464 464
465 465 In addition, increase el the buffer counter in one.
466 466
467 467 Return:
468 468 data : retorna un perfil de voltages (alturas * canales) copiados desde el
469 469 buffer. Si no hay mas archivos a leer retorna None.
470 470
471 471 Affected:
472 472 self.dataOut
473 473 self.profileIndex
474 474 self.flagDiscontinuousBlock
475 475 self.flagIsNewBlock
476 476 '''
477 477
478 478 err_counter = 0
479 479 self.dataOut.flagNoData = True
480 480
481 481 if self.__isBufferEmpty():
482 482
483 483 self.__flagDiscontinuousBlock = False
484 484
485 485 while True:
486 486 if self.__readNextBlock():
487 487 break
488 488
489 489 if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
490 490 return False
491 491
492 492 if self.__flagDiscontinuousBlock:
493 493 print('[Reading] discontinuous block found ... continue with the next block')
494 494 continue
495 495
496 496 if not self.__online:
497 497 return False
498 498
499 499 err_counter += 1
500 500 if err_counter > nTries:
501 501 return False
502 502
503 503 print('[Reading] waiting %d seconds to read a new block' % seconds)
504 504 sleep(seconds)
505 505
506 506 self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
507 507 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
508 508 self.dataOut.flagNoData = False
509 509 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
510 510 self.dataOut.profileIndex = self.profileIndex
511 511
512 512 self.__bufferIndex += self.__nSamples
513 513 self.profileIndex += 1
514 514
515 515 if self.profileIndex == self.dataOut.nProfiles:
516 516 self.profileIndex = 0
517 517
518 518 return True
519 519
520 520 def printInfo(self):
521 521 '''
522 522 '''
523 523 if self.__printInfo == False:
524 524 return
525 525
526 526 # self.systemHeaderObj.printInfo()
527 527 # self.radarControllerHeaderObj.printInfo()
528 528
529 529 self.__printInfo = False
530 530
531 531 def printNumberOfBlock(self):
532 532 '''
533 533 '''
534 534
535 535 print(self.profileIndex)
536 536
537 537 def run(self, **kwargs):
538 538 '''
539 539 This method will be called many times so here you should put all your code
540 540 '''
541 541
542 542 if not self.isConfig:
543 543 self.setup(**kwargs)
544 544
545 545 self.getData(seconds=self.__delay)
546 546
547 547 return
548 548
549 549
550 550 @MPDecorator
551 551 class USRPWriter(Operation):
552 552 '''
553 553 classdocs
554 554 '''
555 555
556 556 def __init__(self, **kwargs):
557 557 '''
558 558 Constructor
559 559 '''
560 560 Operation.__init__(self, **kwargs)
561 561 self.dataOut = None
562 562
563 563 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
564 564 '''
565 565 In this method we should set all initial parameters.
566 566
567 567 Input:
568 568 dataIn : Input data will also be outputa data
569 569
570 570 '''
571 571 self.dataOut = dataIn
572 572
573 573
574 574
575 575
576 576
577 577 self.isConfig = True
578 578
579 579 return
580 580
581 581 def run(self, dataIn, **kwargs):
582 582 '''
583 583 This method will be called many times so here you should put all your code
584 584
585 585 Inputs:
586 586
587 587 dataIn : object with the data
588 588
589 589 '''
590 590
591 591 if not self.isConfig:
592 592 self.setup(dataIn, **kwargs)
593 593
594 594
595 595 if __name__ == '__main__':
596 596
597 597 readObj = USRPReader()
598 598
599 599 while True:
600 600 readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
601 601 # readObj.printInfo()
602 602 readObj.printNumberOfBlock()
General Comments 0
You need to be logged in to leave comments. Login now