##// END OF EJS Templates
schain
RSS
agregado metodos a header class
Jose Chavez -
r980:ea5ace19bd59
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,851 +1,861
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 5 '''
6 6 import sys
7 7 import numpy
8 8 import copy
9 9 import datetime
10 import inspect
10 11
11 12 SPEED_OF_LIGHT = 299792458
12 13 SPEED_OF_LIGHT = 3e8
13 14
14 15 BASIC_STRUCTURE = numpy.dtype([
15 16 ('nSize','<u4'),
16 17 ('nVersion','<u2'),
17 18 ('nDataBlockId','<u4'),
18 19 ('nUtime','<u4'),
19 20 ('nMilsec','<u2'),
20 21 ('nTimezone','<i2'),
21 22 ('nDstflag','<i2'),
22 23 ('nErrorCount','<u4')
23 24 ])
24 25
25 26 SYSTEM_STRUCTURE = numpy.dtype([
26 27 ('nSize','<u4'),
27 28 ('nNumSamples','<u4'),
28 29 ('nNumProfiles','<u4'),
29 30 ('nNumChannels','<u4'),
30 31 ('nADCResolution','<u4'),
31 32 ('nPCDIOBusWidth','<u4'),
32 33 ])
33 34
34 35 RADAR_STRUCTURE = numpy.dtype([
35 36 ('nSize','<u4'),
36 37 ('nExpType','<u4'),
37 38 ('nNTx','<u4'),
38 39 ('fIpp','<f4'),
39 40 ('fTxA','<f4'),
40 41 ('fTxB','<f4'),
41 42 ('nNumWindows','<u4'),
42 43 ('nNumTaus','<u4'),
43 44 ('nCodeType','<u4'),
44 45 ('nLine6Function','<u4'),
45 46 ('nLine5Function','<u4'),
46 47 ('fClock','<f4'),
47 48 ('nPrePulseBefore','<u4'),
48 49 ('nPrePulseAfter','<u4'),
49 50 ('sRangeIPP','<a20'),
50 51 ('sRangeTxA','<a20'),
51 52 ('sRangeTxB','<a20'),
52 53 ])
53 54
54 55 SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
55 56
56 57
57 58 PROCESSING_STRUCTURE = numpy.dtype([
58 59 ('nSize','<u4'),
59 60 ('nDataType','<u4'),
60 61 ('nSizeOfDataBlock','<u4'),
61 62 ('nProfilesperBlock','<u4'),
62 63 ('nDataBlocksperFile','<u4'),
63 64 ('nNumWindows','<u4'),
64 65 ('nProcessFlags','<u4'),
65 66 ('nCoherentIntegrations','<u4'),
66 67 ('nIncoherentIntegrations','<u4'),
67 68 ('nTotalSpectra','<u4')
68 69 ])
69 70
70 71 class Header(object):
71 72
72 73 def __init__(self):
73 74 raise NotImplementedError
74 75
75 76 def copy(self):
76 77 return copy.deepcopy(self)
77 78
78 79 def read(self):
79 80
80 81 raise NotImplementedError
81 82
82 83 def write(self):
83 84
84 85 raise NotImplementedError
86
87 def getAllowedArgs(self):
88 return inspect.getargspec(self.__init__).args
85 89
86 90 def printInfo(self):
87 91
88 92 message = "#"*50 + "\n"
89 93 message += self.__class__.__name__.upper() + "\n"
90 94 message += "#"*50 + "\n"
91 95
92 96 keyList = self.__dict__.keys()
93 97 keyList.sort()
94 98
95 99 for key in keyList:
96 100 message += "%s = %s" %(key, self.__dict__[key]) + "\n"
97 101
98 102 if "size" not in keyList:
99 103 attr = getattr(self, "size")
100 104
101 105 if attr:
102 106 message += "%s = %s" %("size", attr) + "\n"
103 107
104 108 print message
105 109
106 110 class BasicHeader(Header):
107 111
108 112 size = None
109 113 version = None
110 114 dataBlock = None
111 115 utc = None
112 116 ltc = None
113 117 miliSecond = None
114 118 timeZone = None
115 119 dstFlag = None
116 120 errorCount = None
117 121 datatime = None
122 structure = BASIC_STRUCTURE
118 123 __LOCALTIME = None
119 124
120 125 def __init__(self, useLocalTime=True):
121 126
122 127 self.size = 24
123 128 self.version = 0
124 129 self.dataBlock = 0
125 130 self.utc = 0
126 131 self.miliSecond = 0
127 132 self.timeZone = 0
128 133 self.dstFlag = 0
129 134 self.errorCount = 0
130 135
131 136 self.useLocalTime = useLocalTime
132 137
133 138 def read(self, fp):
134 139
135 140 self.length = 0
136 141 try:
137 142 if hasattr(fp, 'read'):
138 143 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
139 144 else:
140 145 header = numpy.fromstring(fp, BASIC_STRUCTURE,1)
141 146 except Exception, e:
142 147 print "BasicHeader: "
143 148 print e
144 149 return 0
145 150
146 151 self.size = int(header['nSize'][0])
147 152 self.version = int(header['nVersion'][0])
148 153 self.dataBlock = int(header['nDataBlockId'][0])
149 154 self.utc = int(header['nUtime'][0])
150 155 self.miliSecond = int(header['nMilsec'][0])
151 156 self.timeZone = int(header['nTimezone'][0])
152 157 self.dstFlag = int(header['nDstflag'][0])
153 158 self.errorCount = int(header['nErrorCount'][0])
154 159
155 160 if self.size < 24:
156 161 return 0
157 162
158 163 self.length = header.nbytes
159 164 return 1
160 165
161 166 def write(self, fp):
162 167
163 168 headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
164 169 header = numpy.array(headerTuple, BASIC_STRUCTURE)
165 170 header.tofile(fp)
166 171
167 172 return 1
168 173
169 174 def get_ltc(self):
170 175
171 176 return self.utc - self.timeZone*60
172 177
173 178 def set_ltc(self, value):
174 179
175 180 self.utc = value + self.timeZone*60
176 181
177 182 def get_datatime(self):
178 183
179 184 return datetime.datetime.utcfromtimestamp(self.ltc)
180 185
181 186 ltc = property(get_ltc, set_ltc)
182 187 datatime = property(get_datatime)
183 188
184 189 class SystemHeader(Header):
185 190
186 191 size = None
187 192 nSamples = None
188 193 nProfiles = None
189 194 nChannels = None
190 195 adcResolution = None
191 196 pciDioBusWidth = None
192
197 structure = SYSTEM_STRUCTURE
198
193 199 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWith=0):
194 200
195 201 self.size = 24
196 202 self.nSamples = nSamples
197 203 self.nProfiles = nProfiles
198 204 self.nChannels = nChannels
199 205 self.adcResolution = adcResolution
200 206 self.pciDioBusWidth = pciDioBusWith
201
207
202 208 def read(self, fp):
203 209 self.length = 0
204 210 try:
205 211 startFp = fp.tell()
206 212 except Exception, e:
207 213 startFp = None
208 214 pass
209 215
210 216 try:
211 217 if hasattr(fp, 'read'):
212 218 header = numpy.fromfile(fp, SYSTEM_STRUCTURE,1)
213 219 else:
214 220 header = numpy.fromstring(fp, SYSTEM_STRUCTURE,1)
215 221 except Exception, e:
216 222 print "System Header: " + str(e)
217 223 return 0
218 224
219 225 self.size = header['nSize'][0]
220 226 self.nSamples = header['nNumSamples'][0]
221 227 self.nProfiles = header['nNumProfiles'][0]
222 228 self.nChannels = header['nNumChannels'][0]
223 229 self.adcResolution = header['nADCResolution'][0]
224 230 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
225 231
226 232
227 233 if startFp is not None:
228 234 endFp = self.size + startFp
229 235
230 236 if fp.tell() > endFp:
231 237 sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name)
232 238 return 0
233 239
234 240 if fp.tell() < endFp:
235 241 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
236 242 return 0
237 243
238 244 self.length = header.nbytes
239 245 return 1
240 246
241 247 def write(self, fp):
242 248
243 249 headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
244 250 header = numpy.array(headerTuple,SYSTEM_STRUCTURE)
245 251 header.tofile(fp)
246 252
247 253 return 1
248 254
249 255 class RadarControllerHeader(Header):
250 256
251 257 expType = None
252 258 nTx = None
253 259 ipp = None
254 260 txA = None
255 261 txB = None
256 262 nWindows = None
257 263 numTaus = None
258 264 codeType = None
259 265 line6Function = None
260 266 line5Function = None
261 267 fClock = None
262 268 prePulseBefore = None
263 269 prePulserAfter = None
264 270 rangeIpp = None
265 271 rangeTxA = None
266 272 rangeTxB = None
267
273 structure = RADAR_STRUCTURE
268 274 __size = None
269 275
270 276 def __init__(self, expType=2, nTx=1,
271 277 ippKm=None, txA=0, txB=0,
272 278 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
273 279 numTaus=0, line6Function=0, line5Function=0, fClock=None,
274 280 prePulseBefore=0, prePulseAfter=0,
275 281 codeType=0, nCode=0, nBaud=0, code=None,
276 282 flip1=0, flip2=0):
277 283
278 284 # self.size = 116
279 285 self.expType = expType
280 286 self.nTx = nTx
281 287 self.ipp = ippKm
282 288 self.txA = txA
283 289 self.txB = txB
284 290 self.rangeIpp = ippKm
285 291 self.rangeTxA = txA
286 292 self.rangeTxB = txB
287 293
288 294 self.nWindows = nWindows
289 295 self.numTaus = numTaus
290 296 self.codeType = codeType
291 297 self.line6Function = line6Function
292 298 self.line5Function = line5Function
293 299 self.fClock = fClock
294 300 self.prePulseBefore = prePulseBefore
295 301 self.prePulserAfter = prePulseAfter
296 302
297 303 self.nHeights = nHeights
298 304 self.firstHeight = firstHeight
299 305 self.deltaHeight = deltaHeight
300 306 self.samplesWin = nHeights
301 307
302 308 self.nCode = nCode
303 309 self.nBaud = nBaud
304 310 self.code = code
305 311 self.flip1 = flip1
306 312 self.flip2 = flip2
307 313
308 314 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
309 315 # self.dynamic = numpy.array([],numpy.dtype('byte'))
310 316
311 317 if self.fClock is None and self.deltaHeight is not None:
312 318 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
313 319
314 320 def read(self, fp):
315 321 self.length = 0
316 322 try:
317 323 startFp = fp.tell()
318 324 except Exception, e:
319 325 startFp = None
320 326 pass
321 327
322 328 try:
323 329 if hasattr(fp, 'read'):
324 330 header = numpy.fromfile(fp, RADAR_STRUCTURE,1)
325 331 else:
326 332 header = numpy.fromstring(fp, RADAR_STRUCTURE,1)
327 333 self.length += header.nbytes
328 334 except Exception, e:
329 335 print "RadarControllerHeader: " + str(e)
330 336 return 0
331 337
332 338 size = int(header['nSize'][0])
333 339 self.expType = int(header['nExpType'][0])
334 340 self.nTx = int(header['nNTx'][0])
335 341 self.ipp = float(header['fIpp'][0])
336 342 self.txA = float(header['fTxA'][0])
337 343 self.txB = float(header['fTxB'][0])
338 344 self.nWindows = int(header['nNumWindows'][0])
339 345 self.numTaus = int(header['nNumTaus'][0])
340 346 self.codeType = int(header['nCodeType'][0])
341 347 self.line6Function = int(header['nLine6Function'][0])
342 348 self.line5Function = int(header['nLine5Function'][0])
343 349 self.fClock = float(header['fClock'][0])
344 350 self.prePulseBefore = int(header['nPrePulseBefore'][0])
345 351 self.prePulserAfter = int(header['nPrePulseAfter'][0])
346 352 self.rangeIpp = header['sRangeIPP'][0]
347 353 self.rangeTxA = header['sRangeTxA'][0]
348 354 self.rangeTxB = header['sRangeTxB'][0]
349 355
350 356 try:
351 357 if hasattr(fp, 'read'):
352 358 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
353 359 else:
354 360 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
355 361 self.length += samplingWindow.nbytes
356 362 except Exception, e:
357 363 print "RadarControllerHeader: " + str(e)
358 364 return 0
359 365 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
360 366 self.firstHeight = samplingWindow['h0']
361 367 self.deltaHeight = samplingWindow['dh']
362 368 self.samplesWin = samplingWindow['nsa']
363 369
364 370
365 371
366 372 try:
367 373 if hasattr(fp, 'read'):
368 374 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
369 375 else:
370 376 self.Taus = numpy.fromstring(fp[self.length:], '<f4', self.numTaus)
371 377 self.length += self.Taus.nbytes
372 378 except Exception, e:
373 379 print "RadarControllerHeader: " + str(e)
374 380 return 0
375 381
376 382
377 383
378 384 self.code_size = 0
379 385 if self.codeType != 0:
380 386
381 387 try:
382 388 if hasattr(fp, 'read'):
383 389 self.nCode = numpy.fromfile(fp, '<u4', 1)
384 390 self.length += self.nCode.nbytes
385 391 self.nBaud = numpy.fromfile(fp, '<u4', 1)
386 392 self.length += self.nBaud.nbytes
387 393 else:
388 394 self.nCode = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
389 395 self.length += self.nCode.nbytes
390 396 self.nBaud = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
391 397 self.length += self.nBaud.nbytes
392 398 except Exception, e:
393 399 print "RadarControllerHeader: " + str(e)
394 400 return 0
395 401 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
396 402
397 403 for ic in range(self.nCode):
398 404 try:
399 405 if hasattr(fp, 'read'):
400 406 temp = numpy.fromfile(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
401 407 else:
402 408 temp = numpy.fromstring(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
403 409 self.length += temp.nbytes
404 410 except Exception, e:
405 411 print "RadarControllerHeader: " + str(e)
406 412 return 0
407 413
408 414 for ib in range(self.nBaud-1,-1,-1):
409 415 code[ic,ib] = temp[ib/32]%2
410 416 temp[ib/32] = temp[ib/32]/2
411 417
412 418 self.code = 2.0*code - 1.0
413 419 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
414 420
415 421 # if self.line5Function == RCfunction.FLIP:
416 422 # self.flip1 = numpy.fromfile(fp,'<u4',1)
417 423 #
418 424 # if self.line6Function == RCfunction.FLIP:
419 425 # self.flip2 = numpy.fromfile(fp,'<u4',1)
420 426 if startFp is not None:
421 427 endFp = size + startFp
422 428
423 429 if fp.tell() != endFp:
424 430 # fp.seek(endFp)
425 431 print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" %(fp.name, fp.tell()-startFp, size)
426 432 # return 0
427 433
428 434 if fp.tell() > endFp:
429 435 sys.stderr.write("Warning %s: Size value read from Radar Controller header is lower than it has to be\n" %fp.name)
430 436 # return 0
431 437
432 438 if fp.tell() < endFp:
433 439 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
434 440
435 441
436 442 return 1
437 443
438 444 def write(self, fp):
439 445
440 446 headerTuple = (self.size,
441 447 self.expType,
442 448 self.nTx,
443 449 self.ipp,
444 450 self.txA,
445 451 self.txB,
446 452 self.nWindows,
447 453 self.numTaus,
448 454 self.codeType,
449 455 self.line6Function,
450 456 self.line5Function,
451 457 self.fClock,
452 458 self.prePulseBefore,
453 459 self.prePulserAfter,
454 460 self.rangeIpp,
455 461 self.rangeTxA,
456 462 self.rangeTxB)
457 463
458 464 header = numpy.array(headerTuple,RADAR_STRUCTURE)
459 465 header.tofile(fp)
460 466
461 467 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
462 468 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
463 469 samplingWindow.tofile(fp)
464 470
465 471 if self.numTaus > 0:
466 472 self.Taus.tofile(fp)
467 473
468 474 if self.codeType !=0:
469 475 nCode = numpy.array(self.nCode, '<u4')
470 476 nCode.tofile(fp)
471 477 nBaud = numpy.array(self.nBaud, '<u4')
472 478 nBaud.tofile(fp)
473 479 code1 = (self.code + 1.0)/2.
474 480
475 481 for ic in range(self.nCode):
476 482 tempx = numpy.zeros(numpy.ceil(self.nBaud/32.))
477 483 start = 0
478 484 end = 32
479 485 for i in range(len(tempx)):
480 486 code_selected = code1[ic,start:end]
481 487 for j in range(len(code_selected)-1,-1,-1):
482 488 if code_selected[j] == 1:
483 489 tempx[i] = tempx[i] + 2**(len(code_selected)-1-j)
484 490 start = start + 32
485 491 end = end + 32
486 492
487 493 tempx = tempx.astype('u4')
488 494 tempx.tofile(fp)
489 495
490 496 # if self.line5Function == RCfunction.FLIP:
491 497 # self.flip1.tofile(fp)
492 498 #
493 499 # if self.line6Function == RCfunction.FLIP:
494 500 # self.flip2.tofile(fp)
495 501
496 502 return 1
497 503
498 504 def get_ippSeconds(self):
499 505 '''
500 506 '''
501 507 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
502 508
503 509 return ippSeconds
504 510
505 511 def set_ippSeconds(self, ippSeconds):
506 512 '''
507 513 '''
508 514
509 515 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000)
510 516
511 517 return
512 518
513 519 def get_size(self):
514 520
515 521 self.__size = 116 + 12*self.nWindows + 4*self.numTaus
516 522
517 523 if self.codeType != 0:
518 524 self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
519 525
520 526 return self.__size
521 527
522 528 def set_size(self, value):
523 529
524 530 raise IOError, "size is a property and it cannot be set, just read"
525 531
526 532 return
527 533
528 534 ippSeconds = property(get_ippSeconds, set_ippSeconds)
529 535 size = property(get_size, set_size)
530 536
531 537 class ProcessingHeader(Header):
532 538
533 539 # size = None
534 540 dtype = None
535 541 blockSize = None
536 542 profilesPerBlock = None
537 543 dataBlocksPerFile = None
538 544 nWindows = None
539 545 processFlags = None
540 546 nCohInt = None
541 547 nIncohInt = None
542 548 totalSpectra = None
543
549 structure = PROCESSING_STRUCTURE
544 550 flag_dc = None
545 551 flag_cspc = None
546 552
547 def __init__(self):
553 def __init__(self, dtype=0, blockSize=0, profilesPerBlock=0, dataBlocksPerFile=0, nWindows=0,processFlags=0, nCohInt=0,
554 nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0, deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
555 code=0, nBaud=None, shif_fft=False, flag_dc=False, flag_cspc=False, flag_decode=False, flag_deflip=False
556 ):
548 557
549 558 # self.size = 0
550 self.dtype = 0
551 self.blockSize = 0
559 self.dtype = dtype
560 self.blockSize = blockSize
552 561 self.profilesPerBlock = 0
553 562 self.dataBlocksPerFile = 0
554 563 self.nWindows = 0
555 564 self.processFlags = 0
556 565 self.nCohInt = 0
557 566 self.nIncohInt = 0
558 567 self.totalSpectra = 0
559 568
560 569 self.nHeights = 0
561 570 self.firstHeight = 0
562 571 self.deltaHeight = 0
563 572 self.samplesWin = 0
564 573 self.spectraComb = 0
565 574 self.nCode = None
566 575 self.code = None
567 576 self.nBaud = None
568 577
569 578 self.shif_fft = False
570 579 self.flag_dc = False
571 580 self.flag_cspc = False
572 581 self.flag_decode = False
573 582 self.flag_deflip = False
574 583 self.length = 0
584
575 585 def read(self, fp):
576 586 self.length = 0
577 587 try:
578 588 startFp = fp.tell()
579 589 except Exception, e:
580 590 startFp = None
581 591 pass
582 592
583 593 try:
584 594 if hasattr(fp, 'read'):
585 595 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
586 596 else:
587 597 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
588 598 self.length += header.nbytes
589 599 except Exception, e:
590 600 print "ProcessingHeader: " + str(e)
591 601 return 0
592 602
593 603 size = int(header['nSize'][0])
594 604 self.dtype = int(header['nDataType'][0])
595 605 self.blockSize = int(header['nSizeOfDataBlock'][0])
596 606 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
597 607 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
598 608 self.nWindows = int(header['nNumWindows'][0])
599 609 self.processFlags = header['nProcessFlags']
600 610 self.nCohInt = int(header['nCoherentIntegrations'][0])
601 611 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
602 612 self.totalSpectra = int(header['nTotalSpectra'][0])
603 613
604 614 try:
605 615 if hasattr(fp, 'read'):
606 616 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
607 617 else:
608 618 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
609 619 self.length += samplingWindow.nbytes
610 620 except Exception, e:
611 621 print "ProcessingHeader: " + str(e)
612 622 return 0
613 623
614 624 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
615 625 self.firstHeight = float(samplingWindow['h0'][0])
616 626 self.deltaHeight = float(samplingWindow['dh'][0])
617 627 self.samplesWin = samplingWindow['nsa'][0]
618 628
619 629
620 630 try:
621 631 if hasattr(fp, 'read'):
622 632 self.spectraComb = numpy.fromfile(fp, 'u1', 2*self.totalSpectra)
623 633 else:
624 634 self.spectraComb = numpy.fromstring(fp[self.length:], 'u1', 2*self.totalSpectra)
625 635 self.length += self.spectraComb.nbytes
626 636 except Exception, e:
627 637 print "ProcessingHeader: " + str(e)
628 638 return 0
629 639
630 640 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
631 641 self.nCode = int(numpy.fromfile(fp,'<u4',1))
632 642 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
633 643 self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud)
634 644
635 645 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
636 646 exp_name_len = int(numpy.fromfile(fp,'<u4',1))
637 647 exp_name = numpy.fromfile(fp,'u1',exp_name_len+1)
638 648
639 649 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
640 650 self.shif_fft = True
641 651 else:
642 652 self.shif_fft = False
643 653
644 654 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
645 655 self.flag_dc = True
646 656 else:
647 657 self.flag_dc = False
648 658
649 659 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
650 660 self.flag_decode = True
651 661 else:
652 662 self.flag_decode = False
653 663
654 664 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
655 665 self.flag_deflip = True
656 666 else:
657 667 self.flag_deflip = False
658 668
659 669 nChannels = 0
660 670 nPairs = 0
661 671 pairList = []
662 672
663 673 for i in range( 0, self.totalSpectra*2, 2 ):
664 674 if self.spectraComb[i] == self.spectraComb[i+1]:
665 675 nChannels = nChannels + 1 #par de canales iguales
666 676 else:
667 677 nPairs = nPairs + 1 #par de canales diferentes
668 678 pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
669 679
670 680 self.flag_cspc = False
671 681 if nPairs > 0:
672 682 self.flag_cspc = True
673 683
674 684
675 685
676 686 if startFp is not None:
677 687 endFp = size + startFp
678 688 if fp.tell() > endFp:
679 689 sys.stderr.write("Warning: Processing header size is lower than it has to be")
680 690 return 0
681 691
682 692 if fp.tell() < endFp:
683 693 sys.stderr.write("Warning: Processing header size is greater than it is considered")
684 694
685 695 return 1
686 696
687 697 def write(self, fp):
688 698 #Clear DEFINE_PROCESS_CODE
689 699 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
690 700
691 701 headerTuple = (self.size,
692 702 self.dtype,
693 703 self.blockSize,
694 704 self.profilesPerBlock,
695 705 self.dataBlocksPerFile,
696 706 self.nWindows,
697 707 self.processFlags,
698 708 self.nCohInt,
699 709 self.nIncohInt,
700 710 self.totalSpectra)
701 711
702 712 header = numpy.array(headerTuple,PROCESSING_STRUCTURE)
703 713 header.tofile(fp)
704 714
705 715 if self.nWindows != 0:
706 716 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
707 717 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
708 718 samplingWindow.tofile(fp)
709 719
710 720 if self.totalSpectra != 0:
711 721 # spectraComb = numpy.array([],numpy.dtype('u1'))
712 722 spectraComb = self.spectraComb
713 723 spectraComb.tofile(fp)
714 724
715 725 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
716 726 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
717 727 # nCode.tofile(fp)
718 728 #
719 729 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
720 730 # nBaud.tofile(fp)
721 731 #
722 732 # code = self.code.reshape(self.nCode*self.nBaud)
723 733 # code = code.astype(numpy.dtype('<f4'))
724 734 # code.tofile(fp)
725 735
726 736 return 1
727 737
728 738 def get_size(self):
729 739
730 740 self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra
731 741
732 742 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
733 743 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
734 744 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
735 745
736 746 return self.__size
737 747
738 748 def set_size(self, value):
739 749
740 750 raise IOError, "size is a property and it cannot be set, just read"
741 751
742 752 return
743 753
744 754 size = property(get_size, set_size)
745 755
746 756 class RCfunction:
747 757 NONE=0
748 758 FLIP=1
749 759 CODE=2
750 760 SAMPLING=3
751 761 LIN6DIV256=4
752 762 SYNCHRO=5
753 763
754 764 class nCodeType:
755 765 NONE=0
756 766 USERDEFINE=1
757 767 BARKER2=2
758 768 BARKER3=3
759 769 BARKER4=4
760 770 BARKER5=5
761 771 BARKER7=6
762 772 BARKER11=7
763 773 BARKER13=8
764 774 AC128=9
765 775 COMPLEMENTARYCODE2=10
766 776 COMPLEMENTARYCODE4=11
767 777 COMPLEMENTARYCODE8=12
768 778 COMPLEMENTARYCODE16=13
769 779 COMPLEMENTARYCODE32=14
770 780 COMPLEMENTARYCODE64=15
771 781 COMPLEMENTARYCODE128=16
772 782 CODE_BINARY28=17
773 783
774 784 class PROCFLAG:
775 785
776 786 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
777 787 DECODE_DATA = numpy.uint32(0x00000002)
778 788 SPECTRA_CALC = numpy.uint32(0x00000004)
779 789 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
780 790 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
781 791 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
782 792
783 793 DATATYPE_CHAR = numpy.uint32(0x00000040)
784 794 DATATYPE_SHORT = numpy.uint32(0x00000080)
785 795 DATATYPE_LONG = numpy.uint32(0x00000100)
786 796 DATATYPE_INT64 = numpy.uint32(0x00000200)
787 797 DATATYPE_FLOAT = numpy.uint32(0x00000400)
788 798 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
789 799
790 800 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
791 801 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
792 802 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
793 803
794 804 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
795 805 DEFLIP_DATA = numpy.uint32(0x00010000)
796 806 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
797 807
798 808 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
799 809 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
800 810 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
801 811 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
802 812 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
803 813
804 814 EXP_NAME_ESP = numpy.uint32(0x00200000)
805 815 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
806 816
807 817 OPERATION_MASK = numpy.uint32(0x0000003F)
808 818 DATATYPE_MASK = numpy.uint32(0x00000FC0)
809 819 DATAARRANGE_MASK = numpy.uint32(0x00007000)
810 820 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
811 821
812 822 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
813 823 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
814 824 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
815 825 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
816 826 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
817 827 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
818 828
819 829 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
820 830
821 831 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
822 832 PROCFLAG.DATATYPE_SHORT,
823 833 PROCFLAG.DATATYPE_LONG,
824 834 PROCFLAG.DATATYPE_INT64,
825 835 PROCFLAG.DATATYPE_FLOAT,
826 836 PROCFLAG.DATATYPE_DOUBLE]
827 837
828 838 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
829 839
830 840 def get_dtype_index(numpy_dtype):
831 841
832 842 index = None
833 843
834 844 for i in range(len(NUMPY_DTYPE_LIST)):
835 845 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
836 846 index = i
837 847 break
838 848
839 849 return index
840 850
841 851 def get_numpy_dtype(index):
842 852
843 853 return NUMPY_DTYPE_LIST[index]
844 854
845 855 def get_procflag_dtype(index):
846 856
847 857 return PROCFLAG_DTYPE_LIST[index]
848 858
849 859 def get_dtype_width(index):
850 860
851 861 return DTYPE_WIDTH[index] No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,631 +1,653
1 1
2 2 '''
3 3 Created on Jul 3, 2014
4 4
5 5 @author: roj-idl71
6 6 '''
7 7 import os
8 8 import datetime
9 9 import numpy
10 from profilehooks import coverage
10 11 from fractions import Fraction
11 12
12 13 try:
13 14 from gevent import sleep
14 15 except:
15 16 from time import sleep
16 17
17 18 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
18 19 from schainpy.model.data.jrodata import Voltage
19 20 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
20 21
21 22 try:
22 23 import digital_rf
23 24 except:
24 25 print 'You should install "digital_rf" module if you want to read Digital RF data'
25 26
26 27 class DigitalRFReader(ProcessingUnit):
27 28 '''
28 29 classdocs
29 30 '''
30 31
31 32 def __init__(self, **kwargs):
32 33 '''
33 34 Constructor
34 35 '''
35 36
36 37 ProcessingUnit.__init__(self, **kwargs)
37 38
38 39 self.dataOut = Voltage()
39 40 self.__printInfo = True
40 41 self.__flagDiscontinuousBlock = False
41 42 self.__bufferIndex = 9999999
42 43
43 44 self.__ippKm = None
44 45 self.__codeType = 0
45 46 self.__nCode = None
46 47 self.__nBaud = None
47 48 self.__code = None
48 49
49 50 def __getCurrentSecond(self):
50 51
51 52 return self.__thisUnixSample/self.__sample_rate
52 53
53 54 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
54 55
55 56 def __setFileHeader(self):
56 57 '''
57 58 In this method will be initialized every parameter of dataOut object (header, no data)
58 59 '''
59 60 ippSeconds = 1.0*self.__nSamples/self.__sample_rate
60 61
61 62 nProfiles = 1.0/ippSeconds #Number of profiles in one second
62 63
63 64 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
64 65 txA=0,
65 66 txB=0,
66 67 nWindows=1,
67 68 nHeights=self.__nSamples,
68 69 firstHeight=self.__firstHeigth,
69 70 deltaHeight=self.__deltaHeigth,
70 71 codeType=self.__codeType,
71 72 nCode=self.__nCode, nBaud=self.__nBaud,
72 73 code = self.__code)
73 74
74 75 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
75 76 nProfiles=nProfiles,
76 77 nChannels=len(self.__channelList),
77 78 adcResolution=14)
78 79
79 80 self.dataOut.type = "Voltage"
80 81
81 82 self.dataOut.data = None
82 83
83 84 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
84 85
85 86 # self.dataOut.nChannels = 0
86 87
87 88 # self.dataOut.nHeights = 0
88 89
89 90 self.dataOut.nProfiles = nProfiles
90 91
91 92 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
92 93
93 94 self.dataOut.channelList = self.__channelList
94 95
95 96 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
96 97
97 98 # self.dataOut.channelIndexList = None
98 99
99 100 self.dataOut.flagNoData = True
100 101
101 102 #Set to TRUE if the data is discontinuous
102 103 self.dataOut.flagDiscontinuousBlock = False
103 104
104 105 self.dataOut.utctime = None
105 106
106 107 self.dataOut.timeZone = self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
107 108
108 109 self.dataOut.dstFlag = 0
109 110
110 111 self.dataOut.errorCount = 0
111 112
112 113 self.dataOut.nCohInt = 1
113 114
114 115 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
115 116
116 117 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
117 118
118 119 self.dataOut.flagShiftFFT = False
119 120
120 121 self.dataOut.ippSeconds = ippSeconds
121 122
122 123 #Time interval between profiles
123 124 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
124 125
125 126 self.dataOut.frequency = self.__frequency
126 127
127 128 self.dataOut.realtime = self.__online
128 129
129 130 def findDatafiles(self, path, startDate=None, endDate=None):
130 131
131 132 if not os.path.isdir(path):
132 133 return []
133 134
134 135 try:
135 136 digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
136 137 except:
137 138 digitalReadObj = digital_rf.DigitalRFReader(path)
138 139
139 140 channelNameList = digitalReadObj.get_channels()
140 141
141 142 if not channelNameList:
142 143 return []
143 144
144 145 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
145 146
146 147 sample_rate = metadata_dict['sample_rate'][0]
147 148
148 149 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
149 150
150 151 try:
151 152 timezone = this_metadata_file['timezone'].value
152 153 except:
153 154 timezone = 0
154 155
155 156 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
156 157
157 158 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
158 159 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
159 160
160 161 if not startDate:
161 162 startDate = startDatetime.date()
162 163
163 164 if not endDate:
164 165 endDate = endDatatime.date()
165 166
166 167 dateList = []
167 168
168 169 thisDatetime = startDatetime
169 170
170 171 while(thisDatetime<=endDatatime):
171 172
172 173 thisDate = thisDatetime.date()
173 174
174 175 if thisDate < startDate:
175 176 continue
176 177
177 178 if thisDate > endDate:
178 179 break
179 180
180 181 dateList.append(thisDate)
181 182 thisDatetime += datetime.timedelta(1)
182 183
183 184 return dateList
184 185
185 186 def setup(self, path = None,
186 187 startDate = None,
187 188 endDate = None,
188 189 startTime = datetime.time(0,0,0),
189 190 endTime = datetime.time(23,59,59),
190 191 channelList = None,
191 192 nSamples = None,
192 193 ippKm = 60,
193 194 online = False,
194 195 delay = 60,
195 196 buffer_size = 1024,
196 197 **kwargs):
197 198 '''
198 199 In this method we should set all initial parameters.
199 200
200 201 Inputs:
201 202 path
202 203 startDate
203 204 endDate
204 205 startTime
205 206 endTime
206 207 set
207 208 expLabel
208 209 ext
209 210 online
210 211 delay
211 212 '''
212 213
213 214 if not os.path.isdir(path):
214 215 raise ValueError, "[Reading] Directory %s does not exist" %path
215 216
216 217 try:
217 218 self.digitalReadObj = digital_rf.DigitalRFReader(path, load_all_metadata=True)
218 219 except:
219 220 self.digitalReadObj = digital_rf.DigitalRFReader(path)
220 221
221 222 channelNameList = self.digitalReadObj.get_channels()
222 223
223 224 if not channelNameList:
224 225 raise ValueError, "[Reading] Directory %s does not have any files" %path
225 226
226 227 if not channelList:
227 228 channelList = range(len(channelNameList))
228 229
229 230 ########## Reading metadata ######################
230 231
231 232 metadata_dict = self.digitalReadObj.get_properties(channelNameList[channelList[0]])
232 233
233 234 self.__sample_rate = metadata_dict['sample_rate_numerator'] / metadata_dict['sample_rate_denominator']
234 235 # self.__samples_per_file = metadata_dict['samples_per_file'][0]
235 236 self.__deltaHeigth = 1e6*0.15/self.__sample_rate ## why 0.15?
236 237
237 238 this_metadata_file = self.digitalReadObj.get_digital_metadata(channelNameList[channelList[0]])
238 239 metadata_bounds = this_metadata_file.get_bounds()
239 240 self.fixed_metadata_dict = this_metadata_file.read(metadata_bounds[0])[metadata_bounds[0]] ##GET FIRST HEADER
240 241
241 242 self.__frequency = None
242 243 try:
243 244 self.__frequency = self.fixed_metadata_dict['frequency']
244 245 except:
245 246 self.__frequency = None
246 247
247 248 try:
248 249 self.__timezone = self.fixed_metadata_dict['timezone']
249 250 except:
250 251 self.__timezone = 0
251 252
252 253 self.__firstHeigth = 0
253 254
254
255 255 try:
256 256 codeType = self.fixed_metadata_dict['codeType']
257 257 except:
258 258 codeType = 0
259 259
260 260 nCode = 1
261 261 nBaud = 1
262 262 code = numpy.ones((nCode, nBaud), dtype=numpy.int)
263 263
264 264 if codeType:
265 265 nCode = self.fixed_metadata_dict['nCode']
266 266 nBaud = self.fixed_metadata_dict['nBaud']
267 267 code = self.fixed_metadata_dict['code']
268 268
269 269 if not ippKm:
270 270 try:
271 271 #seconds to km
272 272 ippKm = 1e6*0.15*self.fixed_metadata_dict['ipp']
273 273 except:
274 274 ippKm = None
275 275 ####################################################
276 276 startUTCSecond = None
277 277 endUTCSecond = None
278 278
279 279 if startDate:
280 280 startDatetime = datetime.datetime.combine(startDate, startTime)
281 281 startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
282 282
283 283 if endDate:
284 284 endDatetime = datetime.datetime.combine(endDate, endTime)
285 285 endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
286 286
287 287 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
288 288
289 289 if not startUTCSecond:
290 290 startUTCSecond = start_index/self.__sample_rate
291 291
292 292 if start_index > startUTCSecond*self.__sample_rate:
293 293 startUTCSecond = start_index/self.__sample_rate
294 294
295 295 if not endUTCSecond:
296 296 endUTCSecond = end_index/self.__sample_rate
297 297
298 298 if end_index < endUTCSecond*self.__sample_rate:
299 299 endUTCSecond = end_index/self.__sample_rate
300 300
301 301 if not nSamples:
302 302 if not ippKm:
303 303 raise ValueError, "[Reading] nSamples or ippKm should be defined"
304 304 nSamples = int(ippKm / (1e6*0.15/self.__sample_rate))
305 305
306 306 channelBoundList = []
307 307 channelNameListFiltered = []
308 308
309 309 for thisIndexChannel in channelList:
310 310 thisChannelName = channelNameList[thisIndexChannel]
311 311 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
312 312 channelBoundList.append((start_index, end_index))
313 313 channelNameListFiltered.append(thisChannelName)
314 314
315 315 self.profileIndex = 0
316 316
317 317 self.__delay = delay
318 318 self.__ippKm = ippKm
319 319 self.__codeType = codeType
320 320 self.__nCode = nCode
321 321 self.__nBaud = nBaud
322 322 self.__code = code
323 323
324 324 self.__datapath = path
325 325 self.__online = online
326 326 self.__channelList = channelList
327 327 self.__channelNameList = channelNameListFiltered
328 328 self.__channelBoundList = channelBoundList
329 329 self.__nSamples = nSamples
330 330 self.__samples_to_read = long(buffer_size*nSamples) #FIJO: AHORA 40
331 331 self.__nChannels = len(self.__channelList)
332 332
333 333 self.__startUTCSecond = startUTCSecond
334 334 self.__endUTCSecond = endUTCSecond
335 335
336 336 self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
337 337
338 338 if online:
339 339 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
340 340 startUTCSecond = numpy.floor(endUTCSecond)
341 341
342 342 self.__thisUnixSample = long(startUTCSecond*self.__sample_rate) - self.__samples_to_read ##por que en el otro metodo lo primero q se hace es sumar samplestoread
343 343
344 344 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
345 345
346 346 self.__setFileHeader()
347 347 self.isConfig = True
348 348
349 349 print "[Reading] Digital RF Data was found from %s to %s " %(
350 350 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
351 351 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
352 352 )
353 353
354 354 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
355 355 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
356 356 )
357 357
358 358 def __reload(self):
359 359 # print
360 360 # print "%s not in range [%s, %s]" %(
361 361 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
362 362 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
363 363 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
364 364 # )
365 365 print "[Reading] reloading metadata ..."
366 366
367 367 try:
368 368 self.digitalReadObj.reload(complete_update=True)
369 369 except:
370 370 self.digitalReadObj.reload()
371 371
372 372 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
373 373
374 374 if start_index > self.__startUTCSecond*self.__sample_rate:
375 375 self.__startUTCSecond = 1.0*start_index/self.__sample_rate
376 376
377 377 if end_index > self.__endUTCSecond*self.__sample_rate:
378 378 self.__endUTCSecond = 1.0*end_index/self.__sample_rate
379 379 print
380 380 print "[Reading] New timerange found [%s, %s] " %(
381 381 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
382 382 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
383 383 )
384 384
385 385 return True
386 386
387 387 return False
388 388
389 def __readNextBlock(self, seconds=30, volt_scale = 218776):
389 def __readNextBlock(self, seconds=30, volt_scale = 1):
390 390 '''
391 391 '''
392 392
393 393 #Set the next data
394 394 self.__flagDiscontinuousBlock = False
395 395 self.__thisUnixSample += self.__samples_to_read
396 396
397 397 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
398 398 print "[Reading] There are no more data into selected time-range"
399 399 if self.__online:
400 400 self.__reload()
401 401 else:
402 402 return False
403 403
404 404 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
405 405 return False
406 406 self.__thisUnixSample -= self.__samples_to_read
407 407
408 408 indexChannel = 0
409 409
410 410 dataOk = False
411 411
412 412 for thisChannelName in self.__channelNameList: ##TODO VARIOS CHANNELS?
413 413
414 414 try:
415 415 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
416 416 self.__samples_to_read,
417 417 thisChannelName)
418 418 except IOError, e:
419 419 #read next profile
420 420 self.__flagDiscontinuousBlock = True
421 421 print "[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e
422 422 break
423 423
424 424 if result.shape[0] != self.__samples_to_read:
425 425 self.__flagDiscontinuousBlock = True
426 426 print "[Reading] %s: Too few samples were found, just %d/%d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
427 427 result.shape[0],
428 428 self.__samples_to_read)
429 429 break
430 430
431 431 self.__data_buffer[indexChannel,:] = result*volt_scale
432 432
433 433 indexChannel += 1
434 434
435 435 dataOk = True
436 436
437 437 self.__utctime = self.__thisUnixSample/self.__sample_rate
438 438
439 439 if not dataOk:
440 440 return False
441 441
442 442 print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
443 443 self.__samples_to_read,
444 444 self.__timeInterval)
445 445
446 446 self.__bufferIndex = 0
447 447
448 448 return True
449 449
450 450 def __isBufferEmpty(self):
451 451 return self.__bufferIndex > self.__samples_to_read - self.__nSamples #40960 - 40
452 452
453 453 def getData(self, seconds=30, nTries=5):
454 454
455 455 '''
456 456 This method gets the data from files and put the data into the dataOut object
457 457
458 458 In addition, increase el the buffer counter in one.
459 459
460 460 Return:
461 461 data : retorna un perfil de voltages (alturas * canales) copiados desde el
462 462 buffer. Si no hay mas archivos a leer retorna None.
463 463
464 464 Affected:
465 465 self.dataOut
466 466 self.profileIndex
467 467 self.flagDiscontinuousBlock
468 468 self.flagIsNewBlock
469 469 '''
470 470
471 471 err_counter = 0
472 472 self.dataOut.flagNoData = True
473 473
474 474 if self.__isBufferEmpty():
475 475
476 476 self.__flagDiscontinuousBlock = False
477 477
478 478 while True:
479 479 if self.__readNextBlock():
480 480 break
481 481 if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
482 482 return False
483 483
484 484 if self.__flagDiscontinuousBlock:
485 485 print '[Reading] discontinuous block found ... continue with the next block'
486 486 continue
487 487
488 488 if not self.__online:
489 489 return False
490 490
491 491 err_counter += 1
492 492 if err_counter > nTries:
493 493 return False
494 494
495 495 print '[Reading] waiting %d seconds to read a new block' %seconds
496 496 sleep(seconds)
497 497
498 498 self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
499 499 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
500 500 self.dataOut.flagNoData = False
501 501 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
502 502 self.dataOut.profileIndex = self.profileIndex
503 503
504 504 self.__bufferIndex += self.__nSamples
505 505 self.profileIndex += 1
506 506
507 507 if self.profileIndex == self.dataOut.nProfiles:
508 508 self.profileIndex = 0
509 509
510 510 return True
511 511
512 512 def printInfo(self):
513 513 '''
514 514 '''
515 515 if self.__printInfo == False:
516 516 return
517 517
518 518 # self.systemHeaderObj.printInfo()
519 519 # self.radarControllerHeaderObj.printInfo()
520 520
521 521 self.__printInfo = False
522 522
523 523 def printNumberOfBlock(self):
524 524 '''
525 525 '''
526 526 return
527 527 #print self.profileIndex
528 528
529 529 def run(self, **kwargs):
530 530 '''
531 531 This method will be called many times so here you should put all your code
532 532 '''
533 533
534 534 if not self.isConfig:
535 535 self.setup(**kwargs)
536 536
537 537 self.getData(seconds=self.__delay)
538 538
539 539 return
540 540
541 541 class DigitalRFWriter(Operation):
542 542 '''
543 543 classdocs
544 544 '''
545 545
546 546 def __init__(self, **kwargs):
547 547 '''
548 548 Constructor
549 549 '''
550 550 Operation.__init__(self, **kwargs)
551 551 self.dataOut = None
552 552
553 def setup(self, dataIn, path, set=0, ext='.h5'):
553 def setup(self, dataOut, path, set=0, metadataFile='metadata', ext='.h5'):
554 554 '''
555 555 In this method we should set all initial parameters.
556 556
557 557 Input:
558 dataIn : Input data will also be outputa data
558 dataOut : Input data will also be outputa data
559 559
560 560 '''
561 561
562 self.__ippSeconds = dataIn.ippSeconds
563 self.__deltaH = dataIn.getDeltaH()
562 self.__ippSeconds = dataOut.ippSeconds
563 self.__deltaH = dataOut.getDeltaH()
564 564 self.__sample_rate = 1e6*0.15/self.__deltaH
565 self.__dtype = dataIn.dtype
566 if len(dataIn.dtype) == 2:
567 self.__dtype = dataIn.dtype[0]
568 self.__nSamples = dataIn.systemHeaderObj.nSamples
569 self.__nProfiles = dataIn.nProfiles
570 self.__blocks_per_file = dataIn.processingHeaderObj.dataBlocksPerFile
571
572 file_cadence_millisecs = 1.0 * self.__blocks_per_file * self.__nProfiles * self.__nSamples / self.__sample_rate * 1000
573 sub_cadence_secs = 10 * file_cadence_millisecs
565 self.__dtype = dataOut.dtype
566 if len(dataOut.dtype) == 2:
567 self.__dtype = dataOut.dtype[0]
568 self.__nSamples = dataOut.systemHeaderObj.nSamples
569 self.__nProfiles = dataOut.nProfiles
570 self.__blocks_per_file = dataOut.processingHeaderObj.dataBlocksPerFile
571 self.arr_data = arr_data = numpy.ones((self.__nSamples, 1), dtype=[('r', self.__dtype), ('i', self.__dtype)])
572
573 file_cadence_millisecs = long(1.0 * self.__blocks_per_file * self.__nProfiles * self.__nSamples / self.__sample_rate * 1000)
574 sub_cadence_secs = file_cadence_millisecs
575
576 #print file_cadence_millisecs
577 #print sub_cadence_secs
578
574 579 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
575 580 sample_rate_numerator = long(sample_rate_fraction.numerator)
576 581 sample_rate_denominator = long(sample_rate_fraction.denominator)
577 start_global_index = dataIn.utctime * self.__sample_rate
578 self.arr_data = arr_data = numpy.ones((self.__nSamples, 1), dtype=[('r', self.__dtype), ('i', self.__dtype)])
582 start_global_index = dataOut.utctime * self.__sample_rate
583
579 584 uuid = 'prueba'
580 585 compression_level = 1
581 586 checksum = False
582 587 is_complex = True
583 588 num_subchannels = 1
584 589 is_continuous = True
585 590 marching_periods = False
586 591
587 self.digitalWriteObj = digital_rf.DigitalRFWriter("/home/jchavez/jicamarca/mocked_data/voltage", self.__dtype, sub_cadence_secs,
592 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, sub_cadence_secs,
588 593 file_cadence_millisecs, start_global_index,
589 594 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
590 595 is_complex, num_subchannels, is_continuous, marching_periods)
596
597 metadata_dir = os.path.join(path, 'metadata')
598 os.system('mkdir %s' % (metadata_dir))
599
600 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, 236, file_cadence_millisecs / 1000,
601 sample_rate_numerator, sample_rate_denominator,
602 metadataFile)
591 603
592 604
593 605 self.isConfig = True
594
606 self.currentSample = 0
595 607 return
596 608
597 def run(self, dataIn, path=None, **kwargs):
609 @coverage
610 def run(self, dataOut, path=None, **kwargs):
598 611 '''
599 612 This method will be called many times so here you should put all your code
600 613
601 614 Inputs:
602 615
603 dataIn : object with the data
616 dataOut : object with the data
604 617
605 618 '''
606
607 self.dataOut = dataIn
619 #print dataOut.__dict__
620 self.dataOut = dataOut
608 621
609 622 if not self.isConfig:
610 self.setup(dataIn, path, **kwargs)
623 self.setup(dataOut, path, **kwargs)
611 624
612 625 samples = len(self.dataOut.data[0])
613 626
614 627 for i in range(samples):
615 self.arr_data[i]['r'] = dataIn.data[0][i].real
616 self.arr_data[i]['i'] = dataIn.data[0][i].imag
617
618 if dataIn.flagNoData:
619 self.digitalWriteObj.close()
620
628 self.arr_data[i]['r'] = dataOut.data[0][i].real
629 self.arr_data[i]['i'] = dataOut.data[0][i].imag
621 630 self.digitalWriteObj.rf_write(self.arr_data)
622
631 start_idx = self.__sample_rate * dataOut.utctime
632 metadata_dict = {}
633 metadata_dict['frequency'] = 49.92e6
634 metadata_dict['blablabla'] = 49.92e6
635 self.currentSample += 1
636 if self.dataOut.flagDataAsBlock:
637 self.digitalMetadataWriteObj.write(start_idx, metadata_dict)
638 elif self.currentSample == 1:
639 print '[Writing] - Writing metadata'
640 self.digitalMetadataWriteObj.write(start_idx, metadata_dict)
641 if self.currentSample == self.__nProfiles: self.currentSample = 0
642 def close(self):
643 print '[Writing] - Closing files '
644 self.digitalWriteObj.close()
623 645 #raise
624 646 if __name__ == '__main__':
625 647
626 648 readObj = DigitalRFReader()
627 649
628 650 while True:
629 651 readObj.run(path='/home/jchavez/jicamarca/mocked_data/')
630 652 # readObj.printInfo()
631 readObj.printNumberOfBlock()
653 #readObj.printNumberOfBlock()
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="Segundo Test" id="191" name="test01"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/jchavez/jicamarca/jro_data/rawdata" /><Parameter format="date" id="191113" name="startDate" value="2010/10/28" /><Parameter format="date" id="191114" name="endDate" value="2017/10/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /><Parameter format="int" id="191119" name="walk" value="0" /></Operation><Operation id="19112" name="printNumberOfBlock" priority="2" type="self" /></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="DigitalRFWriter" priority="2" type="other"><Parameter format="str" id="191221" name="path" value="/home/jchavez/jicamarca/data_sink/" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="Segundo Test" id="191" name="test01"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/jchavez/jicamarca/jro_data/rawdata" /><Parameter format="date" id="191113" name="startDate" value="2010/10/28" /><Parameter format="date" id="191114" name="endDate" value="2017/10/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /><Parameter format="int" id="191119" name="walk" value="0" /></Operation><Operation id="19112" name="printNumberOfBlock" priority="2" type="self" /></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="DigitalRFWriter" priority="2" type="other"><Parameter format="str" id="191221" name="path" value="/home/jchavez/jicamarca/mocked_data/voltage" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,94 +1,98
1 1 import os, sys
2 2
3 3 from schainpy.controller import Project
4 4
5 5 if __name__ == '__main__':
6 6
7 7 desc = "Segundo Test"
8 8 filename = "schain.xml"
9 9
10 10 controllerObj = Project()
11 11
12 12 controllerObj.setup(id = '191', name='test01', description=desc)
13 13
14 14 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
15 15 path='/home/jchavez/jicamarca/jro_data/rawdata/',
16 16 startDate='2010/10/28',
17 17 endDate='2017/10/28',
18 18 startTime='00:00:00',
19 19 endTime='23:59:59',
20 20 online=0,
21 21 walk=0)
22 22
23 23 opObj00 = readUnitConfObj.addOperation(name='printNumberOfBlock')
24 24
25 25 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc',
26 26 inputId=readUnitConfObj.getId())
27 27
28 # opObj11 = procUnitConfObj0.addOperation(name='Scope', optype='external')
29 # opObj11.addParameter(name='id', value='121', format='int')
30 # opObj11.addParameter(name='wintitle', value='Scope', format='str')
31
28 32 opObj10 = procUnitConfObj0.addOperation(name='DigitalRFWriter', optype='other')
29 opObj10.addParameter(name='path', value='/home/jchavez/jicamarca/data_sink/', format='str')
33 opObj10.addParameter(name='path', value='/home/jchavez/jicamarca/mocked_data/voltage', format='str')
30 34 # opObj10.addParameter(name='minHei', value='0', format='float')
31 35 # opObj10.addParameter(name='maxHei', value='8', format='float')
32 36
33 37 # opObj10 = procUnitConfObj0.addOperation(name='filterByHeights')
34 38 # opObj10.addParameter(name='window', value='2', format='float')
35 39
36 40 # opObj10 = procUnitConfObj0.addOperation(name='Decoder', optype='external')
37 41 # opObj10.addParameter(name='code', value='1,-1', format='intlist')
38 42 # opObj10.addParameter(name='nCode', value='2', format='float')
39 43 # opObj10.addParameter(name='nBaud', value='1', format='float')
40 44
41 45
42 46 # opObj10 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
43 47 # opObj10.addParameter(name='n', value='1296', format='float')
44 48
45 49 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc',
46 50 # inputId=procUnitConfObj0.getId())
47 51
48 52 #Creating a processing object with its parameters
49 53 #schainpy.model.proc.jroproc_spectra.SpectraProc.run()
50 54 #If you need to add more parameters can use the "addParameter method"
51 55 # procUnitConfObj1.addParameter(name='nFFTPoints', value='128', format='int')
52 56
53 57 # opObj10 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
54 58 # opObj10.addParameter(name='n', value='2', format='float')
55 59
56 60 #Using internal methods
57 61 #schainpy.model.proc.jroproc_spectra.SpectraProc.selectChannels()
58 62 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
59 63 # opObj10.addParameter(name='channelList', value='0,1', format='intlist')
60 64
61 65 #Using internal methods
62 66 #schainpy.model.proc.jroproc_spectra.SpectraProc.selectHeights()
63 67 # opObj10 = procUnitConfObj1.addOperation(name='selectHeights')
64 68 # opObj10.addParameter(name='minHei', value='90', format='float')
65 69 # opObj10.addParameter(name='maxHei', value='180', format='float')
66 70
67 71 #Using external methods (new modules)
68 72 # #schainpy.model.proc.jroproc_spectra.IncohInt.setup()
69 73 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='other')
70 74 # opObj12.addParameter(name='n', value='1', format='int')
71 75
72 76 #Using external methods (new modules)
73 77 #schainpy.model.graphics.jroplot_spectra.SpectraPlot.setup()
74 78 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
75 79 # opObj11.addParameter(name='id', value='11', format='int')
76 80 # opObj11.addParameter(name='wintitle', value='SpectraPlot', format='str')
77 81 # opObj11.addParameter(name='zmin', value='-60', format='int')
78 82 # opObj11.addParameter(name='zmax', value='10', format='int')
79 83 # opObj11.addParameter(name='save', value='1', format='int')
80 84
81 85 # #Using external methods (new modules)
82 86 # #schainpy.model.graphics.jroplot_spectra.RTIPlot.setup()
83 87 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
84 88 # opObj11.addParameter(name='id', value='30', format='int')
85 89 # opObj11.addParameter(name='wintitle', value='RTI', format='str')
86 90 # opObj11.addParameter(name='zmin', value='-60', format='int')
87 91 # opObj11.addParameter(name='zmax', value='-10', format='int')
88 92 # opObj11.addParameter(name='showprofile', value='1', format='int')
89 93 # # opObj11.addParameter(name='timerange', value=str(5*60*60*60), format='int')
90 94 # opObj11.addParameter(name='xmin', value='14', format='float')
91 95 # opObj11.addParameter(name='xmax', value='23.9', format='float')
92 96 # opObj11.addParameter(name='save', value='1', format='int')
93 97
94 98 controllerObj.start()
General Comments 0
You need to be logged in to leave comments. Login now