##// END OF EJS Templates
schain
RSS
there is a new class AMISR to read HDF5 files from that AMISR system.
Daniel Valdez -
r473:e228fb468666
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,646 +1,657
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
5 5 '''
6 6
7 7 import os, sys
8 8 import copy
9 9 import numpy
10 10 import datetime
11 11
12 12 from jroheaderIO import SystemHeader, RadarControllerHeader
13 13
14 14
15 15 def hildebrand_sekhon(data, navg):
16 16
17 17 data = data.copy()
18 18
19 19 sortdata = numpy.sort(data,axis=None)
20 20 lenOfData = len(sortdata)
21 21 nums_min = lenOfData/10
22 22
23 23 if (lenOfData/10) > 2:
24 24 nums_min = lenOfData/10
25 25 else:
26 26 nums_min = 2
27 27
28 28 sump = 0.
29 29
30 30 sumq = 0.
31 31
32 32 j = 0
33 33
34 34 cont = 1
35 35
36 36 while((cont==1)and(j<lenOfData)):
37 37
38 38 sump += sortdata[j]
39 39
40 40 sumq += sortdata[j]**2
41 41
42 42 j += 1
43 43
44 44 if j > nums_min:
45 45 rtest = float(j)/(j-1) + 1.0/navg
46 46 if ((sumq*j) > (rtest*sump**2)):
47 47 j = j - 1
48 48 sump = sump - sortdata[j]
49 49 sumq = sumq - sortdata[j]**2
50 50 cont = 0
51 51
52 52 lnoise = sump /j
53 53 stdv = numpy.sqrt((sumq - lnoise**2)/(j - 1))
54 54 return lnoise
55 55
56 56 class JROData:
57 57
58 58 # m_BasicHeader = BasicHeader()
59 59 # m_ProcessingHeader = ProcessingHeader()
60 60
61 61 systemHeaderObj = SystemHeader()
62 62
63 63 radarControllerHeaderObj = RadarControllerHeader()
64 64
65 65 # data = None
66 66
67 67 type = None
68 68
69 69 dtype = None
70 70
71 71 # nChannels = None
72 72
73 73 # nHeights = None
74 74
75 75 nProfiles = None
76 76
77 77 heightList = None
78 78
79 79 channelList = None
80 80
81 81 flagNoData = True
82 82
83 83 flagTimeBlock = False
84 84
85 85 useLocalTime = False
86 86
87 87 utctime = None
88 88
89 89 timeZone = None
90 90
91 91 dstFlag = None
92 92
93 93 errorCount = None
94 94
95 95 blocksize = None
96 96
97 97 nCode = None
98 98
99 99 nBaud = None
100 100
101 101 code = None
102 102
103 103 flagDecodeData = False #asumo q la data no esta decodificada
104 104
105 105 flagDeflipData = False #asumo q la data no esta sin flip
106 106
107 107 flagShiftFFT = False
108 108
109 109 ippSeconds = None
110 110
111 111 timeInterval = None
112 112
113 113 nCohInt = None
114 114
115 115 noise = None
116 116
117 117 windowOfFilter = 1
118 118
119 119 #Speed of ligth
120 120 C = 3e8
121 121
122 122 frequency = 49.92e6
123 123
124 124 realtime = False
125 125
126 126 beacon_heiIndexList = None
127 127
128 128 last_block = None
129 129
130 130 blocknow = None
131 131
132 132 def __init__(self):
133 133
134 134 raise ValueError, "This class has not been implemented"
135 135
136 136 def copy(self, inputObj=None):
137 137
138 138 if inputObj == None:
139 139 return copy.deepcopy(self)
140 140
141 141 for key in inputObj.__dict__.keys():
142 142 self.__dict__[key] = inputObj.__dict__[key]
143 143
144 144 def deepcopy(self):
145 145
146 146 return copy.deepcopy(self)
147 147
148 148 def isEmpty(self):
149 149
150 150 return self.flagNoData
151 151
152 152 def getNoise(self):
153 153
154 154 raise ValueError, "Not implemented"
155 155
156 156 def getNChannels(self):
157 157
158 158 return len(self.channelList)
159 159
160 160 def getChannelIndexList(self):
161 161
162 162 return range(self.nChannels)
163 163
164 164 def getNHeights(self):
165 165
166 166 return len(self.heightList)
167 167
168 168 def getHeiRange(self, extrapoints=0):
169 169
170 170 heis = self.heightList
171 171 # deltah = self.heightList[1] - self.heightList[0]
172 172 #
173 173 # heis.append(self.heightList[-1])
174 174
175 175 return heis
176 176
177 177 def getltctime(self):
178 178
179 179 if self.useLocalTime:
180 180 return self.utctime - self.timeZone*60
181 181
182 182 return self.utctime
183 183
184 184 def getDatatime(self):
185 185
186 186 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
187 187 return datatime
188 188
189 189 def getTimeRange(self):
190 190
191 191 datatime = []
192 192
193 193 datatime.append(self.ltctime)
194 194 datatime.append(self.ltctime + self.timeInterval)
195 195
196 196 datatime = numpy.array(datatime)
197 197
198 198 return datatime
199 199
200 200 def getFmax(self):
201 201
202 202 PRF = 1./(self.ippSeconds * self.nCohInt)
203 203
204 204 fmax = PRF/2.
205 205
206 206 return fmax
207 207
208 208 def getVmax(self):
209 209
210 210 _lambda = self.C/self.frequency
211 211
212 212 vmax = self.getFmax() * _lambda
213 213
214 214 return vmax
215 215
216 216 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
217 217 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
218 218 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
219 219 noise = property(getNoise, "I'm the 'nHeights' property.")
220 220 datatime = property(getDatatime, "I'm the 'datatime' property")
221 221 ltctime = property(getltctime, "I'm the 'ltctime' property")
222 222
223 223 class Voltage(JROData):
224 224
225 225 #data es un numpy array de 2 dmensiones (canales, alturas)
226 226 data = None
227 227
228 228 def __init__(self):
229 229 '''
230 230 Constructor
231 231 '''
232 232
233 233 self.radarControllerHeaderObj = RadarControllerHeader()
234 234
235 235 self.systemHeaderObj = SystemHeader()
236 236
237 237 self.type = "Voltage"
238 238
239 239 self.data = None
240 240
241 241 self.dtype = None
242 242
243 243 # self.nChannels = 0
244 244
245 245 # self.nHeights = 0
246 246
247 247 self.nProfiles = None
248 248
249 249 self.heightList = None
250 250
251 251 self.channelList = None
252 252
253 253 # self.channelIndexList = None
254 254
255 255 self.flagNoData = True
256 256
257 257 self.flagTimeBlock = False
258 258
259 259 self.utctime = None
260 260
261 261 self.timeZone = None
262 262
263 263 self.dstFlag = None
264 264
265 265 self.errorCount = None
266 266
267 267 self.nCohInt = None
268 268
269 269 self.blocksize = None
270 270
271 271 self.flagDecodeData = False #asumo q la data no esta decodificada
272 272
273 273 self.flagDeflipData = False #asumo q la data no esta sin flip
274 274
275 275 self.flagShiftFFT = False
276 276
277 277
278 278 def getNoisebyHildebrand(self):
279 279 """
280 280 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
281 281
282 282 Return:
283 283 noiselevel
284 284 """
285 285
286 286 for channel in range(self.nChannels):
287 287 daux = self.data_spc[channel,:,:]
288 288 self.noise[channel] = hildebrand_sekhon(daux, self.nCohInt)
289 289
290 290 return self.noise
291 291
292 292 def getNoise(self, type = 1):
293 293
294 294 self.noise = numpy.zeros(self.nChannels)
295 295
296 296 if type == 1:
297 297 noise = self.getNoisebyHildebrand()
298 298
299 299 return 10*numpy.log10(noise)
300 300
301 301 class Spectra(JROData):
302 302
303 303 #data es un numpy array de 2 dmensiones (canales, perfiles, alturas)
304 304 data_spc = None
305 305
306 306 #data es un numpy array de 2 dmensiones (canales, pares, alturas)
307 307 data_cspc = None
308 308
309 309 #data es un numpy array de 2 dmensiones (canales, alturas)
310 310 data_dc = None
311 311
312 312 nFFTPoints = None
313 313
314 314 nPairs = None
315 315
316 316 pairsList = None
317 317
318 318 nIncohInt = None
319 319
320 320 wavelength = None #Necesario para cacular el rango de velocidad desde la frecuencia
321 321
322 322 nCohInt = None #se requiere para determinar el valor de timeInterval
323 323
324 324 ippFactor = None
325 325
326 326 def __init__(self):
327 327 '''
328 328 Constructor
329 329 '''
330 330
331 331 self.radarControllerHeaderObj = RadarControllerHeader()
332 332
333 333 self.systemHeaderObj = SystemHeader()
334 334
335 335 self.type = "Spectra"
336 336
337 337 # self.data = None
338 338
339 339 self.dtype = None
340 340
341 341 # self.nChannels = 0
342 342
343 343 # self.nHeights = 0
344 344
345 345 self.nProfiles = None
346 346
347 347 self.heightList = None
348 348
349 349 self.channelList = None
350 350
351 351 # self.channelIndexList = None
352 352
353 353 self.flagNoData = True
354 354
355 355 self.flagTimeBlock = False
356 356
357 357 self.utctime = None
358 358
359 359 self.nCohInt = None
360 360
361 361 self.nIncohInt = None
362 362
363 363 self.blocksize = None
364 364
365 365 self.nFFTPoints = None
366 366
367 367 self.wavelength = None
368 368
369 369 self.flagDecodeData = False #asumo q la data no esta decodificada
370 370
371 371 self.flagDeflipData = False #asumo q la data no esta sin flip
372 372
373 373 self.flagShiftFFT = False
374 374
375 375 self.ippFactor = 1
376 376
377 377 self.noise = None
378 378
379 379 self.beacon_heiIndexList = []
380 380
381 381
382 382 def getNoisebyHildebrand(self):
383 383 """
384 384 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
385 385
386 386 Return:
387 387 noiselevel
388 388 """
389 389 noise = numpy.zeros(self.nChannels)
390 390 for channel in range(self.nChannels):
391 391 daux = self.data_spc[channel,:,:]
392 392 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
393 393
394 394 return noise
395 395
396 396 def getNoise(self):
397 397 if self.noise != None:
398 398 return self.noise
399 399 else:
400 400 noise = self.getNoisebyHildebrand()
401 401 return noise
402 402
403 403
404 404 def getFreqRange(self, extrapoints=0):
405 405
406 406 deltafreq = self.getFmax() / (self.nFFTPoints*self.ippFactor)
407 407 freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
408 408
409 409 return freqrange
410 410
411 411 def getVelRange(self, extrapoints=0):
412 412
413 413 deltav = self.getVmax() / (self.nFFTPoints*self.ippFactor)
414 414 velrange = deltav*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltav/2
415 415
416 416 return velrange
417 417
418 418 def getNPairs(self):
419 419
420 420 return len(self.pairsList)
421 421
422 422 def getPairsIndexList(self):
423 423
424 424 return range(self.nPairs)
425 425
426 426 def getNormFactor(self):
427 427 pwcode = 1
428 428 if self.flagDecodeData:
429 429 pwcode = numpy.sum(self.code[0]**2)
430 430 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
431 431 normFactor = self.nProfiles*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
432 432
433 433 return normFactor
434 434
435 435 def getFlagCspc(self):
436 436
437 437 if self.data_cspc == None:
438 438 return True
439 439
440 440 return False
441 441
442 442 def getFlagDc(self):
443 443
444 444 if self.data_dc == None:
445 445 return True
446 446
447 447 return False
448 448
449 449 nPairs = property(getNPairs, "I'm the 'nPairs' property.")
450 450 pairsIndexList = property(getPairsIndexList, "I'm the 'pairsIndexList' property.")
451 451 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
452 452 flag_cspc = property(getFlagCspc)
453 453 flag_dc = property(getFlagDc)
454 454
455 455 class SpectraHeis(JROData):
456 456
457 457 data_spc = None
458 458
459 459 data_cspc = None
460 460
461 461 data_dc = None
462 462
463 463 nFFTPoints = None
464 464
465 465 nPairs = None
466 466
467 467 pairsList = None
468 468
469 469 nIncohInt = None
470 470
471 471 def __init__(self):
472 472
473 473 self.radarControllerHeaderObj = RadarControllerHeader()
474 474
475 475 self.systemHeaderObj = SystemHeader()
476 476
477 477 self.type = "SpectraHeis"
478 478
479 479 self.dtype = None
480 480
481 481 # self.nChannels = 0
482 482
483 483 # self.nHeights = 0
484 484
485 485 self.nProfiles = None
486 486
487 487 self.heightList = None
488 488
489 489 self.channelList = None
490 490
491 491 # self.channelIndexList = None
492 492
493 493 self.flagNoData = True
494 494
495 495 self.flagTimeBlock = False
496 496
497 497 self.nPairs = 0
498 498
499 499 self.utctime = None
500 500
501 501 self.blocksize = None
502 502
503 503 class Fits:
504 504
505 505 heightList = None
506 506
507 507 channelList = None
508 508
509 509 flagNoData = True
510 510
511 511 flagTimeBlock = False
512 512
513 513 useLocalTime = False
514 514
515 515 utctime = None
516 516
517 517 timeZone = None
518 518
519 519 ippSeconds = None
520 520
521 521 timeInterval = None
522 522
523 523 nCohInt = None
524 524
525 525 nIncohInt = None
526 526
527 527 noise = None
528 528
529 529 windowOfFilter = 1
530 530
531 531 #Speed of ligth
532 532 C = 3e8
533 533
534 534 frequency = 49.92e6
535 535
536 536 realtime = False
537 537
538 538
539 539 def __init__(self):
540 540
541 541 self.type = "Fits"
542 542
543 543 self.nProfiles = None
544 544
545 545 self.heightList = None
546 546
547 547 self.channelList = None
548 548
549 549 # self.channelIndexList = None
550 550
551 551 self.flagNoData = True
552 552
553 553 self.utctime = None
554 554
555 555 self.nCohInt = None
556 556
557 557 self.nIncohInt = None
558 558
559 559 self.useLocalTime = True
560 560
561 561 # self.utctime = None
562 562 # self.timeZone = None
563 563 # self.ltctime = None
564 564 # self.timeInterval = None
565 565 # self.header = None
566 566 # self.data_header = None
567 567 # self.data = None
568 568 # self.datatime = None
569 569 # self.flagNoData = False
570 570 # self.expName = ''
571 571 # self.nChannels = None
572 572 # self.nSamples = None
573 573 # self.dataBlocksPerFile = None
574 574 # self.comments = ''
575 575 #
576 576
577 577
578 578 def getltctime(self):
579 579
580 580 if self.useLocalTime:
581 581 return self.utctime - self.timeZone*60
582 582
583 583 return self.utctime
584 584
585 585 def getDatatime(self):
586 586
587 587 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
588 588 return datatime
589 589
590 590 def getTimeRange(self):
591 591
592 592 datatime = []
593 593
594 594 datatime.append(self.ltctime)
595 595 datatime.append(self.ltctime + self.timeInterval)
596 596
597 597 datatime = numpy.array(datatime)
598 598
599 599 return datatime
600 600
601 601 def getHeiRange(self):
602 602
603 603 heis = self.heightList
604 604
605 605 return heis
606 606
607 607 def isEmpty(self):
608 608
609 609 return self.flagNoData
610 610
611 611 def getNHeights(self):
612 612
613 613 return len(self.heightList)
614 614
615 615 def getNChannels(self):
616 616
617 617 return len(self.channelList)
618 618
619 619 def getChannelIndexList(self):
620 620
621 621 return range(self.nChannels)
622 622
623 623 def getNoise(self, type = 1):
624 624
625 625 self.noise = numpy.zeros(self.nChannels)
626 626
627 627 if type == 1:
628 628 noise = self.getNoisebyHildebrand()
629 629
630 630 if type == 2:
631 631 noise = self.getNoisebySort()
632 632
633 633 if type == 3:
634 634 noise = self.getNoisebyWindow()
635 635
636 636 return noise
637 637
638 638 datatime = property(getDatatime, "I'm the 'datatime' property")
639 639 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
640 640 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
641 641 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
642 642 noise = property(getNoise, "I'm the 'nHeights' property.")
643 643 datatime = property(getDatatime, "I'm the 'datatime' property")
644 644 ltctime = property(getltctime, "I'm the 'ltctime' property")
645 645
646 ltctime = property(getltctime, "I'm the 'ltctime' property") No newline at end of file
646 ltctime = property(getltctime, "I'm the 'ltctime' property")
647
648 class AMISR:
649 def __init__(self):
650 self.flagNoData = True
651 self.data = None
652 self.utctime = None
653 self.type = "AMISR"
654
655 def isEmpty(self):
656
657 return self.flagNoData No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,3421 +1,3825
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 import os, sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 import h5py
13 14 from xml.etree.ElementTree import Element, SubElement, ElementTree
14 15 try:
15 16 import pyfits
16 17 except:
17 18 print "pyfits module has not been imported, it should be installed to save files in fits format"
18 19
19 20 from jrodata import *
20 21 from jroheaderIO import *
21 22 from jroprocessing import *
22 23
23 24 LOCALTIME = True #-18000
24 25
25 26 def isNumber(str):
26 27 """
27 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 29
29 30 Excepciones:
30 31 Si un determinado string no puede ser convertido a numero
31 32 Input:
32 33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 34
34 35 Return:
35 36 True : si el string es uno numerico
36 37 False : no es un string numerico
37 38 """
38 39 try:
39 40 float( str )
40 41 return True
41 42 except:
42 43 return False
43 44
44 45 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 46 """
46 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 48
48 49 Inputs:
49 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 51
51 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 53 segundos contados desde 01/01/1970.
53 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 55 segundos contados desde 01/01/1970.
55 56
56 57 Return:
57 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 59 fecha especificado, de lo contrario retorna False.
59 60
60 61 Excepciones:
61 62 Si el archivo no existe o no puede ser abierto
62 63 Si la cabecera no puede ser leida.
63 64
64 65 """
65 66 basicHeaderObj = BasicHeader(LOCALTIME)
66 67
67 68 try:
68 69 fp = open(filename,'rb')
69 70 except:
70 71 raise IOError, "The file %s can't be opened" %(filename)
71 72
72 73 sts = basicHeaderObj.read(fp)
73 74 fp.close()
74 75
75 76 if not(sts):
76 77 print "Skipping the file %s because it has not a valid header" %(filename)
77 78 return 0
78 79
79 80 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
80 81 return 0
81 82
82 83 return 1
83 84
84 85 def isFileinThisTime(filename, startTime, endTime):
85 86 """
86 87 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
87 88
88 89 Inputs:
89 90 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
90 91
91 92 startTime : tiempo inicial del rango seleccionado en formato datetime.time
92 93
93 94 endTime : tiempo final del rango seleccionado en formato datetime.time
94 95
95 96 Return:
96 97 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
97 98 fecha especificado, de lo contrario retorna False.
98 99
99 100 Excepciones:
100 101 Si el archivo no existe o no puede ser abierto
101 102 Si la cabecera no puede ser leida.
102 103
103 104 """
104 105
105 106
106 107 try:
107 108 fp = open(filename,'rb')
108 109 except:
109 110 raise IOError, "The file %s can't be opened" %(filename)
110 111
111 112 basicHeaderObj = BasicHeader(LOCALTIME)
112 113 sts = basicHeaderObj.read(fp)
113 114 fp.close()
114 115
115 116 thisDatetime = basicHeaderObj.datatime
116 117 thisTime = basicHeaderObj.datatime.time()
117 118
118 119 if not(sts):
119 120 print "Skipping the file %s because it has not a valid header" %(filename)
120 121 return None
121 122
122 123 if not ((startTime <= thisTime) and (endTime > thisTime)):
123 124 return None
124 125
125 126 return thisDatetime
126 127
127 128 def getFileFromSet(path,ext,set):
128 129 validFilelist = []
129 130 fileList = os.listdir(path)
130 131
131 132 # 0 1234 567 89A BCDE
132 133 # H YYYY DDD SSS .ext
133 134
134 135 for file in fileList:
135 136 try:
136 137 year = int(file[1:5])
137 138 doy = int(file[5:8])
138 139
139 140
140 141 except:
141 142 continue
142 143
143 144 if (os.path.splitext(file)[-1].lower() != ext.lower()):
144 145 continue
145 146
146 147 validFilelist.append(file)
147 148
148 149 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
149 150
150 151 if len(myfile)!= 0:
151 152 return myfile[0]
152 153 else:
153 154 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
154 155 print 'the filename %s does not exist'%filename
155 156 print '...going to the last file: '
156 157
157 158 if validFilelist:
158 159 validFilelist = sorted( validFilelist, key=str.lower )
159 160 return validFilelist[-1]
160 161
161 162 return None
162 163
163 164
164 165 def getlastFileFromPath(path, ext):
165 166 """
166 167 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
167 168 al final de la depuracion devuelve el ultimo file de la lista que quedo.
168 169
169 170 Input:
170 171 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
171 172 ext : extension de los files contenidos en una carpeta
172 173
173 174 Return:
174 175 El ultimo file de una determinada carpeta, no se considera el path.
175 176 """
176 177 validFilelist = []
177 178 fileList = os.listdir(path)
178 179
179 180 # 0 1234 567 89A BCDE
180 181 # H YYYY DDD SSS .ext
181 182
182 183 for file in fileList:
183 184 try:
184 185 year = int(file[1:5])
185 186 doy = int(file[5:8])
186 187
187 188
188 189 except:
189 190 continue
190 191
191 192 if (os.path.splitext(file)[-1].lower() != ext.lower()):
192 193 continue
193 194
194 195 validFilelist.append(file)
195 196
196 197 if validFilelist:
197 198 validFilelist = sorted( validFilelist, key=str.lower )
198 199 return validFilelist[-1]
199 200
200 201 return None
201 202
202 203 def checkForRealPath(path, foldercounter, year, doy, set, ext):
203 204 """
204 205 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
205 206 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
206 207 el path exacto de un determinado file.
207 208
208 209 Example :
209 210 nombre correcto del file es .../.../D2009307/P2009307367.ext
210 211
211 212 Entonces la funcion prueba con las siguientes combinaciones
212 213 .../.../y2009307367.ext
213 214 .../.../Y2009307367.ext
214 215 .../.../x2009307/y2009307367.ext
215 216 .../.../x2009307/Y2009307367.ext
216 217 .../.../X2009307/y2009307367.ext
217 218 .../.../X2009307/Y2009307367.ext
218 219 siendo para este caso, la ultima combinacion de letras, identica al file buscado
219 220
220 221 Return:
221 222 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
222 223 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
223 224 para el filename
224 225 """
225 226 fullfilename = None
226 227 find_flag = False
227 228 filename = None
228 229
229 230 prefixDirList = [None,'d','D']
230 231 if ext.lower() == ".r": #voltage
231 232 prefixFileList = ['d','D']
232 233 elif ext.lower() == ".pdata": #spectra
233 234 prefixFileList = ['p','P']
234 235 else:
235 236 return None, filename
236 237
237 238 #barrido por las combinaciones posibles
238 239 for prefixDir in prefixDirList:
239 240 thispath = path
240 241 if prefixDir != None:
241 242 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
242 243 if foldercounter == 0:
243 244 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
244 245 else:
245 246 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
246 247 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
247 248 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
248 249 fullfilename = os.path.join( thispath, filename ) #formo el path completo
249 250
250 251 if os.path.exists( fullfilename ): #verifico que exista
251 252 find_flag = True
252 253 break
253 254 if find_flag:
254 255 break
255 256
256 257 if not(find_flag):
257 258 return None, filename
258 259
259 260 return fullfilename, filename
260 261
261 262 def isDoyFolder(folder):
262 263 try:
263 264 year = int(folder[1:5])
264 265 except:
265 266 return 0
266 267
267 268 try:
268 269 doy = int(folder[5:8])
269 270 except:
270 271 return 0
271 272
272 273 return 1
273 274
274 275 class JRODataIO:
275 276
276 277 c = 3E8
277 278
278 279 isConfig = False
279 280
280 281 basicHeaderObj = BasicHeader(LOCALTIME)
281 282
282 283 systemHeaderObj = SystemHeader()
283 284
284 285 radarControllerHeaderObj = RadarControllerHeader()
285 286
286 287 processingHeaderObj = ProcessingHeader()
287 288
288 289 online = 0
289 290
290 291 dtype = None
291 292
292 293 pathList = []
293 294
294 295 filenameList = []
295 296
296 297 filename = None
297 298
298 299 ext = None
299 300
300 301 flagIsNewFile = 1
301 302
302 303 flagTimeBlock = 0
303 304
304 305 flagIsNewBlock = 0
305 306
306 307 fp = None
307 308
308 309 firstHeaderSize = 0
309 310
310 311 basicHeaderSize = 24
311 312
312 313 versionFile = 1103
313 314
314 315 fileSize = None
315 316
316 317 ippSeconds = None
317 318
318 319 fileSizeByHeader = None
319 320
320 321 fileIndex = None
321 322
322 323 profileIndex = None
323 324
324 325 blockIndex = None
325 326
326 327 nTotalBlocks = None
327 328
328 329 maxTimeStep = 30
329 330
330 331 lastUTTime = None
331 332
332 333 datablock = None
333 334
334 335 dataOut = None
335 336
336 337 blocksize = None
337 338
338 339 def __init__(self):
339 340
340 341 raise ValueError, "Not implemented"
341 342
342 343 def run(self):
343 344
344 345 raise ValueError, "Not implemented"
345 346
346 347 def getOutput(self):
347 348
348 349 return self.dataOut
349 350
350 351 class JRODataReader(JRODataIO, ProcessingUnit):
351 352
352 353 nReadBlocks = 0
353 354
354 355 delay = 10 #number of seconds waiting a new file
355 356
356 357 nTries = 3 #quantity tries
357 358
358 359 nFiles = 3 #number of files for searching
359 360
360 361 path = None
361 362
362 363 foldercounter = 0
363 364
364 365 flagNoMoreFiles = 0
365 366
366 367 datetimeList = []
367 368
368 369 __isFirstTimeOnline = 1
369 370
370 371 __printInfo = True
371 372
372 373 profileIndex = None
373 374
374 375 def __init__(self):
375 376
376 377 """
377 378
378 379 """
379 380
380 381 raise ValueError, "This method has not been implemented"
381 382
382 383
383 384 def createObjByDefault(self):
384 385 """
385 386
386 387 """
387 388 raise ValueError, "This method has not been implemented"
388 389
389 390 def getBlockDimension(self):
390 391
391 392 raise ValueError, "No implemented"
392 393
393 394 def __searchFilesOffLine(self,
394 395 path,
395 396 startDate,
396 397 endDate,
397 398 startTime=datetime.time(0,0,0),
398 399 endTime=datetime.time(23,59,59),
399 400 set=None,
400 401 expLabel='',
401 402 ext='.r',
402 403 walk=True):
403 404
404 405 pathList = []
405 406
406 407 if not walk:
407 408 #pathList.append(path)
408 409 multi_path = path.split(',')
409 410 for single_path in multi_path:
410 411 pathList.append(single_path)
411 412
412 413 else:
413 414 #dirList = []
414 415 multi_path = path.split(',')
415 416 for single_path in multi_path:
416 417 dirList = []
417 418 for thisPath in os.listdir(single_path):
418 419 if not os.path.isdir(os.path.join(single_path,thisPath)):
419 420 continue
420 421 if not isDoyFolder(thisPath):
421 422 continue
422 423
423 424 dirList.append(thisPath)
424 425
425 426 if not(dirList):
426 427 return None, None
427 428
428 429 thisDate = startDate
429 430
430 431 while(thisDate <= endDate):
431 432 year = thisDate.timetuple().tm_year
432 433 doy = thisDate.timetuple().tm_yday
433 434
434 435 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
435 436 if len(matchlist) == 0:
436 437 thisDate += datetime.timedelta(1)
437 438 continue
438 439 for match in matchlist:
439 440 pathList.append(os.path.join(single_path,match,expLabel))
440 441
441 442 thisDate += datetime.timedelta(1)
442 443
443 444 if pathList == []:
444 445 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
445 446 return None, None
446 447
447 448 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
448 449
449 450 filenameList = []
450 451 datetimeList = []
451 452 pathDict = {}
452 453 filenameList_to_sort = []
453 454
454 455 for i in range(len(pathList)):
455 456
456 457 thisPath = pathList[i]
457 458
458 459 fileList = glob.glob1(thisPath, "*%s" %ext)
459 460 fileList.sort()
460 461 pathDict.setdefault(fileList[0])
461 462 pathDict[fileList[0]] = i
462 463 filenameList_to_sort.append(fileList[0])
463 464
464 465 filenameList_to_sort.sort()
465 466
466 467 for file in filenameList_to_sort:
467 468 thisPath = pathList[pathDict[file]]
468 469
469 470 fileList = glob.glob1(thisPath, "*%s" %ext)
470 471 fileList.sort()
471 472
472 473 for file in fileList:
473 474
474 475 filename = os.path.join(thisPath,file)
475 476 thisDatetime = isFileinThisTime(filename, startTime, endTime)
476 477
477 478 if not(thisDatetime):
478 479 continue
479 480
480 481 filenameList.append(filename)
481 482 datetimeList.append(thisDatetime)
482 483
483 484 if not(filenameList):
484 485 print "Any file was found for the time range %s - %s" %(startTime, endTime)
485 486 return None, None
486 487
487 488 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
488 489 print
489 490
490 491 for i in range(len(filenameList)):
491 492 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
492 493
493 494 self.filenameList = filenameList
494 495 self.datetimeList = datetimeList
495 496
496 497 return pathList, filenameList
497 498
498 499 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
499 500
500 501 """
501 502 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
502 503 devuelve el archivo encontrado ademas de otros datos.
503 504
504 505 Input:
505 506 path : carpeta donde estan contenidos los files que contiene data
506 507
507 508 expLabel : Nombre del subexperimento (subfolder)
508 509
509 510 ext : extension de los files
510 511
511 512 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
512 513
513 514 Return:
514 515 directory : eL directorio donde esta el file encontrado
515 516 filename : el ultimo file de una determinada carpeta
516 517 year : el anho
517 518 doy : el numero de dia del anho
518 519 set : el set del archivo
519 520
520 521
521 522 """
522 523 dirList = []
523 524
524 525 if not walk:
525 526 fullpath = path
526 527 foldercounter = 0
527 528 else:
528 529 #Filtra solo los directorios
529 530 for thisPath in os.listdir(path):
530 531 if not os.path.isdir(os.path.join(path,thisPath)):
531 532 continue
532 533 if not isDoyFolder(thisPath):
533 534 continue
534 535
535 536 dirList.append(thisPath)
536 537
537 538 if not(dirList):
538 539 return None, None, None, None, None, None
539 540
540 541 dirList = sorted( dirList, key=str.lower )
541 542
542 543 doypath = dirList[-1]
543 544 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
544 545 fullpath = os.path.join(path, doypath, expLabel)
545 546
546 547
547 548 print "%s folder was found: " %(fullpath )
548 549
549 550 if set == None:
550 551 filename = getlastFileFromPath(fullpath, ext)
551 552 else:
552 553 filename = getFileFromSet(fullpath, ext, set)
553 554
554 555 if not(filename):
555 556 return None, None, None, None, None, None
556 557
557 558 print "%s file was found" %(filename)
558 559
559 560 if not(self.__verifyFile(os.path.join(fullpath, filename))):
560 561 return None, None, None, None, None, None
561 562
562 563 year = int( filename[1:5] )
563 564 doy = int( filename[5:8] )
564 565 set = int( filename[8:11] )
565 566
566 567 return fullpath, foldercounter, filename, year, doy, set
567 568
568 569 def __setNextFileOffline(self):
569 570
570 571 idFile = self.fileIndex
571 572
572 573 while (True):
573 574 idFile += 1
574 575 if not(idFile < len(self.filenameList)):
575 576 self.flagNoMoreFiles = 1
576 577 print "No more Files"
577 578 return 0
578 579
579 580 filename = self.filenameList[idFile]
580 581
581 582 if not(self.__verifyFile(filename)):
582 583 continue
583 584
584 585 fileSize = os.path.getsize(filename)
585 586 fp = open(filename,'rb')
586 587 break
587 588
588 589 self.flagIsNewFile = 1
589 590 self.fileIndex = idFile
590 591 self.filename = filename
591 592 self.fileSize = fileSize
592 593 self.fp = fp
593 594
594 595 print "Setting the file: %s"%self.filename
595 596
596 597 return 1
597 598
598 599 def __setNextFileOnline(self):
599 600 """
600 601 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
601 602 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
602 603 siguientes.
603 604
604 605 Affected:
605 606 self.flagIsNewFile
606 607 self.filename
607 608 self.fileSize
608 609 self.fp
609 610 self.set
610 611 self.flagNoMoreFiles
611 612
612 613 Return:
613 614 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
614 615 1 : si el file fue abierto con exito y esta listo a ser leido
615 616
616 617 Excepciones:
617 618 Si un determinado file no puede ser abierto
618 619 """
619 620 nFiles = 0
620 621 fileOk_flag = False
621 622 firstTime_flag = True
622 623
623 624 self.set += 1
624 625
625 626 if self.set > 999:
626 627 self.set = 0
627 628 self.foldercounter += 1
628 629
629 630 #busca el 1er file disponible
630 631 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
631 632 if fullfilename:
632 633 if self.__verifyFile(fullfilename, False):
633 634 fileOk_flag = True
634 635
635 636 #si no encuentra un file entonces espera y vuelve a buscar
636 637 if not(fileOk_flag):
637 638 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
638 639
639 640 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
640 641 tries = self.nTries
641 642 else:
642 643 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
643 644
644 645 for nTries in range( tries ):
645 646 if firstTime_flag:
646 647 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
647 648 time.sleep( self.delay )
648 649 else:
649 650 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
650 651
651 652 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
652 653 if fullfilename:
653 654 if self.__verifyFile(fullfilename):
654 655 fileOk_flag = True
655 656 break
656 657
657 658 if fileOk_flag:
658 659 break
659 660
660 661 firstTime_flag = False
661 662
662 663 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
663 664 self.set += 1
664 665
665 666 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
666 667 self.set = 0
667 668 self.doy += 1
668 669 self.foldercounter = 0
669 670
670 671 if fileOk_flag:
671 672 self.fileSize = os.path.getsize( fullfilename )
672 673 self.filename = fullfilename
673 674 self.flagIsNewFile = 1
674 675 if self.fp != None: self.fp.close()
675 676 self.fp = open(fullfilename, 'rb')
676 677 self.flagNoMoreFiles = 0
677 678 print 'Setting the file: %s' % fullfilename
678 679 else:
679 680 self.fileSize = 0
680 681 self.filename = None
681 682 self.flagIsNewFile = 0
682 683 self.fp = None
683 684 self.flagNoMoreFiles = 1
684 685 print 'No more Files'
685 686
686 687 return fileOk_flag
687 688
688 689
689 690 def setNextFile(self):
690 691 if self.fp != None:
691 692 self.fp.close()
692 693
693 694 if self.online:
694 695 newFile = self.__setNextFileOnline()
695 696 else:
696 697 newFile = self.__setNextFileOffline()
697 698
698 699 if not(newFile):
699 700 return 0
700 701
701 702 self.__readFirstHeader()
702 703 self.nReadBlocks = 0
703 704 return 1
704 705
705 706 def __waitNewBlock(self):
706 707 """
707 708 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
708 709
709 710 Si el modo de lectura es OffLine siempre retorn 0
710 711 """
711 712 if not self.online:
712 713 return 0
713 714
714 715 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
715 716 return 0
716 717
717 718 currentPointer = self.fp.tell()
718 719
719 720 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
720 721
721 722 for nTries in range( self.nTries ):
722 723
723 724 self.fp.close()
724 725 self.fp = open( self.filename, 'rb' )
725 726 self.fp.seek( currentPointer )
726 727
727 728 self.fileSize = os.path.getsize( self.filename )
728 729 currentSize = self.fileSize - currentPointer
729 730
730 731 if ( currentSize >= neededSize ):
731 732 self.__rdBasicHeader()
732 733 return 1
733 734
734 735 if self.fileSize == self.fileSizeByHeader:
735 736 # self.flagEoF = True
736 737 return 0
737 738
738 739 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
739 740 time.sleep( self.delay )
740 741
741 742
742 743 return 0
743 744
744 745 def waitDataBlock(self,pointer_location):
745 746
746 747 currentPointer = pointer_location
747 748
748 749 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
749 750
750 751 for nTries in range( self.nTries ):
751 752 self.fp.close()
752 753 self.fp = open( self.filename, 'rb' )
753 754 self.fp.seek( currentPointer )
754 755
755 756 self.fileSize = os.path.getsize( self.filename )
756 757 currentSize = self.fileSize - currentPointer
757 758
758 759 if ( currentSize >= neededSize ):
759 760 return 1
760 761
761 762 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
762 763 time.sleep( self.delay )
763 764
764 765 return 0
765 766
766 767
767 768 def __jumpToLastBlock(self):
768 769
769 770 if not(self.__isFirstTimeOnline):
770 771 return
771 772
772 773 csize = self.fileSize - self.fp.tell()
773 774 blocksize = self.processingHeaderObj.blockSize
774 775
775 776 #salta el primer bloque de datos
776 777 if csize > self.processingHeaderObj.blockSize:
777 778 self.fp.seek(self.fp.tell() + blocksize)
778 779 else:
779 780 return
780 781
781 782 csize = self.fileSize - self.fp.tell()
782 783 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
783 784 while True:
784 785
785 786 if self.fp.tell()<self.fileSize:
786 787 self.fp.seek(self.fp.tell() + neededsize)
787 788 else:
788 789 self.fp.seek(self.fp.tell() - neededsize)
789 790 break
790 791
791 792 # csize = self.fileSize - self.fp.tell()
792 793 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
793 794 # factor = int(csize/neededsize)
794 795 # if factor > 0:
795 796 # self.fp.seek(self.fp.tell() + factor*neededsize)
796 797
797 798 self.flagIsNewFile = 0
798 799 self.__isFirstTimeOnline = 0
799 800
800 801
801 802 def __setNewBlock(self):
802 803
803 804 if self.fp == None:
804 805 return 0
805 806
806 807 if self.online:
807 808 self.__jumpToLastBlock()
808 809
809 810 if self.flagIsNewFile:
810 811 return 1
811 812
812 813 self.lastUTTime = self.basicHeaderObj.utc
813 814 currentSize = self.fileSize - self.fp.tell()
814 815 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
815 816
816 817 if (currentSize >= neededSize):
817 818 self.__rdBasicHeader()
818 819 return 1
819 820
820 821 if self.__waitNewBlock():
821 822 return 1
822 823
823 824 if not(self.setNextFile()):
824 825 return 0
825 826
826 827 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
827 828
828 829 self.flagTimeBlock = 0
829 830
830 831 if deltaTime > self.maxTimeStep:
831 832 self.flagTimeBlock = 1
832 833
833 834 return 1
834 835
835 836
836 837 def readNextBlock(self):
837 838 if not(self.__setNewBlock()):
838 839 return 0
839 840
840 841 if not(self.readBlock()):
841 842 return 0
842 843
843 844 return 1
844 845
845 846 def __rdProcessingHeader(self, fp=None):
846 847 if fp == None:
847 848 fp = self.fp
848 849
849 850 self.processingHeaderObj.read(fp)
850 851
851 852 def __rdRadarControllerHeader(self, fp=None):
852 853 if fp == None:
853 854 fp = self.fp
854 855
855 856 self.radarControllerHeaderObj.read(fp)
856 857
857 858 def __rdSystemHeader(self, fp=None):
858 859 if fp == None:
859 860 fp = self.fp
860 861
861 862 self.systemHeaderObj.read(fp)
862 863
863 864 def __rdBasicHeader(self, fp=None):
864 865 if fp == None:
865 866 fp = self.fp
866 867
867 868 self.basicHeaderObj.read(fp)
868 869
869 870
870 871 def __readFirstHeader(self):
871 872 self.__rdBasicHeader()
872 873 self.__rdSystemHeader()
873 874 self.__rdRadarControllerHeader()
874 875 self.__rdProcessingHeader()
875 876
876 877 self.firstHeaderSize = self.basicHeaderObj.size
877 878
878 879 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
879 880 if datatype == 0:
880 881 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
881 882 elif datatype == 1:
882 883 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
883 884 elif datatype == 2:
884 885 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
885 886 elif datatype == 3:
886 887 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
887 888 elif datatype == 4:
888 889 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
889 890 elif datatype == 5:
890 891 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
891 892 else:
892 893 raise ValueError, 'Data type was not defined'
893 894
894 895 self.dtype = datatype_str
895 896 self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
896 897 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
897 898 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
898 899 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
899 900 self.getBlockDimension()
900 901
901 902
902 903 def __verifyFile(self, filename, msgFlag=True):
903 904 msg = None
904 905 try:
905 906 fp = open(filename, 'rb')
906 907 currentPosition = fp.tell()
907 908 except:
908 909 if msgFlag:
909 910 print "The file %s can't be opened" % (filename)
910 911 return False
911 912
912 913 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
913 914
914 915 if neededSize == 0:
915 916 basicHeaderObj = BasicHeader(LOCALTIME)
916 917 systemHeaderObj = SystemHeader()
917 918 radarControllerHeaderObj = RadarControllerHeader()
918 919 processingHeaderObj = ProcessingHeader()
919 920
920 921 try:
921 922 if not( basicHeaderObj.read(fp) ): raise IOError
922 923 if not( systemHeaderObj.read(fp) ): raise IOError
923 924 if not( radarControllerHeaderObj.read(fp) ): raise IOError
924 925 if not( processingHeaderObj.read(fp) ): raise IOError
925 926 data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
926 927
927 928 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
928 929
929 930 except:
930 931 if msgFlag:
931 932 print "\tThe file %s is empty or it hasn't enough data" % filename
932 933
933 934 fp.close()
934 935 return False
935 936 else:
936 937 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
937 938
938 939 fp.close()
939 940 fileSize = os.path.getsize(filename)
940 941 currentSize = fileSize - currentPosition
941 942 if currentSize < neededSize:
942 943 if msgFlag and (msg != None):
943 944 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
944 945 return False
945 946
946 947 return True
947 948
948 949 def setup(self,
949 950 path=None,
950 951 startDate=None,
951 952 endDate=None,
952 953 startTime=datetime.time(0,0,0),
953 954 endTime=datetime.time(23,59,59),
954 955 set=None,
955 956 expLabel = "",
956 957 ext = None,
957 958 online = False,
958 959 delay = 60,
959 960 walk = True):
960 961
961 962 if path == None:
962 963 raise ValueError, "The path is not valid"
963 964
964 965 if ext == None:
965 966 ext = self.ext
966 967
967 968 if online:
968 969 print "Searching files in online mode..."
969 970
970 971 for nTries in range( self.nTries ):
971 972 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
972 973
973 974 if fullpath:
974 975 break
975 976
976 977 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
977 978 time.sleep( self.delay )
978 979
979 980 if not(fullpath):
980 981 print "There 'isn't valied files in %s" % path
981 982 return None
982 983
983 984 self.year = year
984 985 self.doy = doy
985 986 self.set = set - 1
986 987 self.path = path
987 988 self.foldercounter = foldercounter
988 989 last_set = None
989 990
990 991 else:
991 992 print "Searching files in offline mode ..."
992 993 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
993 994 startTime=startTime, endTime=endTime,
994 995 set=set, expLabel=expLabel, ext=ext,
995 996 walk=walk)
996 997
997 998 if not(pathList):
998 999 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
999 1000 datetime.datetime.combine(startDate,startTime).ctime(),
1000 1001 datetime.datetime.combine(endDate,endTime).ctime())
1001 1002
1002 1003 sys.exit(-1)
1003 1004
1004 1005
1005 1006 self.fileIndex = -1
1006 1007 self.pathList = pathList
1007 1008 self.filenameList = filenameList
1008 1009 file_name = os.path.basename(filenameList[-1])
1009 1010 basename, ext = os.path.splitext(file_name)
1010 1011 last_set = int(basename[-3:])
1011 1012
1012 1013 self.online = online
1013 1014 self.delay = delay
1014 1015 ext = ext.lower()
1015 1016 self.ext = ext
1016 1017
1017 1018 if not(self.setNextFile()):
1018 1019 if (startDate!=None) and (endDate!=None):
1019 1020 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1020 1021 elif startDate != None:
1021 1022 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1022 1023 else:
1023 1024 print "No files"
1024 1025
1025 1026 sys.exit(-1)
1026 1027
1027 1028 # self.updateDataHeader()
1028 1029 if last_set != None:
1029 1030 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1030 1031 return self.dataOut
1031 1032
1032 1033 def getBasicHeader(self):
1033 1034
1034 1035 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.ippSeconds
1035 1036
1036 1037 self.dataOut.flagTimeBlock = self.flagTimeBlock
1037 1038
1038 1039 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1039 1040
1040 1041 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1041 1042
1042 1043 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1043 1044
1044 1045 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1045 1046
1046 1047 def getFirstHeader(self):
1047 1048
1048 1049 raise ValueError, "This method has not been implemented"
1049 1050
1050 1051 def getData():
1051 1052
1052 1053 raise ValueError, "This method has not been implemented"
1053 1054
1054 1055 def hasNotDataInBuffer():
1055 1056
1056 1057 raise ValueError, "This method has not been implemented"
1057 1058
1058 1059 def readBlock():
1059 1060
1060 1061 raise ValueError, "This method has not been implemented"
1061 1062
1062 1063 def isEndProcess(self):
1063 1064
1064 1065 return self.flagNoMoreFiles
1065 1066
1066 1067 def printReadBlocks(self):
1067 1068
1068 1069 print "Number of read blocks per file %04d" %self.nReadBlocks
1069 1070
1070 1071 def printTotalBlocks(self):
1071 1072
1072 1073 print "Number of read blocks %04d" %self.nTotalBlocks
1073 1074
1074 1075 def printNumberOfBlock(self):
1075 1076
1076 1077 if self.flagIsNewBlock:
1077 1078 print "Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1078 1079 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1079 1080 def printInfo(self):
1080 1081
1081 1082 if self.__printInfo == False:
1082 1083 return
1083 1084
1084 1085 self.basicHeaderObj.printInfo()
1085 1086 self.systemHeaderObj.printInfo()
1086 1087 self.radarControllerHeaderObj.printInfo()
1087 1088 self.processingHeaderObj.printInfo()
1088 1089
1089 1090 self.__printInfo = False
1090 1091
1091 1092
1092 1093 def run(self, **kwargs):
1093 1094
1094 1095 if not(self.isConfig):
1095 1096
1096 1097 # self.dataOut = dataOut
1097 1098 self.setup(**kwargs)
1098 1099 self.isConfig = True
1099 1100
1100 1101 self.getData()
1101 1102
1102 1103 class JRODataWriter(JRODataIO, Operation):
1103 1104
1104 1105 """
1105 1106 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1106 1107 de los datos siempre se realiza por bloques.
1107 1108 """
1108 1109
1109 1110 blockIndex = 0
1110 1111
1111 1112 path = None
1112 1113
1113 1114 setFile = None
1114 1115
1115 1116 profilesPerBlock = None
1116 1117
1117 1118 blocksPerFile = None
1118 1119
1119 1120 nWriteBlocks = 0
1120 1121
1121 1122 def __init__(self, dataOut=None):
1122 1123 raise ValueError, "Not implemented"
1123 1124
1124 1125
1125 1126 def hasAllDataInBuffer(self):
1126 1127 raise ValueError, "Not implemented"
1127 1128
1128 1129
1129 1130 def setBlockDimension(self):
1130 1131 raise ValueError, "Not implemented"
1131 1132
1132 1133
1133 1134 def writeBlock(self):
1134 1135 raise ValueError, "No implemented"
1135 1136
1136 1137
1137 1138 def putData(self):
1138 1139 raise ValueError, "No implemented"
1139 1140
1140 1141
1141 1142 def setBasicHeader(self):
1142 1143
1143 1144 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1144 1145 self.basicHeaderObj.version = self.versionFile
1145 1146 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1146 1147
1147 1148 utc = numpy.floor(self.dataOut.utctime)
1148 1149 milisecond = (self.dataOut.utctime - utc)* 1000.0
1149 1150
1150 1151 self.basicHeaderObj.utc = utc
1151 1152 self.basicHeaderObj.miliSecond = milisecond
1152 1153 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1153 1154 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1154 1155 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1155 1156
1156 1157 def setFirstHeader(self):
1157 1158 """
1158 1159 Obtiene una copia del First Header
1159 1160
1160 1161 Affected:
1161 1162
1162 1163 self.basicHeaderObj
1163 1164 self.systemHeaderObj
1164 1165 self.radarControllerHeaderObj
1165 1166 self.processingHeaderObj self.
1166 1167
1167 1168 Return:
1168 1169 None
1169 1170 """
1170 1171
1171 1172 raise ValueError, "No implemented"
1172 1173
1173 1174 def __writeFirstHeader(self):
1174 1175 """
1175 1176 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1176 1177
1177 1178 Affected:
1178 1179 __dataType
1179 1180
1180 1181 Return:
1181 1182 None
1182 1183 """
1183 1184
1184 1185 # CALCULAR PARAMETROS
1185 1186
1186 1187 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1187 1188 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1188 1189
1189 1190 self.basicHeaderObj.write(self.fp)
1190 1191 self.systemHeaderObj.write(self.fp)
1191 1192 self.radarControllerHeaderObj.write(self.fp)
1192 1193 self.processingHeaderObj.write(self.fp)
1193 1194
1194 1195 self.dtype = self.dataOut.dtype
1195 1196
1196 1197 def __setNewBlock(self):
1197 1198 """
1198 1199 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1199 1200
1200 1201 Return:
1201 1202 0 : si no pudo escribir nada
1202 1203 1 : Si escribio el Basic el First Header
1203 1204 """
1204 1205 if self.fp == None:
1205 1206 self.setNextFile()
1206 1207
1207 1208 if self.flagIsNewFile:
1208 1209 return 1
1209 1210
1210 1211 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1211 1212 self.basicHeaderObj.write(self.fp)
1212 1213 return 1
1213 1214
1214 1215 if not( self.setNextFile() ):
1215 1216 return 0
1216 1217
1217 1218 return 1
1218 1219
1219 1220
1220 1221 def writeNextBlock(self):
1221 1222 """
1222 1223 Selecciona el bloque siguiente de datos y los escribe en un file
1223 1224
1224 1225 Return:
1225 1226 0 : Si no hizo pudo escribir el bloque de datos
1226 1227 1 : Si no pudo escribir el bloque de datos
1227 1228 """
1228 1229 if not( self.__setNewBlock() ):
1229 1230 return 0
1230 1231
1231 1232 self.writeBlock()
1232 1233
1233 1234 return 1
1234 1235
1235 1236 def setNextFile(self):
1236 1237 """
1237 1238 Determina el siguiente file que sera escrito
1238 1239
1239 1240 Affected:
1240 1241 self.filename
1241 1242 self.subfolder
1242 1243 self.fp
1243 1244 self.setFile
1244 1245 self.flagIsNewFile
1245 1246
1246 1247 Return:
1247 1248 0 : Si el archivo no puede ser escrito
1248 1249 1 : Si el archivo esta listo para ser escrito
1249 1250 """
1250 1251 ext = self.ext
1251 1252 path = self.path
1252 1253
1253 1254 if self.fp != None:
1254 1255 self.fp.close()
1255 1256
1256 1257 timeTuple = time.localtime( self.dataOut.utctime)
1257 1258 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1258 1259
1259 1260 fullpath = os.path.join( path, subfolder )
1260 1261 if not( os.path.exists(fullpath) ):
1261 1262 os.mkdir(fullpath)
1262 1263 self.setFile = -1 #inicializo mi contador de seteo
1263 1264 else:
1264 1265 filesList = os.listdir( fullpath )
1265 1266 if len( filesList ) > 0:
1266 1267 filesList = sorted( filesList, key=str.lower )
1267 1268 filen = filesList[-1]
1268 1269 # el filename debera tener el siguiente formato
1269 1270 # 0 1234 567 89A BCDE (hex)
1270 1271 # x YYYY DDD SSS .ext
1271 1272 if isNumber( filen[8:11] ):
1272 1273 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1273 1274 else:
1274 1275 self.setFile = -1
1275 1276 else:
1276 1277 self.setFile = -1 #inicializo mi contador de seteo
1277 1278
1278 1279 setFile = self.setFile
1279 1280 setFile += 1
1280 1281
1281 1282 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1282 1283 timeTuple.tm_year,
1283 1284 timeTuple.tm_yday,
1284 1285 setFile,
1285 1286 ext )
1286 1287
1287 1288 filename = os.path.join( path, subfolder, file )
1288 1289
1289 1290 fp = open( filename,'wb' )
1290 1291
1291 1292 self.blockIndex = 0
1292 1293
1293 1294 #guardando atributos
1294 1295 self.filename = filename
1295 1296 self.subfolder = subfolder
1296 1297 self.fp = fp
1297 1298 self.setFile = setFile
1298 1299 self.flagIsNewFile = 1
1299 1300
1300 1301 self.setFirstHeader()
1301 1302
1302 1303 print 'Writing the file: %s'%self.filename
1303 1304
1304 1305 self.__writeFirstHeader()
1305 1306
1306 1307 return 1
1307 1308
1308 1309 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1309 1310 """
1310 1311 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1311 1312
1312 1313 Inputs:
1313 1314 path : el path destino en el cual se escribiran los files a crear
1314 1315 format : formato en el cual sera salvado un file
1315 1316 set : el setebo del file
1316 1317
1317 1318 Return:
1318 1319 0 : Si no realizo un buen seteo
1319 1320 1 : Si realizo un buen seteo
1320 1321 """
1321 1322
1322 1323 if ext == None:
1323 1324 ext = self.ext
1324 1325
1325 1326 ext = ext.lower()
1326 1327
1327 1328 self.ext = ext
1328 1329
1329 1330 self.path = path
1330 1331
1331 1332 self.setFile = set - 1
1332 1333
1333 1334 self.blocksPerFile = blocksPerFile
1334 1335
1335 1336 self.profilesPerBlock = profilesPerBlock
1336 1337
1337 1338 self.dataOut = dataOut
1338 1339
1339 1340 if not(self.setNextFile()):
1340 1341 print "There isn't a next file"
1341 1342 return 0
1342 1343
1343 1344 self.setBlockDimension()
1344 1345
1345 1346 return 1
1346 1347
1347 1348 def run(self, dataOut, **kwargs):
1348 1349
1349 1350 if not(self.isConfig):
1350 1351
1351 1352 self.setup(dataOut, **kwargs)
1352 1353 self.isConfig = True
1353 1354
1354 1355 self.putData()
1355 1356
1356 1357 class VoltageReader(JRODataReader):
1357 1358 """
1358 1359 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
1359 1360 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
1360 1361 perfiles*alturas*canales) son almacenados en la variable "buffer".
1361 1362
1362 1363 perfiles * alturas * canales
1363 1364
1364 1365 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1365 1366 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
1366 1367 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
1367 1368 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1368 1369
1369 1370 Example:
1370 1371
1371 1372 dpath = "/home/myuser/data"
1372 1373
1373 1374 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1374 1375
1375 1376 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1376 1377
1377 1378 readerObj = VoltageReader()
1378 1379
1379 1380 readerObj.setup(dpath, startTime, endTime)
1380 1381
1381 1382 while(True):
1382 1383
1383 1384 #to get one profile
1384 1385 profile = readerObj.getData()
1385 1386
1386 1387 #print the profile
1387 1388 print profile
1388 1389
1389 1390 #If you want to see all datablock
1390 1391 print readerObj.datablock
1391 1392
1392 1393 if readerObj.flagNoMoreFiles:
1393 1394 break
1394 1395
1395 1396 """
1396 1397
1397 1398 ext = ".r"
1398 1399
1399 1400 optchar = "D"
1400 1401 dataOut = None
1401 1402
1402 1403
1403 1404 def __init__(self):
1404 1405 """
1405 1406 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
1406 1407
1407 1408 Input:
1408 1409 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
1409 1410 almacenar un perfil de datos cada vez que se haga un requerimiento
1410 1411 (getData). El perfil sera obtenido a partir del buffer de datos,
1411 1412 si el buffer esta vacio se hara un nuevo proceso de lectura de un
1412 1413 bloque de datos.
1413 1414 Si este parametro no es pasado se creara uno internamente.
1414 1415
1415 1416 Variables afectadas:
1416 1417 self.dataOut
1417 1418
1418 1419 Return:
1419 1420 None
1420 1421 """
1421 1422
1422 1423 self.isConfig = False
1423 1424
1424 1425 self.datablock = None
1425 1426
1426 1427 self.utc = 0
1427 1428
1428 1429 self.ext = ".r"
1429 1430
1430 1431 self.optchar = "D"
1431 1432
1432 1433 self.basicHeaderObj = BasicHeader(LOCALTIME)
1433 1434
1434 1435 self.systemHeaderObj = SystemHeader()
1435 1436
1436 1437 self.radarControllerHeaderObj = RadarControllerHeader()
1437 1438
1438 1439 self.processingHeaderObj = ProcessingHeader()
1439 1440
1440 1441 self.online = 0
1441 1442
1442 1443 self.fp = None
1443 1444
1444 1445 self.idFile = None
1445 1446
1446 1447 self.dtype = None
1447 1448
1448 1449 self.fileSizeByHeader = None
1449 1450
1450 1451 self.filenameList = []
1451 1452
1452 1453 self.filename = None
1453 1454
1454 1455 self.fileSize = None
1455 1456
1456 1457 self.firstHeaderSize = 0
1457 1458
1458 1459 self.basicHeaderSize = 24
1459 1460
1460 1461 self.pathList = []
1461 1462
1462 1463 self.filenameList = []
1463 1464
1464 1465 self.lastUTTime = 0
1465 1466
1466 1467 self.maxTimeStep = 30
1467 1468
1468 1469 self.flagNoMoreFiles = 0
1469 1470
1470 1471 self.set = 0
1471 1472
1472 1473 self.path = None
1473 1474
1474 1475 self.profileIndex = 2**32-1
1475 1476
1476 1477 self.delay = 3 #seconds
1477 1478
1478 1479 self.nTries = 3 #quantity tries
1479 1480
1480 1481 self.nFiles = 3 #number of files for searching
1481 1482
1482 1483 self.nReadBlocks = 0
1483 1484
1484 1485 self.flagIsNewFile = 1
1485 1486
1486 1487 self.__isFirstTimeOnline = 1
1487 1488
1488 1489 self.ippSeconds = 0
1489 1490
1490 1491 self.flagTimeBlock = 0
1491 1492
1492 1493 self.flagIsNewBlock = 0
1493 1494
1494 1495 self.nTotalBlocks = 0
1495 1496
1496 1497 self.blocksize = 0
1497 1498
1498 1499 self.dataOut = self.createObjByDefault()
1499 1500
1500 1501 def createObjByDefault(self):
1501 1502
1502 1503 dataObj = Voltage()
1503 1504
1504 1505 return dataObj
1505 1506
1506 1507 def __hasNotDataInBuffer(self):
1507 1508 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1508 1509 return 1
1509 1510 return 0
1510 1511
1511 1512
1512 1513 def getBlockDimension(self):
1513 1514 """
1514 1515 Obtiene la cantidad de puntos a leer por cada bloque de datos
1515 1516
1516 1517 Affected:
1517 1518 self.blocksize
1518 1519
1519 1520 Return:
1520 1521 None
1521 1522 """
1522 1523 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
1523 1524 self.blocksize = pts2read
1524 1525
1525 1526
1526 1527 def readBlock(self):
1527 1528 """
1528 1529 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
1529 1530 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
1530 1531 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
1531 1532 es seteado a 0
1532 1533
1533 1534 Inputs:
1534 1535 None
1535 1536
1536 1537 Return:
1537 1538 None
1538 1539
1539 1540 Affected:
1540 1541 self.profileIndex
1541 1542 self.datablock
1542 1543 self.flagIsNewFile
1543 1544 self.flagIsNewBlock
1544 1545 self.nTotalBlocks
1545 1546
1546 1547 Exceptions:
1547 1548 Si un bloque leido no es un bloque valido
1548 1549 """
1549 1550 current_pointer_location = self.fp.tell()
1550 1551 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
1551 1552
1552 1553 try:
1553 1554 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
1554 1555 except:
1555 1556 #print "The read block (%3d) has not enough data" %self.nReadBlocks
1556 1557
1557 1558 if self.waitDataBlock(pointer_location=current_pointer_location):
1558 1559 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
1559 1560 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
1560 1561 # return 0
1561 1562
1562 1563 junk = numpy.transpose(junk, (2,0,1))
1563 1564 self.datablock = junk['real'] + junk['imag']*1j
1564 1565
1565 1566 self.profileIndex = 0
1566 1567
1567 1568 self.flagIsNewFile = 0
1568 1569 self.flagIsNewBlock = 1
1569 1570
1570 1571 self.nTotalBlocks += 1
1571 1572 self.nReadBlocks += 1
1572 1573
1573 1574 return 1
1574 1575
1575 1576 def getFirstHeader(self):
1576 1577
1577 1578 self.dataOut.dtype = self.dtype
1578 1579
1579 1580 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
1580 1581
1581 1582 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
1582 1583
1583 1584 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
1584 1585
1585 1586 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
1586 1587
1587 1588 self.dataOut.ippSeconds = self.ippSeconds
1588 1589
1589 1590 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt
1590 1591
1591 1592 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
1592 1593
1593 1594 self.dataOut.flagShiftFFT = False
1594 1595
1595 1596 if self.radarControllerHeaderObj.code != None:
1596 1597
1597 1598 self.dataOut.nCode = self.radarControllerHeaderObj.nCode
1598 1599
1599 1600 self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
1600 1601
1601 1602 self.dataOut.code = self.radarControllerHeaderObj.code
1602 1603
1603 1604 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
1604 1605
1605 1606 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
1606 1607
1607 1608 self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
1608 1609
1609 1610 self.dataOut.flagDeflipData = False #asumo q la data no esta sin flip
1610 1611
1611 1612 self.dataOut.flagShiftFFT = False
1612 1613
1613 1614 def getData(self):
1614 1615 """
1615 1616 getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
1616 1617 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
1617 1618 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
1618 1619
1619 1620 Ademas incrementa el contador del buffer en 1.
1620 1621
1621 1622 Return:
1622 1623 data : retorna un perfil de voltages (alturas * canales) copiados desde el
1623 1624 buffer. Si no hay mas archivos a leer retorna None.
1624 1625
1625 1626 Variables afectadas:
1626 1627 self.dataOut
1627 1628 self.profileIndex
1628 1629
1629 1630 Affected:
1630 1631 self.dataOut
1631 1632 self.profileIndex
1632 1633 self.flagTimeBlock
1633 1634 self.flagIsNewBlock
1634 1635 """
1635 1636
1636 1637 if self.flagNoMoreFiles:
1637 1638 self.dataOut.flagNoData = True
1638 1639 print 'Process finished'
1639 1640 return 0
1640 1641
1641 1642 self.flagTimeBlock = 0
1642 1643 self.flagIsNewBlock = 0
1643 1644
1644 1645 if self.__hasNotDataInBuffer():
1645 1646
1646 1647 if not( self.readNextBlock() ):
1647 1648 return 0
1648 1649
1649 1650 self.getFirstHeader()
1650 1651
1651 1652 if self.datablock == None:
1652 1653 self.dataOut.flagNoData = True
1653 1654 return 0
1654 1655
1655 1656 self.dataOut.data = self.datablock[:,self.profileIndex,:]
1656 1657
1657 1658 self.dataOut.flagNoData = False
1658 1659
1659 1660 self.getBasicHeader()
1660 1661
1661 1662 self.profileIndex += 1
1662 1663
1663 1664 self.dataOut.realtime = self.online
1664 1665
1665 1666 return self.dataOut.data
1666 1667
1667 1668
1668 1669 class VoltageWriter(JRODataWriter):
1669 1670 """
1670 1671 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
1671 1672 de los datos siempre se realiza por bloques.
1672 1673 """
1673 1674
1674 1675 ext = ".r"
1675 1676
1676 1677 optchar = "D"
1677 1678
1678 1679 shapeBuffer = None
1679 1680
1680 1681
1681 1682 def __init__(self):
1682 1683 """
1683 1684 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
1684 1685
1685 1686 Affected:
1686 1687 self.dataOut
1687 1688
1688 1689 Return: None
1689 1690 """
1690 1691
1691 1692 self.nTotalBlocks = 0
1692 1693
1693 1694 self.profileIndex = 0
1694 1695
1695 1696 self.isConfig = False
1696 1697
1697 1698 self.fp = None
1698 1699
1699 1700 self.flagIsNewFile = 1
1700 1701
1701 1702 self.nTotalBlocks = 0
1702 1703
1703 1704 self.flagIsNewBlock = 0
1704 1705
1705 1706 self.setFile = None
1706 1707
1707 1708 self.dtype = None
1708 1709
1709 1710 self.path = None
1710 1711
1711 1712 self.filename = None
1712 1713
1713 1714 self.basicHeaderObj = BasicHeader(LOCALTIME)
1714 1715
1715 1716 self.systemHeaderObj = SystemHeader()
1716 1717
1717 1718 self.radarControllerHeaderObj = RadarControllerHeader()
1718 1719
1719 1720 self.processingHeaderObj = ProcessingHeader()
1720 1721
1721 1722 def hasAllDataInBuffer(self):
1722 1723 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
1723 1724 return 1
1724 1725 return 0
1725 1726
1726 1727
1727 1728 def setBlockDimension(self):
1728 1729 """
1729 1730 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
1730 1731
1731 1732 Affected:
1732 1733 self.shape_spc_Buffer
1733 1734 self.shape_cspc_Buffer
1734 1735 self.shape_dc_Buffer
1735 1736
1736 1737 Return: None
1737 1738 """
1738 1739 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
1739 1740 self.processingHeaderObj.nHeights,
1740 1741 self.systemHeaderObj.nChannels)
1741 1742
1742 1743 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
1743 1744 self.processingHeaderObj.profilesPerBlock,
1744 1745 self.processingHeaderObj.nHeights),
1745 1746 dtype=numpy.dtype('complex64'))
1746 1747
1747 1748
1748 1749 def writeBlock(self):
1749 1750 """
1750 1751 Escribe el buffer en el file designado
1751 1752
1752 1753 Affected:
1753 1754 self.profileIndex
1754 1755 self.flagIsNewFile
1755 1756 self.flagIsNewBlock
1756 1757 self.nTotalBlocks
1757 1758 self.blockIndex
1758 1759
1759 1760 Return: None
1760 1761 """
1761 1762 data = numpy.zeros( self.shapeBuffer, self.dtype )
1762 1763
1763 1764 junk = numpy.transpose(self.datablock, (1,2,0))
1764 1765
1765 1766 data['real'] = junk.real
1766 1767 data['imag'] = junk.imag
1767 1768
1768 1769 data = data.reshape( (-1) )
1769 1770
1770 1771 data.tofile( self.fp )
1771 1772
1772 1773 self.datablock.fill(0)
1773 1774
1774 1775 self.profileIndex = 0
1775 1776 self.flagIsNewFile = 0
1776 1777 self.flagIsNewBlock = 1
1777 1778
1778 1779 self.blockIndex += 1
1779 1780 self.nTotalBlocks += 1
1780 1781
1781 1782 def putData(self):
1782 1783 """
1783 1784 Setea un bloque de datos y luego los escribe en un file
1784 1785
1785 1786 Affected:
1786 1787 self.flagIsNewBlock
1787 1788 self.profileIndex
1788 1789
1789 1790 Return:
1790 1791 0 : Si no hay data o no hay mas files que puedan escribirse
1791 1792 1 : Si se escribio la data de un bloque en un file
1792 1793 """
1793 1794 if self.dataOut.flagNoData:
1794 1795 return 0
1795 1796
1796 1797 self.flagIsNewBlock = 0
1797 1798
1798 1799 if self.dataOut.flagTimeBlock:
1799 1800
1800 1801 self.datablock.fill(0)
1801 1802 self.profileIndex = 0
1802 1803 self.setNextFile()
1803 1804
1804 1805 if self.profileIndex == 0:
1805 1806 self.setBasicHeader()
1806 1807
1807 1808 self.datablock[:,self.profileIndex,:] = self.dataOut.data
1808 1809
1809 1810 self.profileIndex += 1
1810 1811
1811 1812 if self.hasAllDataInBuffer():
1812 1813 #if self.flagIsNewFile:
1813 1814 self.writeNextBlock()
1814 1815 # self.setFirstHeader()
1815 1816
1816 1817 return 1
1817 1818
1818 1819 def __getProcessFlags(self):
1819 1820
1820 1821 processFlags = 0
1821 1822
1822 1823 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1823 1824 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1824 1825 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1825 1826 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1826 1827 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1827 1828 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1828 1829
1829 1830 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1830 1831
1831 1832
1832 1833
1833 1834 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
1834 1835 PROCFLAG.DATATYPE_SHORT,
1835 1836 PROCFLAG.DATATYPE_LONG,
1836 1837 PROCFLAG.DATATYPE_INT64,
1837 1838 PROCFLAG.DATATYPE_FLOAT,
1838 1839 PROCFLAG.DATATYPE_DOUBLE]
1839 1840
1840 1841
1841 1842 for index in range(len(dtypeList)):
1842 1843 if self.dataOut.dtype == dtypeList[index]:
1843 1844 dtypeValue = datatypeValueList[index]
1844 1845 break
1845 1846
1846 1847 processFlags += dtypeValue
1847 1848
1848 1849 if self.dataOut.flagDecodeData:
1849 1850 processFlags += PROCFLAG.DECODE_DATA
1850 1851
1851 1852 if self.dataOut.flagDeflipData:
1852 1853 processFlags += PROCFLAG.DEFLIP_DATA
1853 1854
1854 1855 if self.dataOut.code != None:
1855 1856 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1856 1857
1857 1858 if self.dataOut.nCohInt > 1:
1858 1859 processFlags += PROCFLAG.COHERENT_INTEGRATION
1859 1860
1860 1861 return processFlags
1861 1862
1862 1863
1863 1864 def __getBlockSize(self):
1864 1865 '''
1865 1866 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
1866 1867 '''
1867 1868
1868 1869 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
1869 1870 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
1870 1871 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
1871 1872 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
1872 1873 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
1873 1874 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
1874 1875
1875 1876 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
1876 1877 datatypeValueList = [1,2,4,8,4,8]
1877 1878 for index in range(len(dtypeList)):
1878 1879 if self.dataOut.dtype == dtypeList[index]:
1879 1880 datatypeValue = datatypeValueList[index]
1880 1881 break
1881 1882
1882 1883 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * datatypeValue * 2)
1883 1884
1884 1885 return blocksize
1885 1886
1886 1887 def setFirstHeader(self):
1887 1888
1888 1889 """
1889 1890 Obtiene una copia del First Header
1890 1891
1891 1892 Affected:
1892 1893 self.systemHeaderObj
1893 1894 self.radarControllerHeaderObj
1894 1895 self.dtype
1895 1896
1896 1897 Return:
1897 1898 None
1898 1899 """
1899 1900
1900 1901 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
1901 1902 self.systemHeaderObj.nChannels = self.dataOut.nChannels
1902 1903 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
1903 1904
1904 1905 self.setBasicHeader()
1905 1906
1906 1907 processingHeaderSize = 40 # bytes
1907 1908 self.processingHeaderObj.dtype = 0 # Voltage
1908 1909 self.processingHeaderObj.blockSize = self.__getBlockSize()
1909 1910 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
1910 1911 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
1911 1912 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
1912 1913 self.processingHeaderObj.processFlags = self.__getProcessFlags()
1913 1914 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
1914 1915 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
1915 1916 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
1916 1917
1917 1918 # if self.dataOut.code != None:
1918 1919 # self.processingHeaderObj.code = self.dataOut.code
1919 1920 # self.processingHeaderObj.nCode = self.dataOut.nCode
1920 1921 # self.processingHeaderObj.nBaud = self.dataOut.nBaud
1921 1922 # codesize = int(8 + 4 * self.dataOut.nCode * self.dataOut.nBaud)
1922 1923 # processingHeaderSize += codesize
1923 1924
1924 1925 if self.processingHeaderObj.nWindows != 0:
1925 1926 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
1926 1927 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
1927 1928 self.processingHeaderObj.nHeights = self.dataOut.nHeights
1928 1929 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
1929 1930 processingHeaderSize += 12
1930 1931
1931 1932 self.processingHeaderObj.size = processingHeaderSize
1932 1933
1933 1934 class SpectraReader(JRODataReader):
1934 1935 """
1935 1936 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
1936 1937 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
1937 1938 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
1938 1939
1939 1940 paresCanalesIguales * alturas * perfiles (Self Spectra)
1940 1941 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
1941 1942 canales * alturas (DC Channels)
1942 1943
1943 1944 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
1944 1945 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
1945 1946 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
1946 1947 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
1947 1948
1948 1949 Example:
1949 1950 dpath = "/home/myuser/data"
1950 1951
1951 1952 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
1952 1953
1953 1954 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
1954 1955
1955 1956 readerObj = SpectraReader()
1956 1957
1957 1958 readerObj.setup(dpath, startTime, endTime)
1958 1959
1959 1960 while(True):
1960 1961
1961 1962 readerObj.getData()
1962 1963
1963 1964 print readerObj.data_spc
1964 1965
1965 1966 print readerObj.data_cspc
1966 1967
1967 1968 print readerObj.data_dc
1968 1969
1969 1970 if readerObj.flagNoMoreFiles:
1970 1971 break
1971 1972
1972 1973 """
1973 1974
1974 1975 pts2read_SelfSpectra = 0
1975 1976
1976 1977 pts2read_CrossSpectra = 0
1977 1978
1978 1979 pts2read_DCchannels = 0
1979 1980
1980 1981 ext = ".pdata"
1981 1982
1982 1983 optchar = "P"
1983 1984
1984 1985 dataOut = None
1985 1986
1986 1987 nRdChannels = None
1987 1988
1988 1989 nRdPairs = None
1989 1990
1990 1991 rdPairList = []
1991 1992
1992 1993 def __init__(self):
1993 1994 """
1994 1995 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
1995 1996
1996 1997 Inputs:
1997 1998 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
1998 1999 almacenar un perfil de datos cada vez que se haga un requerimiento
1999 2000 (getData). El perfil sera obtenido a partir del buffer de datos,
2000 2001 si el buffer esta vacio se hara un nuevo proceso de lectura de un
2001 2002 bloque de datos.
2002 2003 Si este parametro no es pasado se creara uno internamente.
2003 2004
2004 2005 Affected:
2005 2006 self.dataOut
2006 2007
2007 2008 Return : None
2008 2009 """
2009 2010
2010 2011 self.isConfig = False
2011 2012
2012 2013 self.pts2read_SelfSpectra = 0
2013 2014
2014 2015 self.pts2read_CrossSpectra = 0
2015 2016
2016 2017 self.pts2read_DCchannels = 0
2017 2018
2018 2019 self.datablock = None
2019 2020
2020 2021 self.utc = None
2021 2022
2022 2023 self.ext = ".pdata"
2023 2024
2024 2025 self.optchar = "P"
2025 2026
2026 2027 self.basicHeaderObj = BasicHeader(LOCALTIME)
2027 2028
2028 2029 self.systemHeaderObj = SystemHeader()
2029 2030
2030 2031 self.radarControllerHeaderObj = RadarControllerHeader()
2031 2032
2032 2033 self.processingHeaderObj = ProcessingHeader()
2033 2034
2034 2035 self.online = 0
2035 2036
2036 2037 self.fp = None
2037 2038
2038 2039 self.idFile = None
2039 2040
2040 2041 self.dtype = None
2041 2042
2042 2043 self.fileSizeByHeader = None
2043 2044
2044 2045 self.filenameList = []
2045 2046
2046 2047 self.filename = None
2047 2048
2048 2049 self.fileSize = None
2049 2050
2050 2051 self.firstHeaderSize = 0
2051 2052
2052 2053 self.basicHeaderSize = 24
2053 2054
2054 2055 self.pathList = []
2055 2056
2056 2057 self.lastUTTime = 0
2057 2058
2058 2059 self.maxTimeStep = 30
2059 2060
2060 2061 self.flagNoMoreFiles = 0
2061 2062
2062 2063 self.set = 0
2063 2064
2064 2065 self.path = None
2065 2066
2066 2067 self.delay = 60 #seconds
2067 2068
2068 2069 self.nTries = 3 #quantity tries
2069 2070
2070 2071 self.nFiles = 3 #number of files for searching
2071 2072
2072 2073 self.nReadBlocks = 0
2073 2074
2074 2075 self.flagIsNewFile = 1
2075 2076
2076 2077 self.__isFirstTimeOnline = 1
2077 2078
2078 2079 self.ippSeconds = 0
2079 2080
2080 2081 self.flagTimeBlock = 0
2081 2082
2082 2083 self.flagIsNewBlock = 0
2083 2084
2084 2085 self.nTotalBlocks = 0
2085 2086
2086 2087 self.blocksize = 0
2087 2088
2088 2089 self.dataOut = self.createObjByDefault()
2089 2090
2090 2091 self.profileIndex = 1 #Always
2091 2092
2092 2093
2093 2094 def createObjByDefault(self):
2094 2095
2095 2096 dataObj = Spectra()
2096 2097
2097 2098 return dataObj
2098 2099
2099 2100 def __hasNotDataInBuffer(self):
2100 2101 return 1
2101 2102
2102 2103
2103 2104 def getBlockDimension(self):
2104 2105 """
2105 2106 Obtiene la cantidad de puntos a leer por cada bloque de datos
2106 2107
2107 2108 Affected:
2108 2109 self.nRdChannels
2109 2110 self.nRdPairs
2110 2111 self.pts2read_SelfSpectra
2111 2112 self.pts2read_CrossSpectra
2112 2113 self.pts2read_DCchannels
2113 2114 self.blocksize
2114 2115 self.dataOut.nChannels
2115 2116 self.dataOut.nPairs
2116 2117
2117 2118 Return:
2118 2119 None
2119 2120 """
2120 2121 self.nRdChannels = 0
2121 2122 self.nRdPairs = 0
2122 2123 self.rdPairList = []
2123 2124
2124 2125 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
2125 2126 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
2126 2127 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
2127 2128 else:
2128 2129 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
2129 2130 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
2130 2131
2131 2132 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
2132 2133
2133 2134 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
2134 2135 self.blocksize = self.pts2read_SelfSpectra
2135 2136
2136 2137 if self.processingHeaderObj.flag_cspc:
2137 2138 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
2138 2139 self.blocksize += self.pts2read_CrossSpectra
2139 2140
2140 2141 if self.processingHeaderObj.flag_dc:
2141 2142 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
2142 2143 self.blocksize += self.pts2read_DCchannels
2143 2144
2144 2145 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
2145 2146
2146 2147
2147 2148 def readBlock(self):
2148 2149 """
2149 2150 Lee el bloque de datos desde la posicion actual del puntero del archivo
2150 2151 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
2151 2152 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
2152 2153 es seteado a 0
2153 2154
2154 2155 Return: None
2155 2156
2156 2157 Variables afectadas:
2157 2158
2158 2159 self.flagIsNewFile
2159 2160 self.flagIsNewBlock
2160 2161 self.nTotalBlocks
2161 2162 self.data_spc
2162 2163 self.data_cspc
2163 2164 self.data_dc
2164 2165
2165 2166 Exceptions:
2166 2167 Si un bloque leido no es un bloque valido
2167 2168 """
2168 2169 blockOk_flag = False
2169 2170 fpointer = self.fp.tell()
2170 2171
2171 2172 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
2172 2173 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
2173 2174
2174 2175 if self.processingHeaderObj.flag_cspc:
2175 2176 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
2176 2177 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
2177 2178
2178 2179 if self.processingHeaderObj.flag_dc:
2179 2180 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
2180 2181 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
2181 2182
2182 2183
2183 2184 if not(self.processingHeaderObj.shif_fft):
2184 2185 #desplaza a la derecha en el eje 2 determinadas posiciones
2185 2186 shift = int(self.processingHeaderObj.profilesPerBlock/2)
2186 2187 spc = numpy.roll( spc, shift , axis=2 )
2187 2188
2188 2189 if self.processingHeaderObj.flag_cspc:
2189 2190 #desplaza a la derecha en el eje 2 determinadas posiciones
2190 2191 cspc = numpy.roll( cspc, shift, axis=2 )
2191 2192
2192 2193 # self.processingHeaderObj.shif_fft = True
2193 2194
2194 2195 spc = numpy.transpose( spc, (0,2,1) )
2195 2196 self.data_spc = spc
2196 2197
2197 2198 if self.processingHeaderObj.flag_cspc:
2198 2199 cspc = numpy.transpose( cspc, (0,2,1) )
2199 2200 self.data_cspc = cspc['real'] + cspc['imag']*1j
2200 2201 else:
2201 2202 self.data_cspc = None
2202 2203
2203 2204 if self.processingHeaderObj.flag_dc:
2204 2205 self.data_dc = dc['real'] + dc['imag']*1j
2205 2206 else:
2206 2207 self.data_dc = None
2207 2208
2208 2209 self.flagIsNewFile = 0
2209 2210 self.flagIsNewBlock = 1
2210 2211
2211 2212 self.nTotalBlocks += 1
2212 2213 self.nReadBlocks += 1
2213 2214
2214 2215 return 1
2215 2216
2216 2217 def getFirstHeader(self):
2217 2218
2218 2219 self.dataOut.dtype = self.dtype
2219 2220
2220 2221 self.dataOut.nPairs = self.nRdPairs
2221 2222
2222 2223 self.dataOut.pairsList = self.rdPairList
2223 2224
2224 2225 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
2225 2226
2226 2227 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
2227 2228
2228 2229 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
2229 2230
2230 2231 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
2231 2232
2232 2233 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
2233 2234
2234 2235 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
2235 2236
2236 2237 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
2237 2238
2238 2239 self.dataOut.ippSeconds = self.ippSeconds
2239 2240
2240 2241 self.dataOut.timeInterval = self.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.dataOut.nFFTPoints
2241 2242
2242 2243 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
2243 2244
2244 2245 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
2245 2246
2246 2247 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
2247 2248
2248 2249 self.dataOut.flagDecodeData = False #asumo q la data no esta decodificada
2249 2250
2250 2251 self.dataOut.flagDeflipData = True #asumo q la data no esta sin flip
2251 2252
2252 2253 if self.radarControllerHeaderObj.code != None:
2253 2254
2254 2255 self.dataOut.nCode = self.radarControllerHeaderObj.nCode
2255 2256
2256 2257 self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
2257 2258
2258 2259 self.dataOut.code = self.radarControllerHeaderObj.code
2259 2260
2260 2261 self.dataOut.flagDecodeData = True
2261 2262
2262 2263 def getData(self):
2263 2264 """
2264 2265 Copia el buffer de lectura a la clase "Spectra",
2265 2266 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
2266 2267 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
2267 2268
2268 2269 Return:
2269 2270 0 : Si no hay mas archivos disponibles
2270 2271 1 : Si hizo una buena copia del buffer
2271 2272
2272 2273 Affected:
2273 2274 self.dataOut
2274 2275
2275 2276 self.flagTimeBlock
2276 2277 self.flagIsNewBlock
2277 2278 """
2278 2279
2279 2280 if self.flagNoMoreFiles:
2280 2281 self.dataOut.flagNoData = True
2281 2282 print 'Process finished'
2282 2283 return 0
2283 2284
2284 2285 self.flagTimeBlock = 0
2285 2286 self.flagIsNewBlock = 0
2286 2287
2287 2288 if self.__hasNotDataInBuffer():
2288 2289
2289 2290 if not( self.readNextBlock() ):
2290 2291 self.dataOut.flagNoData = True
2291 2292 return 0
2292 2293
2293 2294 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
2294 2295
2295 2296 if self.data_dc == None:
2296 2297 self.dataOut.flagNoData = True
2297 2298 return 0
2298 2299
2299 2300 self.getBasicHeader()
2300 2301
2301 2302 self.getFirstHeader()
2302 2303
2303 2304 self.dataOut.data_spc = self.data_spc
2304 2305
2305 2306 self.dataOut.data_cspc = self.data_cspc
2306 2307
2307 2308 self.dataOut.data_dc = self.data_dc
2308 2309
2309 2310 self.dataOut.flagNoData = False
2310 2311
2311 2312 self.dataOut.realtime = self.online
2312 2313
2313 2314 return self.dataOut.data_spc
2314 2315
2315 2316
2316 2317 class SpectraWriter(JRODataWriter):
2317 2318
2318 2319 """
2319 2320 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
2320 2321 de los datos siempre se realiza por bloques.
2321 2322 """
2322 2323
2323 2324 ext = ".pdata"
2324 2325
2325 2326 optchar = "P"
2326 2327
2327 2328 shape_spc_Buffer = None
2328 2329
2329 2330 shape_cspc_Buffer = None
2330 2331
2331 2332 shape_dc_Buffer = None
2332 2333
2333 2334 data_spc = None
2334 2335
2335 2336 data_cspc = None
2336 2337
2337 2338 data_dc = None
2338 2339
2339 2340 # dataOut = None
2340 2341
2341 2342 def __init__(self):
2342 2343 """
2343 2344 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
2344 2345
2345 2346 Affected:
2346 2347 self.dataOut
2347 2348 self.basicHeaderObj
2348 2349 self.systemHeaderObj
2349 2350 self.radarControllerHeaderObj
2350 2351 self.processingHeaderObj
2351 2352
2352 2353 Return: None
2353 2354 """
2354 2355
2355 2356 self.isConfig = False
2356 2357
2357 2358 self.nTotalBlocks = 0
2358 2359
2359 2360 self.data_spc = None
2360 2361
2361 2362 self.data_cspc = None
2362 2363
2363 2364 self.data_dc = None
2364 2365
2365 2366 self.fp = None
2366 2367
2367 2368 self.flagIsNewFile = 1
2368 2369
2369 2370 self.nTotalBlocks = 0
2370 2371
2371 2372 self.flagIsNewBlock = 0
2372 2373
2373 2374 self.setFile = None
2374 2375
2375 2376 self.dtype = None
2376 2377
2377 2378 self.path = None
2378 2379
2379 2380 self.noMoreFiles = 0
2380 2381
2381 2382 self.filename = None
2382 2383
2383 2384 self.basicHeaderObj = BasicHeader(LOCALTIME)
2384 2385
2385 2386 self.systemHeaderObj = SystemHeader()
2386 2387
2387 2388 self.radarControllerHeaderObj = RadarControllerHeader()
2388 2389
2389 2390 self.processingHeaderObj = ProcessingHeader()
2390 2391
2391 2392
2392 2393 def hasAllDataInBuffer(self):
2393 2394 return 1
2394 2395
2395 2396
2396 2397 def setBlockDimension(self):
2397 2398 """
2398 2399 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
2399 2400
2400 2401 Affected:
2401 2402 self.shape_spc_Buffer
2402 2403 self.shape_cspc_Buffer
2403 2404 self.shape_dc_Buffer
2404 2405
2405 2406 Return: None
2406 2407 """
2407 2408 self.shape_spc_Buffer = (self.dataOut.nChannels,
2408 2409 self.processingHeaderObj.nHeights,
2409 2410 self.processingHeaderObj.profilesPerBlock)
2410 2411
2411 2412 self.shape_cspc_Buffer = (self.dataOut.nPairs,
2412 2413 self.processingHeaderObj.nHeights,
2413 2414 self.processingHeaderObj.profilesPerBlock)
2414 2415
2415 2416 self.shape_dc_Buffer = (self.dataOut.nChannels,
2416 2417 self.processingHeaderObj.nHeights)
2417 2418
2418 2419
2419 2420 def writeBlock(self):
2420 2421 """
2421 2422 Escribe el buffer en el file designado
2422 2423
2423 2424 Affected:
2424 2425 self.data_spc
2425 2426 self.data_cspc
2426 2427 self.data_dc
2427 2428 self.flagIsNewFile
2428 2429 self.flagIsNewBlock
2429 2430 self.nTotalBlocks
2430 2431 self.nWriteBlocks
2431 2432
2432 2433 Return: None
2433 2434 """
2434 2435
2435 2436 spc = numpy.transpose( self.data_spc, (0,2,1) )
2436 2437 if not( self.processingHeaderObj.shif_fft ):
2437 2438 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2438 2439 data = spc.reshape((-1))
2439 2440 data = data.astype(self.dtype[0])
2440 2441 data.tofile(self.fp)
2441 2442
2442 2443 if self.data_cspc != None:
2443 2444 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
2444 2445 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
2445 2446 if not( self.processingHeaderObj.shif_fft ):
2446 2447 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
2447 2448 data['real'] = cspc.real
2448 2449 data['imag'] = cspc.imag
2449 2450 data = data.reshape((-1))
2450 2451 data.tofile(self.fp)
2451 2452
2452 2453 if self.data_dc != None:
2453 2454 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
2454 2455 dc = self.data_dc
2455 2456 data['real'] = dc.real
2456 2457 data['imag'] = dc.imag
2457 2458 data = data.reshape((-1))
2458 2459 data.tofile(self.fp)
2459 2460
2460 2461 self.data_spc.fill(0)
2461 2462
2462 2463 if self.data_dc != None:
2463 2464 self.data_dc.fill(0)
2464 2465
2465 2466 if self.data_cspc != None:
2466 2467 self.data_cspc.fill(0)
2467 2468
2468 2469 self.flagIsNewFile = 0
2469 2470 self.flagIsNewBlock = 1
2470 2471 self.nTotalBlocks += 1
2471 2472 self.nWriteBlocks += 1
2472 2473 self.blockIndex += 1
2473 2474
2474 2475
2475 2476 def putData(self):
2476 2477 """
2477 2478 Setea un bloque de datos y luego los escribe en un file
2478 2479
2479 2480 Affected:
2480 2481 self.data_spc
2481 2482 self.data_cspc
2482 2483 self.data_dc
2483 2484
2484 2485 Return:
2485 2486 0 : Si no hay data o no hay mas files que puedan escribirse
2486 2487 1 : Si se escribio la data de un bloque en un file
2487 2488 """
2488 2489
2489 2490 if self.dataOut.flagNoData:
2490 2491 return 0
2491 2492
2492 2493 self.flagIsNewBlock = 0
2493 2494
2494 2495 if self.dataOut.flagTimeBlock:
2495 2496 self.data_spc.fill(0)
2496 2497 self.data_cspc.fill(0)
2497 2498 self.data_dc.fill(0)
2498 2499 self.setNextFile()
2499 2500
2500 2501 if self.flagIsNewFile == 0:
2501 2502 self.setBasicHeader()
2502 2503
2503 2504 self.data_spc = self.dataOut.data_spc.copy()
2504 2505 if self.dataOut.data_cspc != None:
2505 2506 self.data_cspc = self.dataOut.data_cspc.copy()
2506 2507 self.data_dc = self.dataOut.data_dc.copy()
2507 2508
2508 2509 # #self.processingHeaderObj.dataBlocksPerFile)
2509 2510 if self.hasAllDataInBuffer():
2510 2511 # self.setFirstHeader()
2511 2512 self.writeNextBlock()
2512 2513
2513 2514 return 1
2514 2515
2515 2516
2516 2517 def __getProcessFlags(self):
2517 2518
2518 2519 processFlags = 0
2519 2520
2520 2521 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2521 2522 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2522 2523 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2523 2524 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2524 2525 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2525 2526 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2526 2527
2527 2528 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2528 2529
2529 2530
2530 2531
2531 2532 datatypeValueList = [PROCFLAG.DATATYPE_CHAR,
2532 2533 PROCFLAG.DATATYPE_SHORT,
2533 2534 PROCFLAG.DATATYPE_LONG,
2534 2535 PROCFLAG.DATATYPE_INT64,
2535 2536 PROCFLAG.DATATYPE_FLOAT,
2536 2537 PROCFLAG.DATATYPE_DOUBLE]
2537 2538
2538 2539
2539 2540 for index in range(len(dtypeList)):
2540 2541 if self.dataOut.dtype == dtypeList[index]:
2541 2542 dtypeValue = datatypeValueList[index]
2542 2543 break
2543 2544
2544 2545 processFlags += dtypeValue
2545 2546
2546 2547 if self.dataOut.flagDecodeData:
2547 2548 processFlags += PROCFLAG.DECODE_DATA
2548 2549
2549 2550 if self.dataOut.flagDeflipData:
2550 2551 processFlags += PROCFLAG.DEFLIP_DATA
2551 2552
2552 2553 if self.dataOut.code != None:
2553 2554 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
2554 2555
2555 2556 if self.dataOut.nIncohInt > 1:
2556 2557 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
2557 2558
2558 2559 if self.dataOut.data_dc != None:
2559 2560 processFlags += PROCFLAG.SAVE_CHANNELS_DC
2560 2561
2561 2562 return processFlags
2562 2563
2563 2564
2564 2565 def __getBlockSize(self):
2565 2566 '''
2566 2567 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
2567 2568 '''
2568 2569
2569 2570 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
2570 2571 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
2571 2572 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
2572 2573 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
2573 2574 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
2574 2575 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
2575 2576
2576 2577 dtypeList = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
2577 2578 datatypeValueList = [1,2,4,8,4,8]
2578 2579 for index in range(len(dtypeList)):
2579 2580 if self.dataOut.dtype == dtypeList[index]:
2580 2581 datatypeValue = datatypeValueList[index]
2581 2582 break
2582 2583
2583 2584
2584 2585 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
2585 2586
2586 2587 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
2587 2588 blocksize = (pts2write_SelfSpectra*datatypeValue)
2588 2589
2589 2590 if self.dataOut.data_cspc != None:
2590 2591 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
2591 2592 blocksize += (pts2write_CrossSpectra*datatypeValue*2)
2592 2593
2593 2594 if self.dataOut.data_dc != None:
2594 2595 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
2595 2596 blocksize += (pts2write_DCchannels*datatypeValue*2)
2596 2597
2597 2598 blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
2598 2599
2599 2600 return blocksize
2600 2601
2601 2602 def setFirstHeader(self):
2602 2603
2603 2604 """
2604 2605 Obtiene una copia del First Header
2605 2606
2606 2607 Affected:
2607 2608 self.systemHeaderObj
2608 2609 self.radarControllerHeaderObj
2609 2610 self.dtype
2610 2611
2611 2612 Return:
2612 2613 None
2613 2614 """
2614 2615
2615 2616 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
2616 2617 self.systemHeaderObj.nChannels = self.dataOut.nChannels
2617 2618 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
2618 2619 old_code_size = self.dataOut.radarControllerHeaderObj.code_size
2619 2620 new_code_size = int(numpy.ceil(self.dataOut.nBaud/32.))*self.dataOut.nCode*4
2620 2621 self.radarControllerHeaderObj.size = self.radarControllerHeaderObj.size - old_code_size + new_code_size
2621 2622
2622 2623 self.setBasicHeader()
2623 2624
2624 2625 processingHeaderSize = 40 # bytes
2625 2626 self.processingHeaderObj.dtype = 1 # Spectra
2626 2627 self.processingHeaderObj.blockSize = self.__getBlockSize()
2627 2628 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
2628 2629 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
2629 2630 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
2630 2631 self.processingHeaderObj.processFlags = self.__getProcessFlags()
2631 2632 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
2632 2633 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
2633 2634 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
2634 2635 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
2635 2636
2636 2637 if self.processingHeaderObj.totalSpectra > 0:
2637 2638 channelList = []
2638 2639 for channel in range(self.dataOut.nChannels):
2639 2640 channelList.append(channel)
2640 2641 channelList.append(channel)
2641 2642
2642 2643 pairsList = []
2643 2644 if self.dataOut.nPairs > 0:
2644 2645 for pair in self.dataOut.pairsList:
2645 2646 pairsList.append(pair[0])
2646 2647 pairsList.append(pair[1])
2647 2648
2648 2649 spectraComb = channelList + pairsList
2649 2650 spectraComb = numpy.array(spectraComb,dtype="u1")
2650 2651 self.processingHeaderObj.spectraComb = spectraComb
2651 2652 sizeOfSpcComb = len(spectraComb)
2652 2653 processingHeaderSize += sizeOfSpcComb
2653 2654
2654 2655 # The processing header should not have information about code
2655 2656 # if self.dataOut.code != None:
2656 2657 # self.processingHeaderObj.code = self.dataOut.code
2657 2658 # self.processingHeaderObj.nCode = self.dataOut.nCode
2658 2659 # self.processingHeaderObj.nBaud = self.dataOut.nBaud
2659 2660 # nCodeSize = 4 # bytes
2660 2661 # nBaudSize = 4 # bytes
2661 2662 # codeSize = 4 # bytes
2662 2663 # sizeOfCode = int(nCodeSize + nBaudSize + codeSize * self.dataOut.nCode * self.dataOut.nBaud)
2663 2664 # processingHeaderSize += sizeOfCode
2664 2665
2665 2666 if self.processingHeaderObj.nWindows != 0:
2666 2667 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
2667 2668 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
2668 2669 self.processingHeaderObj.nHeights = self.dataOut.nHeights
2669 2670 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
2670 2671 sizeOfFirstHeight = 4
2671 2672 sizeOfdeltaHeight = 4
2672 2673 sizeOfnHeights = 4
2673 2674 sizeOfWindows = (sizeOfFirstHeight + sizeOfdeltaHeight + sizeOfnHeights)*self.processingHeaderObj.nWindows
2674 2675 processingHeaderSize += sizeOfWindows
2675 2676
2676 2677 self.processingHeaderObj.size = processingHeaderSize
2677 2678
2678 2679 class SpectraHeisWriter(Operation):
2679 2680 # set = None
2680 2681 setFile = None
2681 2682 idblock = None
2682 2683 doypath = None
2683 2684 subfolder = None
2684 2685
2685 2686 def __init__(self):
2686 2687 self.wrObj = FITS()
2687 2688 # self.dataOut = dataOut
2688 2689 self.nTotalBlocks=0
2689 2690 # self.set = None
2690 2691 self.setFile = None
2691 2692 self.idblock = 0
2692 2693 self.wrpath = None
2693 2694 self.doypath = None
2694 2695 self.subfolder = None
2695 2696 self.isConfig = False
2696 2697
2697 2698 def isNumber(str):
2698 2699 """
2699 2700 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
2700 2701
2701 2702 Excepciones:
2702 2703 Si un determinado string no puede ser convertido a numero
2703 2704 Input:
2704 2705 str, string al cual se le analiza para determinar si convertible a un numero o no
2705 2706
2706 2707 Return:
2707 2708 True : si el string es uno numerico
2708 2709 False : no es un string numerico
2709 2710 """
2710 2711 try:
2711 2712 float( str )
2712 2713 return True
2713 2714 except:
2714 2715 return False
2715 2716
2716 2717 def setup(self, dataOut, wrpath):
2717 2718
2718 2719 if not(os.path.exists(wrpath)):
2719 2720 os.mkdir(wrpath)
2720 2721
2721 2722 self.wrpath = wrpath
2722 2723 # self.setFile = 0
2723 2724 self.dataOut = dataOut
2724 2725
2725 2726 def putData(self):
2726 2727 name= time.localtime( self.dataOut.utctime)
2727 2728 ext=".fits"
2728 2729
2729 2730 if self.doypath == None:
2730 2731 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
2731 2732 self.doypath = os.path.join( self.wrpath, self.subfolder )
2732 2733 os.mkdir(self.doypath)
2733 2734
2734 2735 if self.setFile == None:
2735 2736 # self.set = self.dataOut.set
2736 2737 self.setFile = 0
2737 2738 # if self.set != self.dataOut.set:
2738 2739 ## self.set = self.dataOut.set
2739 2740 # self.setFile = 0
2740 2741
2741 2742 #make the filename
2742 2743 file = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
2743 2744
2744 2745 filename = os.path.join(self.wrpath,self.subfolder, file)
2745 2746
2746 2747 idblock = numpy.array([self.idblock],dtype="int64")
2747 2748 header=self.wrObj.cFImage(idblock=idblock,
2748 2749 year=time.gmtime(self.dataOut.utctime).tm_year,
2749 2750 month=time.gmtime(self.dataOut.utctime).tm_mon,
2750 2751 day=time.gmtime(self.dataOut.utctime).tm_mday,
2751 2752 hour=time.gmtime(self.dataOut.utctime).tm_hour,
2752 2753 minute=time.gmtime(self.dataOut.utctime).tm_min,
2753 2754 second=time.gmtime(self.dataOut.utctime).tm_sec)
2754 2755
2755 2756 c=3E8
2756 2757 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
2757 2758 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
2758 2759
2759 2760 colList = []
2760 2761
2761 2762 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
2762 2763
2763 2764 colList.append(colFreq)
2764 2765
2765 2766 nchannel=self.dataOut.nChannels
2766 2767
2767 2768 for i in range(nchannel):
2768 2769 col = self.wrObj.writeData(name="PCh"+str(i+1),
2769 2770 format=str(self.dataOut.nFFTPoints)+'E',
2770 2771 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
2771 2772
2772 2773 colList.append(col)
2773 2774
2774 2775 data=self.wrObj.Ctable(colList=colList)
2775 2776
2776 2777 self.wrObj.CFile(header,data)
2777 2778
2778 2779 self.wrObj.wFile(filename)
2779 2780
2780 2781 #update the setFile
2781 2782 self.setFile += 1
2782 2783 self.idblock += 1
2783 2784
2784 2785 return 1
2785 2786
2786 2787 def run(self, dataOut, **kwargs):
2787 2788
2788 2789 if not(self.isConfig):
2789 2790
2790 2791 self.setup(dataOut, **kwargs)
2791 2792 self.isConfig = True
2792 2793
2793 2794 self.putData()
2794 2795
2795 2796
2796 2797
2797 2798 class ParameterConf:
2798 2799 ELEMENTNAME = 'Parameter'
2799 2800 def __init__(self):
2800 2801 self.name = ''
2801 2802 self.value = ''
2802 2803
2803 2804 def readXml(self, parmElement):
2804 2805 self.name = parmElement.get('name')
2805 2806 self.value = parmElement.get('value')
2806 2807
2807 2808 def getElementName(self):
2808 2809 return self.ELEMENTNAME
2809 2810
2810 2811 class Metadata:
2811 2812
2812 2813 def __init__(self, filename):
2813 2814 self.parmConfObjList = []
2814 2815 self.readXml(filename)
2815 2816
2816 2817 def readXml(self, filename):
2817 2818 self.projectElement = None
2818 2819 self.procUnitConfObjDict = {}
2819 2820 self.projectElement = ElementTree().parse(filename)
2820 2821 self.project = self.projectElement.tag
2821 2822
2822 2823 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
2823 2824
2824 2825 for parmElement in parmElementList:
2825 2826 parmConfObj = ParameterConf()
2826 2827 parmConfObj.readXml(parmElement)
2827 2828 self.parmConfObjList.append(parmConfObj)
2828 2829
2829 2830 class FitsWriter(Operation):
2830 2831
2831 2832 def __init__(self):
2832 2833 self.isConfig = False
2833 2834 self.dataBlocksPerFile = None
2834 2835 self.blockIndex = 0
2835 2836 self.flagIsNewFile = 1
2836 2837 self.fitsObj = None
2837 2838 self.optchar = 'P'
2838 2839 self.ext = '.fits'
2839 2840 self.setFile = 0
2840 2841
2841 2842 def setFitsHeader(self, dataOut, metadatafile):
2842 2843
2843 2844 header_data = pyfits.PrimaryHDU()
2844 2845
2845 2846 metadata4fits = Metadata(metadatafile)
2846 2847 for parameter in metadata4fits.parmConfObjList:
2847 2848 parm_name = parameter.name
2848 2849 parm_value = parameter.value
2849 2850
2850 2851 # if parm_value == 'fromdatadatetime':
2851 2852 # value = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
2852 2853 # elif parm_value == 'fromdataheights':
2853 2854 # value = dataOut.nHeights
2854 2855 # elif parm_value == 'fromdatachannel':
2855 2856 # value = dataOut.nChannels
2856 2857 # elif parm_value == 'fromdatasamples':
2857 2858 # value = dataOut.nFFTPoints
2858 2859 # else:
2859 2860 # value = parm_value
2860 2861
2861 2862 header_data.header[parm_name] = parm_value
2862 2863
2863 2864
2864 2865 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
2865 2866 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
2866 2867 header_data.header['NCHANNELS'] = dataOut.nChannels
2867 2868 #header_data.header['HEIGHTS'] = dataOut.heightList
2868 2869 header_data.header['NHEIGHTS'] = dataOut.nHeights
2869 2870
2870 2871 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
2871 2872 header_data.header['NCOHINT'] = dataOut.nCohInt
2872 2873 header_data.header['NINCOHINT'] = dataOut.nIncohInt
2873 2874 header_data.header['TIMEZONE'] = dataOut.timeZone
2874 2875 header_data.header['NBLOCK'] = self.blockIndex
2875 2876
2876 2877 header_data.writeto(self.filename)
2877 2878
2878 2879 self.addExtension(dataOut.heightList,'HEIGHTLIST')
2879 2880
2880 2881
2881 2882 def setup(self, dataOut, path, dataBlocksPerFile, metadatafile):
2882 2883
2883 2884 self.path = path
2884 2885 self.dataOut = dataOut
2885 2886 self.metadatafile = metadatafile
2886 2887 self.dataBlocksPerFile = dataBlocksPerFile
2887 2888
2888 2889 def open(self):
2889 2890 self.fitsObj = pyfits.open(self.filename, mode='update')
2890 2891
2891 2892
2892 2893 def addExtension(self, data, tagname):
2893 2894 self.open()
2894 2895 extension = pyfits.ImageHDU(data=data, name=tagname)
2895 2896 #extension.header['TAG'] = tagname
2896 2897 self.fitsObj.append(extension)
2897 2898 self.write()
2898 2899
2899 2900 def addData(self, data):
2900 2901 self.open()
2901 2902 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
2902 2903 extension.header['UTCTIME'] = self.dataOut.utctime
2903 2904 self.fitsObj.append(extension)
2904 2905 self.blockIndex += 1
2905 2906 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
2906 2907
2907 2908 self.write()
2908 2909
2909 2910 def write(self):
2910 2911
2911 2912 self.fitsObj.flush(verbose=True)
2912 2913 self.fitsObj.close()
2913 2914
2914 2915
2915 2916 def setNextFile(self):
2916 2917
2917 2918 ext = self.ext
2918 2919 path = self.path
2919 2920
2920 2921 timeTuple = time.localtime( self.dataOut.utctime)
2921 2922 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
2922 2923
2923 2924 fullpath = os.path.join( path, subfolder )
2924 2925 if not( os.path.exists(fullpath) ):
2925 2926 os.mkdir(fullpath)
2926 2927 self.setFile = -1 #inicializo mi contador de seteo
2927 2928 else:
2928 2929 filesList = os.listdir( fullpath )
2929 2930 if len( filesList ) > 0:
2930 2931 filesList = sorted( filesList, key=str.lower )
2931 2932 filen = filesList[-1]
2932 2933
2933 2934 if isNumber( filen[8:11] ):
2934 2935 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
2935 2936 else:
2936 2937 self.setFile = -1
2937 2938 else:
2938 2939 self.setFile = -1 #inicializo mi contador de seteo
2939 2940
2940 2941 setFile = self.setFile
2941 2942 setFile += 1
2942 2943
2943 2944 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
2944 2945 timeTuple.tm_year,
2945 2946 timeTuple.tm_yday,
2946 2947 setFile,
2947 2948 ext )
2948 2949
2949 2950 filename = os.path.join( path, subfolder, file )
2950 2951
2951 2952 self.blockIndex = 0
2952 2953 self.filename = filename
2953 2954 self.setFile = setFile
2954 2955 self.flagIsNewFile = 1
2955 2956
2956 2957 print 'Writing the file: %s'%self.filename
2957 2958
2958 2959 self.setFitsHeader(self.dataOut, self.metadatafile)
2959 2960
2960 2961 return 1
2961 2962
2962 2963 def writeBlock(self):
2963 2964 self.addData(self.dataOut.data_spc)
2964 2965 self.flagIsNewFile = 0
2965 2966
2966 2967
2967 2968 def __setNewBlock(self):
2968 2969
2969 2970 if self.flagIsNewFile:
2970 2971 return 1
2971 2972
2972 2973 if self.blockIndex < self.dataBlocksPerFile:
2973 2974 return 1
2974 2975
2975 2976 if not( self.setNextFile() ):
2976 2977 return 0
2977 2978
2978 2979 return 1
2979 2980
2980 2981 def writeNextBlock(self):
2981 2982 if not( self.__setNewBlock() ):
2982 2983 return 0
2983 2984 self.writeBlock()
2984 2985 return 1
2985 2986
2986 2987 def putData(self):
2987 2988 if self.flagIsNewFile:
2988 2989 self.setNextFile()
2989 2990 self.writeNextBlock()
2990 2991
2991 2992 def run(self, dataOut, **kwargs):
2992 2993 if not(self.isConfig):
2993 2994 self.setup(dataOut, **kwargs)
2994 2995 self.isConfig = True
2995 2996 self.putData()
2996 2997
2997 2998
2998 2999 class FitsReader(ProcessingUnit):
2999 3000
3000 3001 # __TIMEZONE = time.timezone
3001 3002
3002 3003 expName = None
3003 3004 datetimestr = None
3004 3005 utc = None
3005 3006 nChannels = None
3006 3007 nSamples = None
3007 3008 dataBlocksPerFile = None
3008 3009 comments = None
3009 3010 lastUTTime = None
3010 3011 header_dict = None
3011 3012 data = None
3012 3013 data_header_dict = None
3013 3014
3014 3015 def __init__(self):
3015 3016 self.isConfig = False
3016 3017 self.ext = '.fits'
3017 3018 self.setFile = 0
3018 3019 self.flagNoMoreFiles = 0
3019 3020 self.flagIsNewFile = 1
3020 3021 self.flagTimeBlock = None
3021 3022 self.fileIndex = None
3022 3023 self.filename = None
3023 3024 self.fileSize = None
3024 3025 self.fitsObj = None
3025 3026 self.timeZone = None
3026 3027 self.nReadBlocks = 0
3027 3028 self.nTotalBlocks = 0
3028 3029 self.dataOut = self.createObjByDefault()
3029 3030 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
3030 3031 self.blockIndex = 1
3031 3032
3032 3033 def createObjByDefault(self):
3033 3034
3034 3035 dataObj = Fits()
3035 3036
3036 3037 return dataObj
3037 3038
3038 3039 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
3039 3040 try:
3040 3041 fitsObj = pyfits.open(filename,'readonly')
3041 3042 except:
3042 3043 raise IOError, "The file %s can't be opened" %(filename)
3043 3044
3044 3045 header = fitsObj[0].header
3045 3046 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
3046 3047 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
3047 3048
3048 3049 ltc = utc
3049 3050 if useLocalTime:
3050 3051 ltc -= time.timezone
3051 3052 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
3052 3053 thisTime = thisDatetime.time()
3053 3054
3054 3055 if not ((startTime <= thisTime) and (endTime > thisTime)):
3055 3056 return None
3056 3057
3057 3058 return thisDatetime
3058 3059
3059 3060 def __setNextFileOnline(self):
3060 3061 raise ValueError, "No implemented"
3061 3062
3062 3063 def __setNextFileOffline(self):
3063 3064 idFile = self.fileIndex
3064 3065
3065 3066 while (True):
3066 3067 idFile += 1
3067 3068 if not(idFile < len(self.filenameList)):
3068 3069 self.flagNoMoreFiles = 1
3069 3070 print "No more Files"
3070 3071 return 0
3071 3072
3072 3073 filename = self.filenameList[idFile]
3073 3074
3074 3075 # if not(self.__verifyFile(filename)):
3075 3076 # continue
3076 3077
3077 3078 fileSize = os.path.getsize(filename)
3078 3079 fitsObj = pyfits.open(filename,'readonly')
3079 3080 break
3080 3081
3081 3082 self.flagIsNewFile = 1
3082 3083 self.fileIndex = idFile
3083 3084 self.filename = filename
3084 3085 self.fileSize = fileSize
3085 3086 self.fitsObj = fitsObj
3086 3087 self.blockIndex = 0
3087 3088 print "Setting the file: %s"%self.filename
3088 3089
3089 3090 return 1
3090 3091
3091 3092 def readHeader(self):
3092 3093 headerObj = self.fitsObj[0]
3093 3094
3094 3095 self.header_dict = headerObj.header
3095 3096 if 'EXPNAME' in headerObj.header.keys():
3096 3097 self.expName = headerObj.header['EXPNAME']
3097 3098
3098 3099 if 'DATATYPE' in headerObj.header.keys():
3099 3100 self.dataType = headerObj.header['DATATYPE']
3100 3101
3101 3102 self.datetimestr = headerObj.header['DATETIME']
3102 3103 channelList = headerObj.header['CHANNELLIST']
3103 3104 channelList = channelList.split('[')
3104 3105 channelList = channelList[1].split(']')
3105 3106 channelList = channelList[0].split(',')
3106 3107 channelList = [int(ch) for ch in channelList]
3107 3108 self.channelList = channelList
3108 3109 self.nChannels = headerObj.header['NCHANNELS']
3109 3110 self.nHeights = headerObj.header['NHEIGHTS']
3110 3111 self.ippSeconds = headerObj.header['IPPSECONDS']
3111 3112 self.nCohInt = headerObj.header['NCOHINT']
3112 3113 self.nIncohInt = headerObj.header['NINCOHINT']
3113 3114 self.dataBlocksPerFile = headerObj.header['NBLOCK']
3114 3115 self.timeZone = headerObj.header['TIMEZONE']
3115 3116
3116 3117 self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
3117 3118
3118 3119 if 'COMMENT' in headerObj.header.keys():
3119 3120 self.comments = headerObj.header['COMMENT']
3120 3121
3121 3122 self.readHeightList()
3122 3123
3123 3124 def readHeightList(self):
3124 3125 self.blockIndex = self.blockIndex + 1
3125 3126 obj = self.fitsObj[self.blockIndex]
3126 3127 self.heightList = obj.data
3127 3128 self.blockIndex = self.blockIndex + 1
3128 3129
3129 3130 def readExtension(self):
3130 3131 obj = self.fitsObj[self.blockIndex]
3131 3132 self.heightList = obj.data
3132 3133 self.blockIndex = self.blockIndex + 1
3133 3134
3134 3135 def setNextFile(self):
3135 3136
3136 3137 if self.online:
3137 3138 newFile = self.__setNextFileOnline()
3138 3139 else:
3139 3140 newFile = self.__setNextFileOffline()
3140 3141
3141 3142 if not(newFile):
3142 3143 return 0
3143 3144
3144 3145 self.readHeader()
3145 3146
3146 3147 self.nReadBlocks = 0
3147 3148 # self.blockIndex = 1
3148 3149 return 1
3149 3150
3150 3151 def __searchFilesOffLine(self,
3151 3152 path,
3152 3153 startDate,
3153 3154 endDate,
3154 3155 startTime=datetime.time(0,0,0),
3155 3156 endTime=datetime.time(23,59,59),
3156 3157 set=None,
3157 3158 expLabel='',
3158 3159 ext='.fits',
3159 3160 walk=True):
3160 3161
3161 3162 pathList = []
3162 3163
3163 3164 if not walk:
3164 3165 pathList.append(path)
3165 3166
3166 3167 else:
3167 3168 dirList = []
3168 3169 for thisPath in os.listdir(path):
3169 3170 if not os.path.isdir(os.path.join(path,thisPath)):
3170 3171 continue
3171 3172 if not isDoyFolder(thisPath):
3172 3173 continue
3173 3174
3174 3175 dirList.append(thisPath)
3175 3176
3176 3177 if not(dirList):
3177 3178 return None, None
3178 3179
3179 3180 thisDate = startDate
3180 3181
3181 3182 while(thisDate <= endDate):
3182 3183 year = thisDate.timetuple().tm_year
3183 3184 doy = thisDate.timetuple().tm_yday
3184 3185
3185 3186 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
3186 3187 if len(matchlist) == 0:
3187 3188 thisDate += datetime.timedelta(1)
3188 3189 continue
3189 3190 for match in matchlist:
3190 3191 pathList.append(os.path.join(path,match,expLabel))
3191 3192
3192 3193 thisDate += datetime.timedelta(1)
3193 3194
3194 3195 if pathList == []:
3195 3196 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
3196 3197 return None, None
3197 3198
3198 3199 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
3199 3200
3200 3201 filenameList = []
3201 3202 datetimeList = []
3202 3203
3203 3204 for i in range(len(pathList)):
3204 3205
3205 3206 thisPath = pathList[i]
3206 3207
3207 3208 fileList = glob.glob1(thisPath, "*%s" %ext)
3208 3209 fileList.sort()
3209 3210
3210 3211 for file in fileList:
3211 3212
3212 3213 filename = os.path.join(thisPath,file)
3213 3214 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
3214 3215
3215 3216 if not(thisDatetime):
3216 3217 continue
3217 3218
3218 3219 filenameList.append(filename)
3219 3220 datetimeList.append(thisDatetime)
3220 3221
3221 3222 if not(filenameList):
3222 3223 print "Any file was found for the time range %s - %s" %(startTime, endTime)
3223 3224 return None, None
3224 3225
3225 3226 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
3226 3227 print
3227 3228
3228 3229 for i in range(len(filenameList)):
3229 3230 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
3230 3231
3231 3232 self.filenameList = filenameList
3232 3233 self.datetimeList = datetimeList
3233 3234
3234 3235 return pathList, filenameList
3235 3236
3236 3237 def setup(self, path=None,
3237 3238 startDate=None,
3238 3239 endDate=None,
3239 3240 startTime=datetime.time(0,0,0),
3240 3241 endTime=datetime.time(23,59,59),
3241 3242 set=0,
3242 3243 expLabel = "",
3243 3244 ext = None,
3244 3245 online = False,
3245 3246 delay = 60,
3246 3247 walk = True):
3247 3248
3248 3249 if path == None:
3249 3250 raise ValueError, "The path is not valid"
3250 3251
3251 3252 if ext == None:
3252 3253 ext = self.ext
3253 3254
3254 3255 if not(online):
3255 3256 print "Searching files in offline mode ..."
3256 3257 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
3257 3258 startTime=startTime, endTime=endTime,
3258 3259 set=set, expLabel=expLabel, ext=ext,
3259 3260 walk=walk)
3260 3261
3261 3262 if not(pathList):
3262 3263 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
3263 3264 datetime.datetime.combine(startDate,startTime).ctime(),
3264 3265 datetime.datetime.combine(endDate,endTime).ctime())
3265 3266
3266 3267 sys.exit(-1)
3267 3268
3268 3269 self.fileIndex = -1
3269 3270 self.pathList = pathList
3270 3271 self.filenameList = filenameList
3271 3272
3272 3273 self.online = online
3273 3274 self.delay = delay
3274 3275 ext = ext.lower()
3275 3276 self.ext = ext
3276 3277
3277 3278 if not(self.setNextFile()):
3278 3279 if (startDate!=None) and (endDate!=None):
3279 3280 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
3280 3281 elif startDate != None:
3281 3282 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
3282 3283 else:
3283 3284 print "No files"
3284 3285
3285 3286 sys.exit(-1)
3286 3287
3287 3288
3288 3289
3289 3290 def readBlock(self):
3290 3291 dataObj = self.fitsObj[self.blockIndex]
3291 3292
3292 3293 self.data = dataObj.data
3293 3294 self.data_header_dict = dataObj.header
3294 3295 self.utc = self.data_header_dict['UTCTIME']
3295 3296
3296 3297 self.flagIsNewFile = 0
3297 3298 self.blockIndex += 1
3298 3299 self.nTotalBlocks += 1
3299 3300 self.nReadBlocks += 1
3300 3301
3301 3302 return 1
3302 3303
3303 3304 def __jumpToLastBlock(self):
3304 3305 raise ValueError, "No implemented"
3305 3306
3306 3307 def __waitNewBlock(self):
3307 3308 """
3308 3309 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
3309 3310
3310 3311 Si el modo de lectura es OffLine siempre retorn 0
3311 3312 """
3312 3313 if not self.online:
3313 3314 return 0
3314 3315
3315 3316 if (self.nReadBlocks >= self.dataBlocksPerFile):
3316 3317 return 0
3317 3318
3318 3319 currentPointer = self.fp.tell()
3319 3320
3320 3321 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
3321 3322
3322 3323 for nTries in range( self.nTries ):
3323 3324
3324 3325 self.fp.close()
3325 3326 self.fp = open( self.filename, 'rb' )
3326 3327 self.fp.seek( currentPointer )
3327 3328
3328 3329 self.fileSize = os.path.getsize( self.filename )
3329 3330 currentSize = self.fileSize - currentPointer
3330 3331
3331 3332 if ( currentSize >= neededSize ):
3332 3333 self.__rdBasicHeader()
3333 3334 return 1
3334 3335
3335 3336 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
3336 3337 time.sleep( self.delay )
3337 3338
3338 3339
3339 3340 return 0
3340 3341
3341 3342 def __setNewBlock(self):
3342 3343
3343 3344 if self.online:
3344 3345 self.__jumpToLastBlock()
3345 3346
3346 3347 if self.flagIsNewFile:
3347 3348 return 1
3348 3349
3349 3350 self.lastUTTime = self.utc
3350 3351
3351 3352 if self.online:
3352 3353 if self.__waitNewBlock():
3353 3354 return 1
3354 3355
3355 3356 if self.nReadBlocks < self.dataBlocksPerFile:
3356 3357 return 1
3357 3358
3358 3359 if not(self.setNextFile()):
3359 3360 return 0
3360 3361
3361 3362 deltaTime = self.utc - self.lastUTTime
3362 3363
3363 3364 self.flagTimeBlock = 0
3364 3365
3365 3366 if deltaTime > self.maxTimeStep:
3366 3367 self.flagTimeBlock = 1
3367 3368
3368 3369 return 1
3369 3370
3370 3371
3371 3372 def readNextBlock(self):
3372 3373 if not(self.__setNewBlock()):
3373 3374 return 0
3374 3375
3375 3376 if not(self.readBlock()):
3376 3377 return 0
3377 3378
3378 3379 return 1
3379 3380
3380 3381
3381 3382 def getData(self):
3382 3383
3383 3384 if self.flagNoMoreFiles:
3384 3385 self.dataOut.flagNoData = True
3385 3386 print 'Process finished'
3386 3387 return 0
3387 3388
3388 3389 self.flagTimeBlock = 0
3389 3390 self.flagIsNewBlock = 0
3390 3391
3391 3392 if not(self.readNextBlock()):
3392 3393 return 0
3393 3394
3394 3395 if self.data == None:
3395 3396 self.dataOut.flagNoData = True
3396 3397 return 0
3397 3398
3398 3399 self.dataOut.data = self.data
3399 3400 self.dataOut.data_header = self.data_header_dict
3400 3401 self.dataOut.utctime = self.utc
3401 3402
3402 3403 self.dataOut.header = self.header_dict
3403 3404 self.dataOut.expName = self.expName
3404 3405 self.dataOut.nChannels = self.nChannels
3405 3406 self.dataOut.timeZone = self.timeZone
3406 3407 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
3407 3408 self.dataOut.comments = self.comments
3408 3409 self.dataOut.timeInterval = self.timeInterval
3409 3410 self.dataOut.channelList = self.channelList
3410 3411 self.dataOut.heightList = self.heightList
3411 3412 self.dataOut.flagNoData = False
3412 3413
3413 3414 return self.dataOut.data
3414 3415
3415 3416 def run(self, **kwargs):
3416 3417
3417 3418 if not(self.isConfig):
3418 3419 self.setup(**kwargs)
3419 3420 self.isConfig = True
3420 3421
3421 self.getData() No newline at end of file
3422 self.getData()
3423
3424
3425 class RadacHeader():
3426 def __init__(self, fp):
3427 header = 'Raw11/Data/RadacHeader'
3428 self.beamCode = fp.get(header+'/BeamCode')
3429 self.code = fp.get(header+'/Code')
3430 self.frameCount = fp.get(header+'/FrameCount')
3431 self.modeGroup = fp.get(header+'/ModeGroup')
3432 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
3433 self.pulseCount = fp.get(header+'/PulseCount')
3434 self.radacTime = fp.get(header+'/RadacTime')
3435 self.timeCount = fp.get(header+'/TimeCount')
3436 self.timeStatus = fp.get(header+'/TimeStatus')
3437
3438 self.nrecords = self.pulseCount.shape[0] #numero de bloques
3439 self.npulses = self.pulseCount.shape[1] #numero de perfiles
3440 self.nsamples = self.nsamplesPulse[0,0] #numero de alturas
3441
3442
3443 def getIndexRangeToPulse(self, idrecord=0):
3444 indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
3445 startPulseCountId = indexToZero[0][0]
3446 endPulseCountId = startPulseCountId - 1
3447 range1 = numpy.arange(startPulseCountId,self.npulses,1)
3448 range2 = numpy.arange(0,startPulseCountId,1)
3449 return range1, range2
3450
3451
3452 class AMISRReader(ProcessingUnit):
3453
3454 path = None
3455 startDate = None
3456 endDate = None
3457 startTime = None
3458 endTime = None
3459 walk = None
3460 isConfig = False
3461
3462 def __init__(self):
3463 self.set = None
3464 self.subset = None
3465 self.extension_file = '.h5'
3466 self.dtc_str = 'dtc'
3467 self.dtc_id = 0
3468 self.status = True
3469 self.isConfig = False
3470 self.dirnameList = []
3471 self.filenameList = []
3472 self.fileIndex = None
3473 self.flagNoMoreFiles = False
3474 self.flagIsNewFile = 0
3475 self.filename = ''
3476 self.amisrFilePointer = None
3477 self.radacHeaderObj = None
3478 self.dataOut = self.__createObjByDefault()
3479 self.datablock = None
3480 self.rest_datablock = None
3481 self.range = None
3482 self.idrecord_count = 0
3483 self.profileIndex = 0
3484 self.idpulse_range1 = None
3485 self.idpulse_range2 = None
3486 self.beamCodeByFrame = None
3487 self.radacTimeByFrame = None
3488 #atributos originales tal y como esta en el archivo de datos
3489 self.beamCodesFromFile = None
3490 self.radacTimeFromFile = None
3491 self.rangeFromFile = None
3492 self.dataByFrame = None
3493 self.dataset = None
3494
3495
3496 def __createObjByDefault(self):
3497
3498 dataObj = AMISR()
3499
3500 return dataObj
3501
3502 def __setParameters(self,path,startDate,endDate,startTime,endTime,walk):
3503 self.path = path
3504 self.startDate = startDate
3505 self.endDate = endDate
3506 self.startTime = startTime
3507 self.endTime = endTime
3508 self.walk = walk
3509
3510 def __checkPath(self):
3511 if os.path.exists(self.path):
3512 self.status = 1
3513 else:
3514 self.status = 0
3515 print 'Path:%s does not exists'%self.path
3516
3517 return
3518
3519 def __selDates(self, amisr_dirname_format):
3520 year = int(amisr_dirname_format[0:4])
3521 month = int(amisr_dirname_format[4:6])
3522 dom = int(amisr_dirname_format[6:8])
3523 thisDate = datetime.date(year,month,dom)
3524
3525 if (thisDate>=self.startDate and thisDate <= self.endDate):
3526 return amisr_dirname_format
3527
3528 def __findDataForDates(self):
3529
3530 import re
3531
3532 if not(self.status):
3533 return None
3534
3535 pat = '\d+.\d+'
3536 dirnameList = [re.search(pat,x).string for x in os.listdir(self.path)]
3537 dirnameList = [self.__selDates(x) for x in dirnameList]
3538 dirnameList = filter(lambda x:x!=None,dirnameList)
3539 if len(dirnameList)>0:
3540 self.status = 1
3541 self.dirnameList = dirnameList
3542 self.dirnameList.sort()
3543 else:
3544 self.status = 0
3545 return None
3546
3547 def __getTimeFromData(self):
3548 pass
3549
3550 def __filterByGlob1(self, dirName):
3551 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
3552 filterDict = {}
3553 filterDict.setdefault(dirName)
3554 filterDict[dirName] = filter_files
3555 return filterDict
3556
3557 def __getFilenameList(self, fileListInKeys, dirList):
3558 for value in fileListInKeys:
3559 dirName = value.keys()[0]
3560 for file in value[dirName]:
3561 filename = os.path.join(dirName, file)
3562 self.filenameList.append(filename)
3563
3564
3565 def __selectDataForTimes(self):
3566 #aun no esta implementado el filtro for tiempo
3567 if not(self.status):
3568 return None
3569
3570 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
3571
3572 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
3573
3574 self.__getFilenameList(fileListInKeys, dirList)
3575
3576 if len(self.filenameList)>0:
3577 self.status = 1
3578 self.filenameList.sort()
3579 else:
3580 self.status = 0
3581 return None
3582
3583
3584 def __searchFilesOffline(self,
3585 path,
3586 startDate,
3587 endDate,
3588 startTime=datetime.time(0,0,0),
3589 endTime=datetime.time(23,59,59),
3590 walk=True):
3591
3592 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
3593
3594 self.__checkPath()
3595
3596 self.__findDataForDates()
3597
3598 self.__selectDataForTimes()
3599
3600 for i in range(len(self.filenameList)):
3601 print "%s" %(self.filenameList[i])
3602
3603 return
3604
3605 def __setNextFileOffline(self):
3606 idFile = self.fileIndex
3607
3608 while (True):
3609 idFile += 1
3610 if not(idFile < len(self.filenameList)):
3611 self.flagNoMoreFiles = 1
3612 print "No more Files"
3613 return 0
3614
3615 filename = self.filenameList[idFile]
3616
3617 amisrFilePointer = h5py.File(filename,'r')
3618
3619 break
3620
3621 self.flagIsNewFile = 1
3622 self.fileIndex = idFile
3623 self.filename = filename
3624
3625 self.amisrFilePointer = amisrFilePointer
3626
3627 print "Setting the file: %s"%self.filename
3628
3629 return 1
3630
3631 def __readHeader(self):
3632 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
3633 self.flagIsNewFile = 1
3634
3635
3636
3637 def __setNextFile(self):
3638
3639 newFile = self.__setNextFileOffline()
3640
3641 if not(newFile):
3642 return 0
3643
3644 self.__readHeader()
3645
3646 self.readDataBlock()
3647
3648
3649 def setup(self,path=None,
3650 startDate=None,
3651 endDate=None,
3652 startTime=datetime.time(0,0,0),
3653 endTime=datetime.time(23,59,59),
3654 walk=True):
3655
3656 #Busqueda de archivos offline
3657 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
3658
3659 if not(self.filenameList):
3660 print "There is no files into the folder: %s"%(path)
3661
3662 sys.exit(-1)
3663
3664 self.fileIndex = -1
3665
3666 self.__setNextFile()
3667
3668 def readRanges(self):
3669 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
3670 self.rangeFromFile = dataset.values()
3671 return range
3672
3673
3674 def readRadacTime(self,idrecord, range1, range2):
3675 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
3676
3677 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
3678 #radacTimeByFrame = dataset[idrecord - 1,range1]
3679 #radacTimeByFrame = dataset[idrecord,range2]
3680
3681 return radacTimeByFrame
3682
3683 def readBeamCode(self, idrecord, range1, range2):
3684 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
3685 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
3686 self.beamCodesFromFile = dataset.value
3687
3688 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
3689 #beamcodeByFrame[range2] = dataset[idrecord, range2]
3690 beamcodeByFrame[range1] = dataset[idrecord, range1]
3691 beamcodeByFrame[range2] = dataset[idrecord, range2]
3692
3693 return beamcodeByFrame
3694
3695
3696 def __setDataByFrame(self):
3697 ndata = 2 # porque es complejo
3698 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
3699 return dataByFrame
3700
3701 def __readDataSet(self):
3702 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
3703 return dataset
3704
3705 def __setDataBlock(self,):
3706 real = self.dataByFrame[:,:,0] #asumo que 0 es real
3707 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
3708 datablock = real + imag*1j #armo el complejo
3709 return datablock
3710
3711 def readSamples_version1(self,idrecord):
3712 #estas tres primeras lineas solo se deben ejecutar una vez
3713 if self.flagIsNewFile:
3714 self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0)
3715 self.dataByFrame = self.__setDataByFrame()
3716 self.beamCodeByFrame = self.readBeamCode(idrecord, self.idpulse_range1, self.idpulse_range2)
3717 self.radacTimeByFrame = self.readRadacTime(idrecord, self.idpulse_range1, self.idpulse_range2)
3718 #reading dataset
3719 self.dataset = self.__readDataSet()
3720
3721 if idrecord == 0:
3722
3723 if len(numpy.where(self.dataByFrame!=0.0)[0]) or len(numpy.where(self.dataByFrame!=0.0)[1]) or len(numpy.where(self.dataByFrame!=0.0)[2]):
3724 #falta agregar una condicion para datos discontinuos
3725 #por defecto une los datos del record anterior
3726 self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
3727 #timepulse
3728 self.radacTimeByFrame[self.idpulse_range2] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2]
3729 else:
3730 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
3731
3732 self.radacTimeByFrame[self.idpulse_range1] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1]
3733
3734 datablock = self.__setDataBlock()
3735
3736 return datablock
3737
3738 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord - 1,self.idpulse_range1, :, :]
3739 self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
3740 datablock = self.__setDataBlock()
3741 self.flagIsNewFile = 0
3742
3743 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
3744
3745
3746 return datablock
3747
3748
3749 def readSamples(self,idrecord):
3750 if self.flagIsNewFile:
3751 self.dataByFrame = self.__setDataByFrame()
3752 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
3753
3754 #reading dataset
3755 self.dataset = self.__readDataSet()
3756
3757 self.flagIsNewFile = 0
3758 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
3759 self.dataByFrame = self.dataset[idrecord, :, :, :]
3760 datablock = self.__setDataBlock()
3761 return datablock
3762
3763
3764 def readDataBlock(self):
3765
3766 #self.datablock = self.readSamples(self.idrecord_count)
3767 self.datablock = self.readSamples(self.idrecord_count)
3768 #print 'record:', self.idrecord_count
3769
3770 self.idrecord_count += 1
3771 self.profileIndex = 0
3772
3773 if self.idrecord_count >= self.radacHeaderObj.nrecords:
3774 self.idrecord_count = 0
3775 self.flagIsNewFile = 1
3776
3777 def readNextBlock(self):
3778
3779 self.readDataBlock()
3780
3781 if self.flagIsNewFile:
3782 self.__setNextFile()
3783 pass
3784
3785 def __hasNotDataInBuffer(self):
3786 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
3787 if self.profileIndex >= self.radacHeaderObj.npulses:
3788 return 1
3789 return 0
3790
3791
3792 def getData(self):
3793
3794 if self.flagNoMoreFiles:
3795 self.dataOut.flagNoData = True
3796 print 'Process finished'
3797 return 0
3798
3799 if self.__hasNotDataInBuffer():
3800 self.readNextBlock()
3801 # if not( self.readNextBlock() ):
3802 # return 0
3803 # self.getFirstHeader()
3804
3805 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
3806 self.dataOut.flagNoData = True
3807 return 0
3808
3809 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
3810
3811 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
3812
3813 self.dataOut.flagNoData = False
3814
3815 self.profileIndex += 1
3816
3817 return self.dataOut.data
3818
3819
3820 def run(self, **kwargs):
3821 if not(self.isConfig):
3822 self.setup(**kwargs)
3823 self.isConfig = True
3824
3825 self.getData()
General Comments 0
You need to be logged in to leave comments. Login now