##// END OF EJS Templates
schain
RSS
receiving as unit reader
José Chávez -
r961:80664bd78a14
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 You should install "digital_rf_hdf5" module if you want to read USRP data
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="Voltage" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="plotter"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="other"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1794 +1,1810
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 #import h5py
15 14 import traceback
16 15
17 16 try:
18 17 from gevent import sleep
19 18 except:
20 19 from time import sleep
21 20
22 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 23
25 24 LOCALTIME = True
26 25
27 26 def isNumber(cad):
28 27 """
29 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 29
31 30 Excepciones:
32 31 Si un determinado string no puede ser convertido a numero
33 32 Input:
34 33 str, string al cual se le analiza para determinar si convertible a un numero o no
35 34
36 35 Return:
37 36 True : si el string es uno numerico
38 37 False : no es un string numerico
39 38 """
40 39 try:
41 40 float( cad )
42 41 return True
43 42 except:
44 43 return False
45 44
46 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 46 """
48 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 48
50 49 Inputs:
51 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 51
53 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 53 segundos contados desde 01/01/1970.
55 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 55 segundos contados desde 01/01/1970.
57 56
58 57 Return:
59 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 59 fecha especificado, de lo contrario retorna False.
61 60
62 61 Excepciones:
63 62 Si el archivo no existe o no puede ser abierto
64 63 Si la cabecera no puede ser leida.
65 64
66 65 """
67 66 basicHeaderObj = BasicHeader(LOCALTIME)
68 67
69 68 try:
70 69 fp = open(filename,'rb')
71 70 except IOError:
72 71 print "The file %s can't be opened" %(filename)
73 72 return 0
74 73
75 74 sts = basicHeaderObj.read(fp)
76 75 fp.close()
77 76
78 77 if not(sts):
79 78 print "Skipping the file %s because it has not a valid header" %(filename)
80 79 return 0
81 80
82 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 82 return 0
84 83
85 84 return 1
86 85
87 86 def isTimeInRange(thisTime, startTime, endTime):
88 87
89 88 if endTime >= startTime:
90 89 if (thisTime < startTime) or (thisTime > endTime):
91 90 return 0
92 91
93 92 return 1
94 93 else:
95 94 if (thisTime < startTime) and (thisTime > endTime):
96 95 return 0
97 96
98 97 return 1
99 98
100 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 100 """
102 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 102
104 103 Inputs:
105 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 105
107 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 107
109 108 endDate : fecha final del rango seleccionado en formato datetime.date
110 109
111 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 111
113 112 endTime : tiempo final del rango seleccionado en formato datetime.time
114 113
115 114 Return:
116 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 116 fecha especificado, de lo contrario retorna False.
118 117
119 118 Excepciones:
120 119 Si el archivo no existe o no puede ser abierto
121 120 Si la cabecera no puede ser leida.
122 121
123 122 """
124 123
125 124
126 125 try:
127 126 fp = open(filename,'rb')
128 127 except IOError:
129 128 print "The file %s can't be opened" %(filename)
130 129 return None
131 130
132 131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 132 systemHeaderObj = SystemHeader()
134 133 radarControllerHeaderObj = RadarControllerHeader()
135 134 processingHeaderObj = ProcessingHeader()
136 135
137 136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 137
139 138 sts = firstBasicHeaderObj.read(fp)
140 139
141 140 if not(sts):
142 141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 142 return None
144 143
145 144 if not systemHeaderObj.read(fp):
146 145 return None
147 146
148 147 if not radarControllerHeaderObj.read(fp):
149 148 return None
150 149
151 150 if not processingHeaderObj.read(fp):
152 151 return None
153 152
154 153 filesize = os.path.getsize(filename)
155 154
156 155 offset = processingHeaderObj.blockSize + 24 #header size
157 156
158 157 if filesize <= offset:
159 158 print "[Reading] %s: This file has not enough data" %filename
160 159 return None
161 160
162 161 fp.seek(-offset, 2)
163 162
164 163 sts = lastBasicHeaderObj.read(fp)
165 164
166 165 fp.close()
167 166
168 167 thisDatetime = lastBasicHeaderObj.datatime
169 168 thisTime_last_block = thisDatetime.time()
170 169
171 170 thisDatetime = firstBasicHeaderObj.datatime
172 171 thisDate = thisDatetime.date()
173 172 thisTime_first_block = thisDatetime.time()
174 173
175 174 #General case
176 175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 176 #-----------o----------------------------o-----------
178 177 # startTime endTime
179 178
180 179 if endTime >= startTime:
181 180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 181 return None
183 182
184 183 return thisDatetime
185 184
186 185 #If endTime < startTime then endTime belongs to the next day
187 186
188 187
189 188 #<<<<<<<<<<<o o>>>>>>>>>>>
190 189 #-----------o----------------------------o-----------
191 190 # endTime startTime
192 191
193 192 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 193 return None
195 194
196 195 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 196 return None
198 197
199 198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 199 return None
201 200
202 201 return thisDatetime
203 202
204 203 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 204 """
206 205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 206
208 207 Inputs:
209 208 folder : nombre completo del directorio.
210 209 Su formato deberia ser "/path_root/?YYYYDDD"
211 210
212 211 siendo:
213 212 YYYY : Anio (ejemplo 2015)
214 213 DDD : Dia del anio (ejemplo 305)
215 214
216 215 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 216
218 217 endDate : fecha final del rango seleccionado en formato datetime.date
219 218
220 219 Return:
221 220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 221 fecha especificado, de lo contrario retorna False.
223 222 Excepciones:
224 223 Si el directorio no tiene el formato adecuado
225 224 """
226 225
227 226 basename = os.path.basename(folder)
228 227
229 228 if not isRadarFolder(basename):
230 229 print "The folder %s has not the rigth format" %folder
231 230 return 0
232 231
233 232 if startDate and endDate:
234 233 thisDate = getDateFromRadarFolder(basename)
235 234
236 235 if thisDate < startDate:
237 236 return 0
238 237
239 238 if thisDate > endDate:
240 239 return 0
241 240
242 241 return 1
243 242
244 243 def isFileInDateRange(filename, startDate=None, endDate=None):
245 244 """
246 245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 246
248 247 Inputs:
249 248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 249
251 250 Su formato deberia ser "?YYYYDDDsss"
252 251
253 252 siendo:
254 253 YYYY : Anio (ejemplo 2015)
255 254 DDD : Dia del anio (ejemplo 305)
256 255 sss : set
257 256
258 257 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 258
260 259 endDate : fecha final del rango seleccionado en formato datetime.date
261 260
262 261 Return:
263 262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 263 fecha especificado, de lo contrario retorna False.
265 264 Excepciones:
266 265 Si el archivo no tiene el formato adecuado
267 266 """
268 267
269 268 basename = os.path.basename(filename)
270 269
271 270 if not isRadarFile(basename):
272 271 print "The filename %s has not the rigth format" %filename
273 272 return 0
274 273
275 274 if startDate and endDate:
276 275 thisDate = getDateFromRadarFile(basename)
277 276
278 277 if thisDate < startDate:
279 278 return 0
280 279
281 280 if thisDate > endDate:
282 281 return 0
283 282
284 283 return 1
285 284
286 285 def getFileFromSet(path, ext, set):
287 286 validFilelist = []
288 287 fileList = os.listdir(path)
289 288
290 289 # 0 1234 567 89A BCDE
291 290 # H YYYY DDD SSS .ext
292 291
293 292 for thisFile in fileList:
294 293 try:
295 294 year = int(thisFile[1:5])
296 295 doy = int(thisFile[5:8])
297 296 except:
298 297 continue
299 298
300 299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 300 continue
302 301
303 302 validFilelist.append(thisFile)
304 303
305 304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 305
307 306 if len(myfile)!= 0:
308 307 return myfile[0]
309 308 else:
310 309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 310 print 'the filename %s does not exist'%filename
312 311 print '...going to the last file: '
313 312
314 313 if validFilelist:
315 314 validFilelist = sorted( validFilelist, key=str.lower )
316 315 return validFilelist[-1]
317 316
318 317 return None
319 318
320 319 def getlastFileFromPath(path, ext):
321 320 """
322 321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 323
325 324 Input:
326 325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 326 ext : extension de los files contenidos en una carpeta
328 327
329 328 Return:
330 329 El ultimo file de una determinada carpeta, no se considera el path.
331 330 """
332 331 validFilelist = []
333 332 fileList = os.listdir(path)
334 333
335 334 # 0 1234 567 89A BCDE
336 335 # H YYYY DDD SSS .ext
337 336
338 337 for thisFile in fileList:
339 338
340 339 year = thisFile[1:5]
341 340 if not isNumber(year):
342 341 continue
343 342
344 343 doy = thisFile[5:8]
345 344 if not isNumber(doy):
346 345 continue
347 346
348 347 year = int(year)
349 348 doy = int(doy)
350 349
351 350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 351 continue
353 352
354 353 validFilelist.append(thisFile)
355 354
356 355 if validFilelist:
357 356 validFilelist = sorted( validFilelist, key=str.lower )
358 357 return validFilelist[-1]
359 358
360 359 return None
361 360
362 361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 362 """
364 363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 365 el path exacto de un determinado file.
367 366
368 367 Example :
369 368 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 369
371 370 Entonces la funcion prueba con las siguientes combinaciones
372 371 .../.../y2009307367.ext
373 372 .../.../Y2009307367.ext
374 373 .../.../x2009307/y2009307367.ext
375 374 .../.../x2009307/Y2009307367.ext
376 375 .../.../X2009307/y2009307367.ext
377 376 .../.../X2009307/Y2009307367.ext
378 377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 378
380 379 Return:
381 380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 382 para el filename
384 383 """
385 384 fullfilename = None
386 385 find_flag = False
387 386 filename = None
388 387
389 388 prefixDirList = [None,'d','D']
390 389 if ext.lower() == ".r": #voltage
391 390 prefixFileList = ['d','D']
392 391 elif ext.lower() == ".pdata": #spectra
393 392 prefixFileList = ['p','P']
394 393 else:
395 394 return None, filename
396 395
397 396 #barrido por las combinaciones posibles
398 397 for prefixDir in prefixDirList:
399 398 thispath = path
400 399 if prefixDir != None:
401 400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 401 if foldercounter == 0:
403 402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 403 else:
405 404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 408
410 409 if os.path.exists( fullfilename ): #verifico que exista
411 410 find_flag = True
412 411 break
413 412 if find_flag:
414 413 break
415 414
416 415 if not(find_flag):
417 416 return None, filename
418 417
419 418 return fullfilename, filename
420 419
421 420 def isRadarFolder(folder):
422 421 try:
423 422 year = int(folder[1:5])
424 423 doy = int(folder[5:8])
425 424 except:
426 425 return 0
427 426
428 427 return 1
429 428
430 429 def isRadarFile(file):
431 430 try:
432 431 year = int(file[1:5])
433 432 doy = int(file[5:8])
434 433 set = int(file[8:11])
435 434 except:
436 435 return 0
437 436
438 437 return 1
439 438
440 439 def getDateFromRadarFile(file):
441 440 try:
442 441 year = int(file[1:5])
443 442 doy = int(file[5:8])
444 443 set = int(file[8:11])
445 444 except:
446 445 return None
447 446
448 447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 448 return thisDate
450 449
451 450 def getDateFromRadarFolder(folder):
452 451 try:
453 452 year = int(folder[1:5])
454 453 doy = int(folder[5:8])
455 454 except:
456 455 return None
457 456
458 457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 458 return thisDate
460 459
461 460 class JRODataIO:
462 461
463 462 c = 3E8
464 463
465 464 isConfig = False
466 465
467 466 basicHeaderObj = None
468 467
469 468 systemHeaderObj = None
470 469
471 470 radarControllerHeaderObj = None
472 471
473 472 processingHeaderObj = None
474 473
475 474 dtype = None
476 475
477 476 pathList = []
478 477
479 478 filenameList = []
480 479
481 480 filename = None
482 481
483 482 ext = None
484 483
485 484 flagIsNewFile = 1
486 485
487 486 flagDiscontinuousBlock = 0
488 487
489 488 flagIsNewBlock = 0
490 489
491 490 fp = None
492 491
493 492 firstHeaderSize = 0
494 493
495 494 basicHeaderSize = 24
496 495
497 496 versionFile = 1103
498 497
499 498 fileSize = None
500 499
501 500 # ippSeconds = None
502 501
503 502 fileSizeByHeader = None
504 503
505 504 fileIndex = None
506 505
507 506 profileIndex = None
508 507
509 508 blockIndex = None
510 509
511 510 nTotalBlocks = None
512 511
513 512 maxTimeStep = 30
514 513
515 514 lastUTTime = None
516 515
517 516 datablock = None
518 517
519 518 dataOut = None
520 519
521 520 blocksize = None
522 521
523 522 getByBlock = False
524 523
525 524 def __init__(self):
526 525
527 526 raise NotImplementedError
528 527
529 528 def run(self):
530 529
531 530 raise NotImplementedError
532 531
533 532 def getDtypeWidth(self):
534 533
535 534 dtype_index = get_dtype_index(self.dtype)
536 535 dtype_width = get_dtype_width(dtype_index)
537 536
538 537 return dtype_width
539 538
540 539 def getAllowedArgs(self):
541 540 return inspect.getargspec(self.run).args
542 541
543 542 class JRODataReader(JRODataIO):
544 543
545 544
546 545 online = 0
547 546
548 547 realtime = 0
549 548
550 549 nReadBlocks = 0
551 550
552 551 delay = 10 #number of seconds waiting a new file
553 552
554 553 nTries = 3 #quantity tries
555 554
556 555 nFiles = 3 #number of files for searching
557 556
558 557 path = None
559 558
560 559 foldercounter = 0
561 560
562 561 flagNoMoreFiles = 0
563 562
564 563 datetimeList = []
565 564
566 565 __isFirstTimeOnline = 1
567 566
568 567 __printInfo = True
569 568
570 569 profileIndex = None
571 570
572 571 nTxs = 1
573 572
574 573 txIndex = None
575 574
576 575 #Added--------------------
577 576
578 577 selBlocksize = None
579 578
580 579 selBlocktime = None
581 580
582 581
583 582 def __init__(self):
584 583
585 584 """
586 585 This class is used to find data files
587 586
588 587 Example:
589 588 reader = JRODataReader()
590 589 fileList = reader.findDataFiles()
591 590
592 591 """
593 592 pass
594 593
595 594
596 595 def createObjByDefault(self):
597 596 """
598 597
599 598 """
600 599 raise NotImplementedError
601 600
602 601 def getBlockDimension(self):
603 602
604 603 raise NotImplementedError
605 604
606 605 def __searchFilesOffLine(self,
607 606 path,
608 607 startDate=None,
609 608 endDate=None,
610 609 startTime=datetime.time(0,0,0),
611 610 endTime=datetime.time(23,59,59),
612 611 set=None,
613 612 expLabel='',
614 613 ext='.r',
615 614 queue=None,
616 615 cursor=None,
617 616 skip=None,
618 617 walk=True):
619 618
620 619 self.filenameList = []
621 620 self.datetimeList = []
622 621
623 622 pathList = []
624 623
625 624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
626 625
627 626 if dateList == []:
628 627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
629 628 return None, None
630 629
631 630 if len(dateList) > 1:
632 631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
633 632 else:
634 633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
635 634
636 635 filenameList = []
637 636 datetimeList = []
638 637
639 638 for thisPath in pathList:
640 639 # thisPath = pathList[pathDict[file]]
641 640
642 641 fileList = glob.glob1(thisPath, "*%s" %ext)
643 642 fileList.sort()
644 643
645 644 skippedFileList = []
646 645
647 646 if cursor is not None and skip is not None:
648 647 # if cursor*skip > len(fileList):
649 648 if skip == 0:
650 649 if queue is not None:
651 650 queue.put(len(fileList))
652 651 skippedFileList = []
653 652 else:
654 653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
655 654
656 655 else:
657 656 skippedFileList = fileList
658 657
659 658 for file in skippedFileList:
660 659
661 660 filename = os.path.join(thisPath,file)
662 661
663 662 if not isFileInDateRange(filename, startDate, endDate):
664 663 continue
665 664
666 665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
667 666
668 667 if not(thisDatetime):
669 668 continue
670 669
671 670 filenameList.append(filename)
672 671 datetimeList.append(thisDatetime)
673 672
674 673 if not(filenameList):
675 674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
676 675 return None, None
677 676
678 677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
679 678 print
680 679
681 680 for i in range(len(filenameList)):
682 681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
683 682
684 683 self.filenameList = filenameList
685 684 self.datetimeList = datetimeList
686 685
687 686 return pathList, filenameList
688 687
689 688 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
690 689
691 690 """
692 691 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
693 692 devuelve el archivo encontrado ademas de otros datos.
694 693
695 694 Input:
696 695 path : carpeta donde estan contenidos los files que contiene data
697 696
698 697 expLabel : Nombre del subexperimento (subfolder)
699 698
700 699 ext : extension de los files
701 700
702 701 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
703 702
704 703 Return:
705 704 directory : eL directorio donde esta el file encontrado
706 705 filename : el ultimo file de una determinada carpeta
707 706 year : el anho
708 707 doy : el numero de dia del anho
709 708 set : el set del archivo
710 709
711 710
712 711 """
713 712 if not os.path.isdir(path):
714 713 return None, None, None, None, None, None
715 714
716 715 dirList = []
717 716
718 717 if not walk:
719 718 fullpath = path
720 719 foldercounter = 0
721 720 else:
722 721 #Filtra solo los directorios
723 722 for thisPath in os.listdir(path):
724 723 if not os.path.isdir(os.path.join(path,thisPath)):
725 724 continue
726 725 if not isRadarFolder(thisPath):
727 726 continue
728 727
729 728 dirList.append(thisPath)
730 729
731 730 if not(dirList):
732 731 return None, None, None, None, None, None
733 732
734 733 dirList = sorted( dirList, key=str.lower )
735 734
736 735 doypath = dirList[-1]
737 736 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 737 fullpath = os.path.join(path, doypath, expLabel)
739 738
740 739
741 740 print "[Reading] %s folder was found: " %(fullpath )
742 741
743 742 if set == None:
744 743 filename = getlastFileFromPath(fullpath, ext)
745 744 else:
746 745 filename = getFileFromSet(fullpath, ext, set)
747 746
748 747 if not(filename):
749 748 return None, None, None, None, None, None
750 749
751 750 print "[Reading] %s file was found" %(filename)
752 751
753 752 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 753 return None, None, None, None, None, None
755 754
756 755 year = int( filename[1:5] )
757 756 doy = int( filename[5:8] )
758 757 set = int( filename[8:11] )
759 758
760 759 return fullpath, foldercounter, filename, year, doy, set
761 760
762 761 def __setNextFileOffline(self):
763 762
764 763 idFile = self.fileIndex
765 764
766 765 while (True):
767 766 idFile += 1
768 767 if not(idFile < len(self.filenameList)):
769 768 self.flagNoMoreFiles = 1
770 769 # print "[Reading] No more Files"
771 770 return 0
772 771
773 772 filename = self.filenameList[idFile]
774 773
775 774 if not(self.__verifyFile(filename)):
776 775 continue
777 776
778 777 fileSize = os.path.getsize(filename)
779 778 fp = open(filename,'rb')
780 779 break
781 780
782 781 self.flagIsNewFile = 1
783 782 self.fileIndex = idFile
784 783 self.filename = filename
785 784 self.fileSize = fileSize
786 785 self.fp = fp
787 786
788 787 # print "[Reading] Setting the file: %s"%self.filename
789 788
790 789 return 1
791 790
792 791 def __setNextFileOnline(self):
793 792 """
794 793 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 794 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 795 siguientes.
797 796
798 797 Affected:
799 798 self.flagIsNewFile
800 799 self.filename
801 800 self.fileSize
802 801 self.fp
803 802 self.set
804 803 self.flagNoMoreFiles
805 804
806 805 Return:
807 806 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 807 1 : si el file fue abierto con exito y esta listo a ser leido
809 808
810 809 Excepciones:
811 810 Si un determinado file no puede ser abierto
812 811 """
813 812 nFiles = 0
814 813 fileOk_flag = False
815 814 firstTime_flag = True
816 815
817 816 self.set += 1
818 817
819 818 if self.set > 999:
820 819 self.set = 0
821 820 self.foldercounter += 1
822 821
823 822 #busca el 1er file disponible
824 823 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 824 if fullfilename:
826 825 if self.__verifyFile(fullfilename, False):
827 826 fileOk_flag = True
828 827
829 828 #si no encuentra un file entonces espera y vuelve a buscar
830 829 if not(fileOk_flag):
831 830 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832 831
833 832 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 833 tries = self.nTries
835 834 else:
836 835 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837 836
838 837 for nTries in range( tries ):
839 838 if firstTime_flag:
840 839 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 840 sleep( self.delay )
842 841 else:
843 842 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844 843
845 844 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 845 if fullfilename:
847 846 if self.__verifyFile(fullfilename):
848 847 fileOk_flag = True
849 848 break
850 849
851 850 if fileOk_flag:
852 851 break
853 852
854 853 firstTime_flag = False
855 854
856 855 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 856 self.set += 1
858 857
859 858 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 859 self.set = 0
861 860 self.doy += 1
862 861 self.foldercounter = 0
863 862
864 863 if fileOk_flag:
865 864 self.fileSize = os.path.getsize( fullfilename )
866 865 self.filename = fullfilename
867 866 self.flagIsNewFile = 1
868 867 if self.fp != None: self.fp.close()
869 868 self.fp = open(fullfilename, 'rb')
870 869 self.flagNoMoreFiles = 0
871 870 # print '[Reading] Setting the file: %s' % fullfilename
872 871 else:
873 872 self.fileSize = 0
874 873 self.filename = None
875 874 self.flagIsNewFile = 0
876 875 self.fp = None
877 876 self.flagNoMoreFiles = 1
878 877 # print '[Reading] No more files to read'
879 878
880 879 return fileOk_flag
881 880
882 881 def setNextFile(self):
883 882 if self.fp != None:
884 883 self.fp.close()
885 884
886 885 if self.online:
887 886 newFile = self.__setNextFileOnline()
888 887 else:
889 888 newFile = self.__setNextFileOffline()
890 889
891 890 if not(newFile):
892 891 print '[Reading] No more files to read'
893 892 return 0
894 893
895 894 if self.verbose:
896 895 print '[Reading] Setting the file: %s' % self.filename
897 896
898 897 self.__readFirstHeader()
899 898 self.nReadBlocks = 0
900 899 return 1
901 900
902 901 def __waitNewBlock(self):
903 902 """
904 903 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
905 904
906 905 Si el modo de lectura es OffLine siempre retorn 0
907 906 """
908 907 if not self.online:
909 908 return 0
910 909
911 910 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
912 911 return 0
913 912
914 913 currentPointer = self.fp.tell()
915 914
916 915 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917 916
918 917 for nTries in range( self.nTries ):
919 918
920 919 self.fp.close()
921 920 self.fp = open( self.filename, 'rb' )
922 921 self.fp.seek( currentPointer )
923 922
924 923 self.fileSize = os.path.getsize( self.filename )
925 924 currentSize = self.fileSize - currentPointer
926 925
927 926 if ( currentSize >= neededSize ):
928 927 self.basicHeaderObj.read(self.fp)
929 928 return 1
930 929
931 930 if self.fileSize == self.fileSizeByHeader:
932 931 # self.flagEoF = True
933 932 return 0
934 933
935 934 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
936 935 sleep( self.delay )
937 936
938 937
939 938 return 0
940 939
941 940 def waitDataBlock(self,pointer_location):
942 941
943 942 currentPointer = pointer_location
944 943
945 944 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
946 945
947 946 for nTries in range( self.nTries ):
948 947 self.fp.close()
949 948 self.fp = open( self.filename, 'rb' )
950 949 self.fp.seek( currentPointer )
951 950
952 951 self.fileSize = os.path.getsize( self.filename )
953 952 currentSize = self.fileSize - currentPointer
954 953
955 954 if ( currentSize >= neededSize ):
956 955 return 1
957 956
958 957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
959 958 sleep( self.delay )
960 959
961 960 return 0
962 961
963 962 def __jumpToLastBlock(self):
964 963
965 964 if not(self.__isFirstTimeOnline):
966 965 return
967 966
968 967 csize = self.fileSize - self.fp.tell()
969 968 blocksize = self.processingHeaderObj.blockSize
970 969
971 970 #salta el primer bloque de datos
972 971 if csize > self.processingHeaderObj.blockSize:
973 972 self.fp.seek(self.fp.tell() + blocksize)
974 973 else:
975 974 return
976 975
977 976 csize = self.fileSize - self.fp.tell()
978 977 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
979 978 while True:
980 979
981 980 if self.fp.tell()<self.fileSize:
982 981 self.fp.seek(self.fp.tell() + neededsize)
983 982 else:
984 983 self.fp.seek(self.fp.tell() - neededsize)
985 984 break
986 985
987 986 # csize = self.fileSize - self.fp.tell()
988 987 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
989 988 # factor = int(csize/neededsize)
990 989 # if factor > 0:
991 990 # self.fp.seek(self.fp.tell() + factor*neededsize)
992 991
993 992 self.flagIsNewFile = 0
994 993 self.__isFirstTimeOnline = 0
995 994
996 995 def __setNewBlock(self):
997 996
998 997 if self.fp == None:
999 998 return 0
1000 999
1001 1000 # if self.online:
1002 1001 # self.__jumpToLastBlock()
1003 1002
1004 1003 if self.flagIsNewFile:
1005 1004 self.lastUTTime = self.basicHeaderObj.utc
1006 1005 return 1
1007 1006
1008 1007 if self.realtime:
1009 1008 self.flagDiscontinuousBlock = 1
1010 1009 if not(self.setNextFile()):
1011 1010 return 0
1012 1011 else:
1013 1012 return 1
1014 1013
1015 1014 currentSize = self.fileSize - self.fp.tell()
1016 1015 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1017 1016
1018 1017 if (currentSize >= neededSize):
1019 1018 self.basicHeaderObj.read(self.fp)
1020 1019 self.lastUTTime = self.basicHeaderObj.utc
1021 1020 return 1
1022 1021
1023 1022 if self.__waitNewBlock():
1024 1023 self.lastUTTime = self.basicHeaderObj.utc
1025 1024 return 1
1026 1025
1027 1026 if not(self.setNextFile()):
1028 1027 return 0
1029 1028
1030 1029 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1031 1030 self.lastUTTime = self.basicHeaderObj.utc
1032 1031
1033 1032 self.flagDiscontinuousBlock = 0
1034 1033
1035 1034 if deltaTime > self.maxTimeStep:
1036 1035 self.flagDiscontinuousBlock = 1
1037 1036
1038 1037 return 1
1039 1038
1040 1039 def readNextBlock(self):
1041 1040
1042 1041 #Skip block out of startTime and endTime
1043 1042 while True:
1044 1043 if not(self.__setNewBlock()):
1045 1044 return 0
1046 1045
1047 1046 if not(self.readBlock()):
1048 1047 return 0
1049 1048
1050 1049 self.getBasicHeader()
1051 1050
1052 1051 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1053 1052
1054 1053 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1055 1054 self.processingHeaderObj.dataBlocksPerFile,
1056 1055 self.dataOut.datatime.ctime())
1057 1056 continue
1058 1057
1059 1058 break
1060 1059
1061 1060 if self.verbose:
1062 1061 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1063 1062 self.processingHeaderObj.dataBlocksPerFile,
1064 1063 self.dataOut.datatime.ctime())
1065 1064 return 1
1066 1065
1067 1066 def __readFirstHeader(self):
1068 1067
1069 1068 self.basicHeaderObj.read(self.fp)
1070 1069 self.systemHeaderObj.read(self.fp)
1071 1070 self.radarControllerHeaderObj.read(self.fp)
1072 1071 self.processingHeaderObj.read(self.fp)
1073 1072
1074 1073 self.firstHeaderSize = self.basicHeaderObj.size
1075 1074
1076 1075 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1077 1076 if datatype == 0:
1078 1077 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1079 1078 elif datatype == 1:
1080 1079 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1081 1080 elif datatype == 2:
1082 1081 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1083 1082 elif datatype == 3:
1084 1083 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1085 1084 elif datatype == 4:
1086 1085 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1087 1086 elif datatype == 5:
1088 1087 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1089 1088 else:
1090 1089 raise ValueError, 'Data type was not defined'
1091 1090
1092 1091 self.dtype = datatype_str
1093 1092 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1094 1093 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1095 1094 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1096 1095 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1097 1096 self.getBlockDimension()
1098 1097
1099 1098 def __verifyFile(self, filename, msgFlag=True):
1100 1099
1101 1100 msg = None
1102 1101
1103 1102 try:
1104 1103 fp = open(filename, 'rb')
1105 1104 except IOError:
1106 1105
1107 1106 if msgFlag:
1108 1107 print "[Reading] File %s can't be opened" % (filename)
1109 1108
1110 1109 return False
1111 1110
1112 1111 currentPosition = fp.tell()
1113 1112 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1114 1113
1115 1114 if neededSize == 0:
1116 1115 basicHeaderObj = BasicHeader(LOCALTIME)
1117 1116 systemHeaderObj = SystemHeader()
1118 1117 radarControllerHeaderObj = RadarControllerHeader()
1119 1118 processingHeaderObj = ProcessingHeader()
1120 1119
1121 1120 if not( basicHeaderObj.read(fp) ):
1122 1121 fp.close()
1123 1122 return False
1124 1123
1125 1124 if not( systemHeaderObj.read(fp) ):
1126 1125 fp.close()
1127 1126 return False
1128 1127
1129 1128 if not( radarControllerHeaderObj.read(fp) ):
1130 1129 fp.close()
1131 1130 return False
1132 1131
1133 1132 if not( processingHeaderObj.read(fp) ):
1134 1133 fp.close()
1135 1134 return False
1136 1135
1137 1136 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1138 1137 else:
1139 1138 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1140 1139
1141 1140 fp.close()
1142 1141
1143 1142 fileSize = os.path.getsize(filename)
1144 1143 currentSize = fileSize - currentPosition
1145 1144
1146 1145 if currentSize < neededSize:
1147 1146 if msgFlag and (msg != None):
1148 1147 print msg
1149 1148 return False
1150 1149
1151 1150 return True
1152 1151
1153 1152 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1154 1153
1155 1154 path_empty = True
1156 1155
1157 1156 dateList = []
1158 1157 pathList = []
1159 1158
1160 1159 multi_path = path.split(',')
1161 1160
1162 1161 if not walk:
1163 1162
1164 1163 for single_path in multi_path:
1165 1164
1166 1165 if not os.path.isdir(single_path):
1167 1166 continue
1168 1167
1169 1168 fileList = glob.glob1(single_path, "*"+ext)
1170 1169
1171 1170 if not fileList:
1172 1171 continue
1173 1172
1174 1173 path_empty = False
1175 1174
1176 1175 fileList.sort()
1177 1176
1178 1177 for thisFile in fileList:
1179 1178
1180 1179 if not os.path.isfile(os.path.join(single_path, thisFile)):
1181 1180 continue
1182 1181
1183 1182 if not isRadarFile(thisFile):
1184 1183 continue
1185 1184
1186 1185 if not isFileInDateRange(thisFile, startDate, endDate):
1187 1186 continue
1188 1187
1189 1188 thisDate = getDateFromRadarFile(thisFile)
1190 1189
1191 1190 if thisDate in dateList:
1192 1191 continue
1193 1192
1194 1193 dateList.append(thisDate)
1195 1194 pathList.append(single_path)
1196 1195
1197 1196 else:
1198 1197 for single_path in multi_path:
1199 1198
1200 1199 if not os.path.isdir(single_path):
1201 1200 continue
1202 1201
1203 1202 dirList = []
1204 1203
1205 1204 for thisPath in os.listdir(single_path):
1206 1205
1207 1206 if not os.path.isdir(os.path.join(single_path,thisPath)):
1208 1207 continue
1209 1208
1210 1209 if not isRadarFolder(thisPath):
1211 1210 continue
1212 1211
1213 1212 if not isFolderInDateRange(thisPath, startDate, endDate):
1214 1213 continue
1215 1214
1216 1215 dirList.append(thisPath)
1217 1216
1218 1217 if not dirList:
1219 1218 continue
1220 1219
1221 1220 dirList.sort()
1222 1221
1223 1222 for thisDir in dirList:
1224 1223
1225 1224 datapath = os.path.join(single_path, thisDir, expLabel)
1226 1225 fileList = glob.glob1(datapath, "*"+ext)
1227 1226
1228 1227 if not fileList:
1229 1228 continue
1230 1229
1231 1230 path_empty = False
1232 1231
1233 1232 thisDate = getDateFromRadarFolder(thisDir)
1234 1233
1235 1234 pathList.append(datapath)
1236 1235 dateList.append(thisDate)
1237 1236
1238 1237 dateList.sort()
1239 1238
1240 1239 if walk:
1241 1240 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1242 1241 else:
1243 1242 pattern_path = multi_path[0]
1244 1243
1245 1244 if path_empty:
1246 1245 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1247 1246 else:
1248 1247 if not dateList:
1249 1248 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1250 1249
1251 1250 if include_path:
1252 1251 return dateList, pathList
1253 1252
1254 1253 return dateList
1255 1254
1256 1255 def setup(self,
1257 1256 path=None,
1258 1257 startDate=None,
1259 1258 endDate=None,
1260 1259 startTime=datetime.time(0,0,0),
1261 1260 endTime=datetime.time(23,59,59),
1262 1261 set=None,
1263 1262 expLabel = "",
1264 1263 ext = None,
1265 1264 online = False,
1266 1265 delay = 60,
1267 1266 walk = True,
1268 1267 getblock = False,
1269 1268 nTxs = 1,
1270 1269 realtime=False,
1271 1270 blocksize=None,
1272 1271 blocktime=None,
1273 1272 queue=None,
1274 1273 skip=None,
1275 1274 cursor=None,
1276 1275 warnings=True,
1277 verbose=True):
1276 verbose=True,
1277 server=None):
1278 1278
1279 if server is not None:
1280 server = kwargs.get('server', 'zmq.pipe')
1281 if 'tcp://' in server:
1282 address = server
1283 else:
1284 address = 'ipc:///tmp/%s' % server
1285 self.address = address
1286 self.context = zmq.Context()
1287 self.receiver = self.context.socket(zmq.PULL)
1288 self.receiver.bind(self.address)
1289 time.sleep(0.5)
1290 print '[Starting] ReceiverData from {}'.format(self.address)
1291 else:
1279 1292 if path == None:
1280 1293 raise ValueError, "[Reading] The path is not valid"
1281 1294
1282 1295 if ext == None:
1283 1296 ext = self.ext
1284 1297
1285 1298 if online:
1286 1299 print "[Reading] Searching files in online mode..."
1287 1300
1288 1301 for nTries in range( self.nTries ):
1289 1302 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1290 1303
1291 1304 if fullpath:
1292 1305 break
1293 1306
1294 1307 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1295 1308 sleep( self.delay )
1296 1309
1297 1310 if not(fullpath):
1298 1311 print "[Reading] There 'isn't any valid file in %s" % path
1299 1312 return
1300 1313
1301 1314 self.year = year
1302 1315 self.doy = doy
1303 1316 self.set = set - 1
1304 1317 self.path = path
1305 1318 self.foldercounter = foldercounter
1306 1319 last_set = None
1307
1308 1320 else:
1309 1321 print "[Reading] Searching files in offline mode ..."
1310 1322 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1311 1323 startTime=startTime, endTime=endTime,
1312 1324 set=set, expLabel=expLabel, ext=ext,
1313 1325 walk=walk, cursor=cursor,
1314 1326 skip=skip, queue=queue)
1315 1327
1316 1328 if not(pathList):
1317 1329 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1318 1330 # datetime.datetime.combine(startDate,startTime).ctime(),
1319 1331 # datetime.datetime.combine(endDate,endTime).ctime())
1320 1332
1321 1333 # sys.exit(-1)
1322 1334
1323 1335 self.fileIndex = -1
1324 1336 self.pathList = []
1325 1337 self.filenameList = []
1326 1338 return
1327 1339
1328 1340 self.fileIndex = -1
1329 1341 self.pathList = pathList
1330 1342 self.filenameList = filenameList
1331 1343 file_name = os.path.basename(filenameList[-1])
1332 1344 basename, ext = os.path.splitext(file_name)
1333 1345 last_set = int(basename[-3:])
1334 1346
1335 1347 self.online = online
1336 1348 self.realtime = realtime
1337 1349 self.delay = delay
1338 1350 ext = ext.lower()
1339 1351 self.ext = ext
1340 1352 self.getByBlock = getblock
1341 1353 self.nTxs = nTxs
1342 1354 self.startTime = startTime
1343 1355 self.endTime = endTime
1344 1356
1345 1357 #Added-----------------
1346 1358 self.selBlocksize = blocksize
1347 1359 self.selBlocktime = blocktime
1348 1360
1349 1361 # Verbose-----------
1350 1362 self.verbose = verbose
1351 1363 self.warnings = warnings
1352 1364
1353 1365 if not(self.setNextFile()):
1354 1366 if (startDate!=None) and (endDate!=None):
1355 1367 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1356 1368 elif startDate != None:
1357 1369 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1358 1370 else:
1359 1371 print "[Reading] No files"
1360 1372
1361 1373 self.fileIndex = -1
1362 1374 self.pathList = []
1363 1375 self.filenameList = []
1364 1376 return
1365 1377
1366 1378 # self.getBasicHeader()
1367 1379
1368 1380 if last_set != None:
1369 1381 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1370 1382 return
1371 1383
1372 1384 def getBasicHeader(self):
1373 1385
1374 1386 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1375 1387
1376 1388 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1377 1389
1378 1390 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1379 1391
1380 1392 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1381 1393
1382 1394 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1383 1395
1384 1396 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1385 1397
1386 1398 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1387 1399
1388 1400 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1389 1401
1390 1402
1391 1403 def getFirstHeader(self):
1392 1404
1393 1405 raise NotImplementedError
1394 1406
1395 1407 def getData(self):
1396 1408
1397 1409 raise NotImplementedError
1398 1410
1399 1411 def hasNotDataInBuffer(self):
1400 1412
1401 1413 raise NotImplementedError
1402 1414
1403 1415 def readBlock(self):
1404 1416
1405 1417 raise NotImplementedError
1406 1418
1407 1419 def isEndProcess(self):
1408 1420
1409 1421 return self.flagNoMoreFiles
1410 1422
1411 1423 def printReadBlocks(self):
1412 1424
1413 1425 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1414 1426
1415 1427 def printTotalBlocks(self):
1416 1428
1417 1429 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1418 1430
1419 1431 def printNumberOfBlock(self):
1420 1432
1421 1433 if self.flagIsNewBlock:
1422 1434 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1423 1435 self.processingHeaderObj.dataBlocksPerFile,
1424 1436 self.dataOut.datatime.ctime())
1425 1437
1426 1438 def printInfo(self):
1427 1439
1428 1440 if self.__printInfo == False:
1429 1441 return
1430 1442
1431 1443 self.basicHeaderObj.printInfo()
1432 1444 self.systemHeaderObj.printInfo()
1433 1445 self.radarControllerHeaderObj.printInfo()
1434 1446 self.processingHeaderObj.printInfo()
1435 1447
1436 1448 self.__printInfo = False
1437 1449
1438 1450
1439 1451 def run(self,
1440 1452 path=None,
1441 1453 startDate=None,
1442 1454 endDate=None,
1443 1455 startTime=datetime.time(0,0,0),
1444 1456 endTime=datetime.time(23,59,59),
1445 1457 set=None,
1446 1458 expLabel = "",
1447 1459 ext = None,
1448 1460 online = False,
1449 1461 delay = 60,
1450 1462 walk = True,
1451 1463 getblock = False,
1452 1464 nTxs = 1,
1453 1465 realtime=False,
1454 1466 blocksize=None,
1455 1467 blocktime=None,
1456 1468 queue=None,
1457 1469 skip=None,
1458 1470 cursor=None,
1459 1471 warnings=True,
1472 server=None,
1460 1473 verbose=True, **kwargs):
1461 1474
1462 1475 if not(self.isConfig):
1463 1476 # self.dataOut = dataOut
1464 1477 self.setup( path=path,
1465 1478 startDate=startDate,
1466 1479 endDate=endDate,
1467 1480 startTime=startTime,
1468 1481 endTime=endTime,
1469 1482 set=set,
1470 1483 expLabel=expLabel,
1471 1484 ext=ext,
1472 1485 online=online,
1473 1486 delay=delay,
1474 1487 walk=walk,
1475 1488 getblock=getblock,
1476 1489 nTxs=nTxs,
1477 1490 realtime=realtime,
1478 1491 blocksize=blocksize,
1479 1492 blocktime=blocktime,
1480 1493 queue=queue,
1481 1494 skip=skip,
1482 1495 cursor=cursor,
1483 1496 warnings=warnings,
1497 server=None,
1484 1498 verbose=verbose)
1485 1499 self.isConfig = True
1486
1500 if self.server is None:
1487 1501 self.getData()
1502 else:
1503 self.getFromZMQ()
1488 1504
1489 1505 class JRODataWriter(JRODataIO):
1490 1506
1491 1507 """
1492 1508 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1493 1509 de los datos siempre se realiza por bloques.
1494 1510 """
1495 1511
1496 1512 blockIndex = 0
1497 1513
1498 1514 path = None
1499 1515
1500 1516 setFile = None
1501 1517
1502 1518 profilesPerBlock = None
1503 1519
1504 1520 blocksPerFile = None
1505 1521
1506 1522 nWriteBlocks = 0
1507 1523
1508 1524 fileDate = None
1509 1525
1510 1526 def __init__(self, dataOut=None):
1511 1527 raise NotImplementedError
1512 1528
1513 1529
1514 1530 def hasAllDataInBuffer(self):
1515 1531 raise NotImplementedError
1516 1532
1517 1533
1518 1534 def setBlockDimension(self):
1519 1535 raise NotImplementedError
1520 1536
1521 1537
1522 1538 def writeBlock(self):
1523 1539 raise NotImplementedError
1524 1540
1525 1541
1526 1542 def putData(self):
1527 1543 raise NotImplementedError
1528 1544
1529 1545
1530 1546 def getProcessFlags(self):
1531 1547
1532 1548 processFlags = 0
1533 1549
1534 1550 dtype_index = get_dtype_index(self.dtype)
1535 1551 procflag_dtype = get_procflag_dtype(dtype_index)
1536 1552
1537 1553 processFlags += procflag_dtype
1538 1554
1539 1555 if self.dataOut.flagDecodeData:
1540 1556 processFlags += PROCFLAG.DECODE_DATA
1541 1557
1542 1558 if self.dataOut.flagDeflipData:
1543 1559 processFlags += PROCFLAG.DEFLIP_DATA
1544 1560
1545 1561 if self.dataOut.code is not None:
1546 1562 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1547 1563
1548 1564 if self.dataOut.nCohInt > 1:
1549 1565 processFlags += PROCFLAG.COHERENT_INTEGRATION
1550 1566
1551 1567 if self.dataOut.type == "Spectra":
1552 1568 if self.dataOut.nIncohInt > 1:
1553 1569 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1554 1570
1555 1571 if self.dataOut.data_dc is not None:
1556 1572 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1557 1573
1558 1574 if self.dataOut.flagShiftFFT:
1559 1575 processFlags += PROCFLAG.SHIFT_FFT_DATA
1560 1576
1561 1577 return processFlags
1562 1578
1563 1579 def setBasicHeader(self):
1564 1580
1565 1581 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1566 1582 self.basicHeaderObj.version = self.versionFile
1567 1583 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1568 1584
1569 1585 utc = numpy.floor(self.dataOut.utctime)
1570 1586 milisecond = (self.dataOut.utctime - utc)* 1000.0
1571 1587
1572 1588 self.basicHeaderObj.utc = utc
1573 1589 self.basicHeaderObj.miliSecond = milisecond
1574 1590 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1575 1591 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1576 1592 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1577 1593
1578 1594 def setFirstHeader(self):
1579 1595 """
1580 1596 Obtiene una copia del First Header
1581 1597
1582 1598 Affected:
1583 1599
1584 1600 self.basicHeaderObj
1585 1601 self.systemHeaderObj
1586 1602 self.radarControllerHeaderObj
1587 1603 self.processingHeaderObj self.
1588 1604
1589 1605 Return:
1590 1606 None
1591 1607 """
1592 1608
1593 1609 raise NotImplementedError
1594 1610
1595 1611 def __writeFirstHeader(self):
1596 1612 """
1597 1613 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1598 1614
1599 1615 Affected:
1600 1616 __dataType
1601 1617
1602 1618 Return:
1603 1619 None
1604 1620 """
1605 1621
1606 1622 # CALCULAR PARAMETROS
1607 1623
1608 1624 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1609 1625 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1610 1626
1611 1627 self.basicHeaderObj.write(self.fp)
1612 1628 self.systemHeaderObj.write(self.fp)
1613 1629 self.radarControllerHeaderObj.write(self.fp)
1614 1630 self.processingHeaderObj.write(self.fp)
1615 1631
1616 1632 def __setNewBlock(self):
1617 1633 """
1618 1634 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1619 1635
1620 1636 Return:
1621 1637 0 : si no pudo escribir nada
1622 1638 1 : Si escribio el Basic el First Header
1623 1639 """
1624 1640 if self.fp == None:
1625 1641 self.setNextFile()
1626 1642
1627 1643 if self.flagIsNewFile:
1628 1644 return 1
1629 1645
1630 1646 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1631 1647 self.basicHeaderObj.write(self.fp)
1632 1648 return 1
1633 1649
1634 1650 if not( self.setNextFile() ):
1635 1651 return 0
1636 1652
1637 1653 return 1
1638 1654
1639 1655
1640 1656 def writeNextBlock(self):
1641 1657 """
1642 1658 Selecciona el bloque siguiente de datos y los escribe en un file
1643 1659
1644 1660 Return:
1645 1661 0 : Si no hizo pudo escribir el bloque de datos
1646 1662 1 : Si no pudo escribir el bloque de datos
1647 1663 """
1648 1664 if not( self.__setNewBlock() ):
1649 1665 return 0
1650 1666
1651 1667 self.writeBlock()
1652 1668
1653 1669 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1654 1670 self.processingHeaderObj.dataBlocksPerFile)
1655 1671
1656 1672 return 1
1657 1673
1658 1674 def setNextFile(self):
1659 1675 """
1660 1676 Determina el siguiente file que sera escrito
1661 1677
1662 1678 Affected:
1663 1679 self.filename
1664 1680 self.subfolder
1665 1681 self.fp
1666 1682 self.setFile
1667 1683 self.flagIsNewFile
1668 1684
1669 1685 Return:
1670 1686 0 : Si el archivo no puede ser escrito
1671 1687 1 : Si el archivo esta listo para ser escrito
1672 1688 """
1673 1689 ext = self.ext
1674 1690 path = self.path
1675 1691
1676 1692 if self.fp != None:
1677 1693 self.fp.close()
1678 1694
1679 1695 timeTuple = time.localtime( self.dataOut.utctime)
1680 1696 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1681 1697
1682 1698 fullpath = os.path.join( path, subfolder )
1683 1699 setFile = self.setFile
1684 1700
1685 1701 if not( os.path.exists(fullpath) ):
1686 1702 os.mkdir(fullpath)
1687 1703 setFile = -1 #inicializo mi contador de seteo
1688 1704 else:
1689 1705 filesList = os.listdir( fullpath )
1690 1706 if len( filesList ) > 0:
1691 1707 filesList = sorted( filesList, key=str.lower )
1692 1708 filen = filesList[-1]
1693 1709 # el filename debera tener el siguiente formato
1694 1710 # 0 1234 567 89A BCDE (hex)
1695 1711 # x YYYY DDD SSS .ext
1696 1712 if isNumber( filen[8:11] ):
1697 1713 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1698 1714 else:
1699 1715 setFile = -1
1700 1716 else:
1701 1717 setFile = -1 #inicializo mi contador de seteo
1702 1718
1703 1719 setFile += 1
1704 1720
1705 1721 #If this is a new day it resets some values
1706 1722 if self.dataOut.datatime.date() > self.fileDate:
1707 1723 setFile = 0
1708 1724 self.nTotalBlocks = 0
1709 1725
1710 1726 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1711 1727
1712 1728 filename = os.path.join( path, subfolder, filen )
1713 1729
1714 1730 fp = open( filename,'wb' )
1715 1731
1716 1732 self.blockIndex = 0
1717 1733
1718 1734 #guardando atributos
1719 1735 self.filename = filename
1720 1736 self.subfolder = subfolder
1721 1737 self.fp = fp
1722 1738 self.setFile = setFile
1723 1739 self.flagIsNewFile = 1
1724 1740 self.fileDate = self.dataOut.datatime.date()
1725 1741
1726 1742 self.setFirstHeader()
1727 1743
1728 1744 print '[Writing] Opening file: %s'%self.filename
1729 1745
1730 1746 self.__writeFirstHeader()
1731 1747
1732 1748 return 1
1733 1749
1734 1750 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1735 1751 """
1736 1752 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1737 1753
1738 1754 Inputs:
1739 1755 path : directory where data will be saved
1740 1756 profilesPerBlock : number of profiles per block
1741 1757 set : initial file set
1742 1758 datatype : An integer number that defines data type:
1743 1759 0 : int8 (1 byte)
1744 1760 1 : int16 (2 bytes)
1745 1761 2 : int32 (4 bytes)
1746 1762 3 : int64 (8 bytes)
1747 1763 4 : float32 (4 bytes)
1748 1764 5 : double64 (8 bytes)
1749 1765
1750 1766 Return:
1751 1767 0 : Si no realizo un buen seteo
1752 1768 1 : Si realizo un buen seteo
1753 1769 """
1754 1770
1755 1771 if ext == None:
1756 1772 ext = self.ext
1757 1773
1758 1774 self.ext = ext.lower()
1759 1775
1760 1776 self.path = path
1761 1777
1762 1778 if set is None:
1763 1779 self.setFile = -1
1764 1780 else:
1765 1781 self.setFile = set - 1
1766 1782
1767 1783 self.blocksPerFile = blocksPerFile
1768 1784
1769 1785 self.profilesPerBlock = profilesPerBlock
1770 1786
1771 1787 self.dataOut = dataOut
1772 1788 self.fileDate = self.dataOut.datatime.date()
1773 1789 #By default
1774 1790 self.dtype = self.dataOut.dtype
1775 1791
1776 1792 if datatype is not None:
1777 1793 self.dtype = get_numpy_dtype(datatype)
1778 1794
1779 1795 if not(self.setNextFile()):
1780 1796 print "[Writing] There isn't a next file"
1781 1797 return 0
1782 1798
1783 1799 self.setBlockDimension()
1784 1800
1785 1801 return 1
1786 1802
1787 1803 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1788 1804
1789 1805 if not(self.isConfig):
1790 1806
1791 1807 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1792 1808 self.isConfig = True
1793 1809
1794 1810 self.putData()
Show file before | Show file after | Hide comments
@@ -1,637 +1,644
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 import zmq
13 14 # from _sha import blocksize
14 15
15 16 class VoltageReader(JRODataReader, ProcessingUnit):
16 17 """
17 18 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
18 19 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
19 20 perfiles*alturas*canales) son almacenados en la variable "buffer".
20 21
21 22 perfiles * alturas * canales
22 23
23 24 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
24 25 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
25 26 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
26 27 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
27 28
28 29 Example:
29 30
30 31 dpath = "/home/myuser/data"
31 32
32 33 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33 34
34 35 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35 36
36 37 readerObj = VoltageReader()
37 38
38 39 readerObj.setup(dpath, startTime, endTime)
39 40
40 41 while(True):
41 42
42 43 #to get one profile
43 44 profile = readerObj.getData()
44 45
45 46 #print the profile
46 47 print profile
47 48
48 49 #If you want to see all datablock
49 50 print readerObj.datablock
50 51
51 52 if readerObj.flagNoMoreFiles:
52 53 break
53 54
54 55 """
55 56
56 57 ext = ".r"
57 58
58 59 optchar = "D"
59 60 dataOut = None
60 61
61 62 def __init__(self, **kwargs):
62 63 """
63 64 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
64 65
65 66 Input:
66 67 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
67 68 almacenar un perfil de datos cada vez que se haga un requerimiento
68 69 (getData). El perfil sera obtenido a partir del buffer de datos,
69 70 si el buffer esta vacio se hara un nuevo proceso de lectura de un
70 71 bloque de datos.
71 72 Si este parametro no es pasado se creara uno internamente.
72 73
73 74 Variables afectadas:
74 75 self.dataOut
75 76
76 77 Return:
77 78 None
78 79 """
79 80
80 81 ProcessingUnit.__init__(self, **kwargs)
81 82
82 83 self.isConfig = False
83 84
84 85 self.datablock = None
85 86
86 87 self.utc = 0
87 88
88 89 self.ext = ".r"
89 90
90 91 self.optchar = "D"
91 92
92 93 self.basicHeaderObj = BasicHeader(LOCALTIME)
93 94
94 95 self.systemHeaderObj = SystemHeader()
95 96
96 97 self.radarControllerHeaderObj = RadarControllerHeader()
97 98
98 99 self.processingHeaderObj = ProcessingHeader()
99 100
100 101 self.online = 0
101 102
102 103 self.fp = None
103 104
104 105 self.idFile = None
105 106
106 107 self.dtype = None
107 108
108 109 self.fileSizeByHeader = None
109 110
110 111 self.filenameList = []
111 112
112 113 self.filename = None
113 114
114 115 self.fileSize = None
115 116
116 117 self.firstHeaderSize = 0
117 118
118 119 self.basicHeaderSize = 24
119 120
120 121 self.pathList = []
121 122
122 123 self.filenameList = []
123 124
124 125 self.lastUTTime = 0
125 126
126 127 self.maxTimeStep = 30
127 128
128 129 self.flagNoMoreFiles = 0
129 130
130 131 self.set = 0
131 132
132 133 self.path = None
133 134
134 135 self.profileIndex = 2**32-1
135 136
136 137 self.delay = 3 #seconds
137 138
138 139 self.nTries = 3 #quantity tries
139 140
140 141 self.nFiles = 3 #number of files for searching
141 142
142 143 self.nReadBlocks = 0
143 144
144 145 self.flagIsNewFile = 1
145 146
146 147 self.__isFirstTimeOnline = 1
147 148
148 149 # self.ippSeconds = 0
149 150
150 151 self.flagDiscontinuousBlock = 0
151 152
152 153 self.flagIsNewBlock = 0
153 154
154 155 self.nTotalBlocks = 0
155 156
156 157 self.blocksize = 0
157 158
158 159 self.dataOut = self.createObjByDefault()
159 160
160 161 self.nTxs = 1
161 162
162 163 self.txIndex = 0
163 164
164 165 def createObjByDefault(self):
165 166
166 167 dataObj = Voltage()
167 168
168 169 return dataObj
169 170
170 171 def __hasNotDataInBuffer(self):
171 172
172 173 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
173 174 return 1
174 175
175 176 return 0
176 177
177 178
178 179 def getBlockDimension(self):
179 180 """
180 181 Obtiene la cantidad de puntos a leer por cada bloque de datos
181 182
182 183 Affected:
183 184 self.blocksize
184 185
185 186 Return:
186 187 None
187 188 """
188 189 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
189 190 self.blocksize = pts2read
190 191
191 192
192 193 def readBlock(self):
193 194 """
194 195 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
195 196 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
196 197 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
197 198 es seteado a 0
198 199
199 200 Inputs:
200 201 None
201 202
202 203 Return:
203 204 None
204 205
205 206 Affected:
206 207 self.profileIndex
207 208 self.datablock
208 209 self.flagIsNewFile
209 210 self.flagIsNewBlock
210 211 self.nTotalBlocks
211 212
212 213 Exceptions:
213 214 Si un bloque leido no es un bloque valido
214 215 """
215 216 current_pointer_location = self.fp.tell()
216 217 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217 218
218 219 try:
219 220 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
220 221 except:
221 222 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222 223
223 224 if self.waitDataBlock(pointer_location=current_pointer_location):
224 225 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
225 226 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
226 227 # return 0
227 228
228 229 #Dimensions : nChannels, nProfiles, nSamples
229 230
230 231 junk = numpy.transpose(junk, (2,0,1))
231 232 self.datablock = junk['real'] + junk['imag']*1j
232 233
233 234 self.profileIndex = 0
234 235
235 236 self.flagIsNewFile = 0
236 237 self.flagIsNewBlock = 1
237 238
238 239 self.nTotalBlocks += 1
239 240 self.nReadBlocks += 1
240 241
241 242 return 1
242 243
243 244 def getFirstHeader(self):
244 245
245 246 self.getBasicHeader()
246 247
247 248 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
248 249
249 250 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
250 251
251 252 if self.nTxs > 1:
252 253 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
253 254
254 255 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
255 256
256 257 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
257 258 #
258 259 # if self.radarControllerHeaderObj.code is not None:
259 260 #
260 261 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
261 262 #
262 263 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
263 264 #
264 265 # self.dataOut.code = self.radarControllerHeaderObj.code
265 266
266 267 self.dataOut.dtype = self.dtype
267 268
268 269 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
269 270
270 271 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
271 272
272 273 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
273 274
274 275 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
275 276
276 277 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
277 278
278 279 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
279 280
280 281 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
281 282
282 283 def reshapeData(self):
283 284
284 285 if self.nTxs < 0:
285 286 return
286 287
287 288 if self.nTxs == 1:
288 289 return
289 290
290 291 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
291 292 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
292 293
293 294 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
294 295 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
295 296
296 297 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
297 298
298 299 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
299 300 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
300 301 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
301 302
302 303 return
303 304
305
306 def getFromZMQ(self):
307 self.dataOut = self.receiver.recv_pyobj()
308 print '[Receiving] {} - {}'.format(self.dataOut.type,
309 self.dataOut.datatime.ctime())
310
304 311 def getData(self):
305 312 """
306 313 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
307 314 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
308 315 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
309 316 "readNextBlock"
310 317
311 318 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
312 319
313 320 Return:
314 321
315 322 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
316 323 es igual al total de perfiles leidos desde el archivo.
317 324
318 325 Si self.getByBlock == False:
319 326
320 327 self.dataOut.data = buffer[:, thisProfile, :]
321 328
322 329 shape = [nChannels, nHeis]
323 330
324 331 Si self.getByBlock == True:
325 332
326 333 self.dataOut.data = buffer[:, :, :]
327 334
328 335 shape = [nChannels, nProfiles, nHeis]
329 336
330 337 Variables afectadas:
331 338 self.dataOut
332 339 self.profileIndex
333 340
334 341 Affected:
335 342 self.dataOut
336 343 self.profileIndex
337 344 self.flagDiscontinuousBlock
338 345 self.flagIsNewBlock
339 346 """
340 347
341 348 if self.flagNoMoreFiles:
342 349 self.dataOut.flagNoData = True
343 350 print 'Process finished'
344 351 return 0
345 352
346 353 self.flagDiscontinuousBlock = 0
347 354 self.flagIsNewBlock = 0
348 355
349 356 if self.__hasNotDataInBuffer():
350 357
351 358 if not( self.readNextBlock() ):
352 359 return 0
353 360
354 361 self.getFirstHeader()
355 362
356 363 self.reshapeData()
357 364
358 365 if self.datablock is None:
359 366 self.dataOut.flagNoData = True
360 367 return 0
361 368
362 369 if not self.getByBlock:
363 370
364 371 """
365 372 Return profile by profile
366 373
367 374 If nTxs > 1 then one profile is divided by nTxs and number of total
368 375 blocks is increased by nTxs (nProfiles *= nTxs)
369 376 """
370 377 self.dataOut.flagDataAsBlock = False
371 378 self.dataOut.data = self.datablock[:,self.profileIndex,:]
372 379 self.dataOut.profileIndex = self.profileIndex
373 380
374 381 self.profileIndex += 1
375 382
376 383 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
377 384 # """
378 385 # Return all block
379 386 # """
380 387 # self.dataOut.flagDataAsBlock = True
381 388 # self.dataOut.data = self.datablock
382 389 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
383 390 #
384 391 # self.profileIndex = self.dataOut.nProfiles
385 392
386 393 else:
387 394 """
388 395 Return a block
389 396 """
390 397 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 398 if self.selBlocktime != None:
392 399 if self.dataOut.nCohInt is not None:
393 400 nCohInt = self.dataOut.nCohInt
394 401 else:
395 402 nCohInt = 1
396 403 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397 404
398 405 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
399 406 self.profileIndex += self.selBlocksize
400 407 datasize = self.dataOut.data.shape[1]
401 408
402 409 if datasize < self.selBlocksize:
403 410 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
404 411 buffer[:,:datasize,:] = self.dataOut.data
405 412
406 413 while datasize < self.selBlocksize: #Not enough profiles to fill the block
407 414 if not( self.readNextBlock() ):
408 415 return 0
409 416 self.getFirstHeader()
410 417 self.reshapeData()
411 418 if self.datablock is None:
412 419 self.dataOut.flagNoData = True
413 420 return 0
414 421 #stack data
415 422 blockIndex = self.selBlocksize - datasize
416 423 datablock1 = self.datablock[:,:blockIndex,:]
417 424
418 425 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
419 426 datasize += datablock1.shape[1]
420 427
421 428 self.dataOut.data = buffer
422 429 self.profileIndex = blockIndex
423 430
424 431 self.dataOut.flagDataAsBlock = True
425 432 self.dataOut.nProfiles = self.dataOut.data.shape[1]
426 433
427 434 self.dataOut.flagNoData = False
428 435
429 436 self.getBasicHeader()
430 437
431 438 self.dataOut.realtime = self.online
432 439
433 440 return self.dataOut.data
434 441
435 442 class VoltageWriter(JRODataWriter, Operation):
436 443 """
437 444 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
438 445 de los datos siempre se realiza por bloques.
439 446 """
440 447
441 448 ext = ".r"
442 449
443 450 optchar = "D"
444 451
445 452 shapeBuffer = None
446 453
447 454
448 455 def __init__(self, **kwargs):
449 456 """
450 457 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
451 458
452 459 Affected:
453 460 self.dataOut
454 461
455 462 Return: None
456 463 """
457 464 Operation.__init__(self, **kwargs)
458 465
459 466 self.nTotalBlocks = 0
460 467
461 468 self.profileIndex = 0
462 469
463 470 self.isConfig = False
464 471
465 472 self.fp = None
466 473
467 474 self.flagIsNewFile = 1
468 475
469 476 self.blockIndex = 0
470 477
471 478 self.flagIsNewBlock = 0
472 479
473 480 self.setFile = None
474 481
475 482 self.dtype = None
476 483
477 484 self.path = None
478 485
479 486 self.filename = None
480 487
481 488 self.basicHeaderObj = BasicHeader(LOCALTIME)
482 489
483 490 self.systemHeaderObj = SystemHeader()
484 491
485 492 self.radarControllerHeaderObj = RadarControllerHeader()
486 493
487 494 self.processingHeaderObj = ProcessingHeader()
488 495
489 496 def hasAllDataInBuffer(self):
490 497 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
491 498 return 1
492 499 return 0
493 500
494 501
495 502 def setBlockDimension(self):
496 503 """
497 504 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
498 505
499 506 Affected:
500 507 self.shape_spc_Buffer
501 508 self.shape_cspc_Buffer
502 509 self.shape_dc_Buffer
503 510
504 511 Return: None
505 512 """
506 513 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
507 514 self.processingHeaderObj.nHeights,
508 515 self.systemHeaderObj.nChannels)
509 516
510 517 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
511 518 self.processingHeaderObj.profilesPerBlock,
512 519 self.processingHeaderObj.nHeights),
513 520 dtype=numpy.dtype('complex64'))
514 521
515 522 def writeBlock(self):
516 523 """
517 524 Escribe el buffer en el file designado
518 525
519 526 Affected:
520 527 self.profileIndex
521 528 self.flagIsNewFile
522 529 self.flagIsNewBlock
523 530 self.nTotalBlocks
524 531 self.blockIndex
525 532
526 533 Return: None
527 534 """
528 535 data = numpy.zeros( self.shapeBuffer, self.dtype )
529 536
530 537 junk = numpy.transpose(self.datablock, (1,2,0))
531 538
532 539 data['real'] = junk.real
533 540 data['imag'] = junk.imag
534 541
535 542 data = data.reshape( (-1) )
536 543
537 544 data.tofile( self.fp )
538 545
539 546 self.datablock.fill(0)
540 547
541 548 self.profileIndex = 0
542 549 self.flagIsNewFile = 0
543 550 self.flagIsNewBlock = 1
544 551
545 552 self.blockIndex += 1
546 553 self.nTotalBlocks += 1
547 554
548 555 # print "[Writing] Block = %04d" %self.blockIndex
549 556
550 557 def putData(self):
551 558 """
552 559 Setea un bloque de datos y luego los escribe en un file
553 560
554 561 Affected:
555 562 self.flagIsNewBlock
556 563 self.profileIndex
557 564
558 565 Return:
559 566 0 : Si no hay data o no hay mas files que puedan escribirse
560 567 1 : Si se escribio la data de un bloque en un file
561 568 """
562 569 if self.dataOut.flagNoData:
563 570 return 0
564 571
565 572 self.flagIsNewBlock = 0
566 573
567 574 if self.dataOut.flagDiscontinuousBlock:
568 575 self.datablock.fill(0)
569 576 self.profileIndex = 0
570 577 self.setNextFile()
571 578
572 579 if self.profileIndex == 0:
573 580 self.setBasicHeader()
574 581
575 582 self.datablock[:,self.profileIndex,:] = self.dataOut.data
576 583
577 584 self.profileIndex += 1
578 585
579 586 if self.hasAllDataInBuffer():
580 587 #if self.flagIsNewFile:
581 588 self.writeNextBlock()
582 589 # self.setFirstHeader()
583 590
584 591 return 1
585 592
586 593 def __getBlockSize(self):
587 594 '''
588 595 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
589 596 '''
590 597
591 598 dtype_width = self.getDtypeWidth()
592 599
593 600 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
594 601
595 602 return blocksize
596 603
597 604 def setFirstHeader(self):
598 605
599 606 """
600 607 Obtiene una copia del First Header
601 608
602 609 Affected:
603 610 self.systemHeaderObj
604 611 self.radarControllerHeaderObj
605 612 self.dtype
606 613
607 614 Return:
608 615 None
609 616 """
610 617
611 618 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
612 619 self.systemHeaderObj.nChannels = self.dataOut.nChannels
613 620 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
614 621
615 622 self.processingHeaderObj.dtype = 0 # Voltage
616 623 self.processingHeaderObj.blockSize = self.__getBlockSize()
617 624 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
618 625 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
619 626 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
620 627 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
621 628 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
622 629 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
623 630
624 631 if self.dataOut.code is not None:
625 632 self.processingHeaderObj.code = self.dataOut.code
626 633 self.processingHeaderObj.nCode = self.dataOut.nCode
627 634 self.processingHeaderObj.nBaud = self.dataOut.nBaud
628 635
629 636 if self.processingHeaderObj.nWindows != 0:
630 637 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
631 638 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
632 639 self.processingHeaderObj.nHeights = self.dataOut.nHeights
633 640 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
634 641
635 642 self.processingHeaderObj.processFlags = self.getProcessFlags()
636 643
637 644 self.setBasicHeader()
General Comments 0
You need to be logged in to leave comments. Login now