##// END OF EJS Templates
schain-jc
RSS
casi merge con online_data_hour
José Chávez -
r1036:754fcef10e66 merge
parent child
Show More
Side by Side Unified
Context file:
r1036:754fcef10e66 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1813 +1,1976
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 14 import traceback
15 15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 24
25 25 LOCALTIME = True
26 26
27 27 def isNumber(cad):
28 28 """
29 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 30
31 31 Excepciones:
32 32 Si un determinado string no puede ser convertido a numero
33 33 Input:
34 34 str, string al cual se le analiza para determinar si convertible a un numero o no
35 35
36 36 Return:
37 37 True : si el string es uno numerico
38 38 False : no es un string numerico
39 39 """
40 40 try:
41 41 float( cad )
42 42 return True
43 43 except:
44 44 return False
45 45
46 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 47 """
48 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 49
50 50 Inputs:
51 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 52
53 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 56 segundos contados desde 01/01/1970.
57 57
58 58 Return:
59 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 60 fecha especificado, de lo contrario retorna False.
61 61
62 62 Excepciones:
63 63 Si el archivo no existe o no puede ser abierto
64 64 Si la cabecera no puede ser leida.
65 65
66 66 """
67 67 basicHeaderObj = BasicHeader(LOCALTIME)
68 68
69 69 try:
70 70 fp = open(filename,'rb')
71 71 except IOError:
72 72 print "The file %s can't be opened" %(filename)
73 73 return 0
74 74
75 75 sts = basicHeaderObj.read(fp)
76 76 fp.close()
77 77
78 78 if not(sts):
79 79 print "Skipping the file %s because it has not a valid header" %(filename)
80 80 return 0
81 81
82 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 83 return 0
84 84
85 85 return 1
86 86
87 87 def isTimeInRange(thisTime, startTime, endTime):
88 88
89 89 if endTime >= startTime:
90 90 if (thisTime < startTime) or (thisTime > endTime):
91 91 return 0
92 92
93 93 return 1
94 94 else:
95 95 if (thisTime < startTime) and (thisTime > endTime):
96 96 return 0
97 97
98 98 return 1
99 99
100 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 101 """
102 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 103
104 104 Inputs:
105 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 106
107 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 108
109 109 endDate : fecha final del rango seleccionado en formato datetime.date
110 110
111 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 112
113 113 endTime : tiempo final del rango seleccionado en formato datetime.time
114 114
115 115 Return:
116 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 117 fecha especificado, de lo contrario retorna False.
118 118
119 119 Excepciones:
120 120 Si el archivo no existe o no puede ser abierto
121 121 Si la cabecera no puede ser leida.
122 122
123 123 """
124 124
125 125
126 126 try:
127 127 fp = open(filename,'rb')
128 128 except IOError:
129 129 print "The file %s can't be opened" %(filename)
130 130 return None
131 131
132 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 133 systemHeaderObj = SystemHeader()
134 134 radarControllerHeaderObj = RadarControllerHeader()
135 135 processingHeaderObj = ProcessingHeader()
136 136
137 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 138
139 139 sts = firstBasicHeaderObj.read(fp)
140 140
141 141 if not(sts):
142 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 143 return None
144 144
145 145 if not systemHeaderObj.read(fp):
146 146 return None
147 147
148 148 if not radarControllerHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not processingHeaderObj.read(fp):
152 152 return None
153 153
154 154 filesize = os.path.getsize(filename)
155 155
156 156 offset = processingHeaderObj.blockSize + 24 #header size
157 157
158 158 if filesize <= offset:
159 159 print "[Reading] %s: This file has not enough data" %filename
160 160 return None
161 161
162 162 fp.seek(-offset, 2)
163 163
164 164 sts = lastBasicHeaderObj.read(fp)
165 165
166 166 fp.close()
167 167
168 168 thisDatetime = lastBasicHeaderObj.datatime
169 169 thisTime_last_block = thisDatetime.time()
170 170
171 171 thisDatetime = firstBasicHeaderObj.datatime
172 172 thisDate = thisDatetime.date()
173 173 thisTime_first_block = thisDatetime.time()
174 174
175 175 #General case
176 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 177 #-----------o----------------------------o-----------
178 178 # startTime endTime
179 179
180 180 if endTime >= startTime:
181 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 182 return None
183 183
184 184 return thisDatetime
185 185
186 186 #If endTime < startTime then endTime belongs to the next day
187 187
188 188
189 189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 190 #-----------o----------------------------o-----------
191 191 # endTime startTime
192 192
193 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 194 return None
195 195
196 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 197 return None
198 198
199 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 200 return None
201 201
202 202 return thisDatetime
203 203
204 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 205 """
206 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 207
208 208 Inputs:
209 209 folder : nombre completo del directorio.
210 210 Su formato deberia ser "/path_root/?YYYYDDD"
211 211
212 212 siendo:
213 213 YYYY : Anio (ejemplo 2015)
214 214 DDD : Dia del anio (ejemplo 305)
215 215
216 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 217
218 218 endDate : fecha final del rango seleccionado en formato datetime.date
219 219
220 220 Return:
221 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 222 fecha especificado, de lo contrario retorna False.
223 223 Excepciones:
224 224 Si el directorio no tiene el formato adecuado
225 225 """
226 226
227 227 basename = os.path.basename(folder)
228 228
229 229 if not isRadarFolder(basename):
230 230 print "The folder %s has not the rigth format" %folder
231 231 return 0
232 232
233 233 if startDate and endDate:
234 234 thisDate = getDateFromRadarFolder(basename)
235 235
236 236 if thisDate < startDate:
237 237 return 0
238 238
239 239 if thisDate > endDate:
240 240 return 0
241 241
242 242 return 1
243 243
244 244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 245 """
246 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 247
248 248 Inputs:
249 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 250
251 251 Su formato deberia ser "?YYYYDDDsss"
252 252
253 253 siendo:
254 254 YYYY : Anio (ejemplo 2015)
255 255 DDD : Dia del anio (ejemplo 305)
256 256 sss : set
257 257
258 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 259
260 260 endDate : fecha final del rango seleccionado en formato datetime.date
261 261
262 262 Return:
263 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 264 fecha especificado, de lo contrario retorna False.
265 265 Excepciones:
266 266 Si el archivo no tiene el formato adecuado
267 267 """
268 268
269 269 basename = os.path.basename(filename)
270 270
271 271 if not isRadarFile(basename):
272 272 print "The filename %s has not the rigth format" %filename
273 273 return 0
274 274
275 275 if startDate and endDate:
276 276 thisDate = getDateFromRadarFile(basename)
277 277
278 278 if thisDate < startDate:
279 279 return 0
280 280
281 281 if thisDate > endDate:
282 282 return 0
283 283
284 284 return 1
285 285
286 286 def getFileFromSet(path, ext, set):
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294 try:
295 295 year = int(thisFile[1:5])
296 296 doy = int(thisFile[5:8])
297 297 except:
298 298 continue
299 299
300 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 301 continue
302 302
303 303 validFilelist.append(thisFile)
304 304
305 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 306
307 307 if len(myfile)!= 0:
308 308 return myfile[0]
309 309 else:
310 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 311 print 'the filename %s does not exist'%filename
312 312 print '...going to the last file: '
313 313
314 314 if validFilelist:
315 315 validFilelist = sorted( validFilelist, key=str.lower )
316 316 return validFilelist[-1]
317 317
318 318 return None
319 319
320 320 def getlastFileFromPath(path, ext):
321 321 """
322 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 324
325 325 Input:
326 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 327 ext : extension de los files contenidos en una carpeta
328 328
329 329 Return:
330 330 El ultimo file de una determinada carpeta, no se considera el path.
331 331 """
332 332 validFilelist = []
333 333 fileList = os.listdir(path)
334 334
335 335 # 0 1234 567 89A BCDE
336 336 # H YYYY DDD SSS .ext
337 337
338 338 for thisFile in fileList:
339 339
340 340 year = thisFile[1:5]
341 341 if not isNumber(year):
342 342 continue
343 343
344 344 doy = thisFile[5:8]
345 345 if not isNumber(doy):
346 346 continue
347 347
348 348 year = int(year)
349 349 doy = int(doy)
350 350
351 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 352 continue
353 353
354 354 validFilelist.append(thisFile)
355 355
356 356 if validFilelist:
357 357 validFilelist = sorted( validFilelist, key=str.lower )
358 358 return validFilelist[-1]
359 359
360 360 return None
361 361
362 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 """
364 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 366 el path exacto de un determinado file.
367 367
368 368 Example :
369 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 370
371 371 Entonces la funcion prueba con las siguientes combinaciones
372 372 .../.../y2009307367.ext
373 373 .../.../Y2009307367.ext
374 374 .../.../x2009307/y2009307367.ext
375 375 .../.../x2009307/Y2009307367.ext
376 376 .../.../X2009307/y2009307367.ext
377 377 .../.../X2009307/Y2009307367.ext
378 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 379
380 380 Return:
381 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 383 para el filename
384 384 """
385 385 fullfilename = None
386 386 find_flag = False
387 387 filename = None
388 388
389 389 prefixDirList = [None,'d','D']
390 390 if ext.lower() == ".r": #voltage
391 391 prefixFileList = ['d','D']
392 392 elif ext.lower() == ".pdata": #spectra
393 393 prefixFileList = ['p','P']
394 394 else:
395 395 return None, filename
396 396
397 397 #barrido por las combinaciones posibles
398 398 for prefixDir in prefixDirList:
399 399 thispath = path
400 400 if prefixDir != None:
401 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 402 if foldercounter == 0:
403 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 404 else:
405 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 409
410 410 if os.path.exists( fullfilename ): #verifico que exista
411 411 find_flag = True
412 412 break
413 413 if find_flag:
414 414 break
415 415
416 416 if not(find_flag):
417 417 return None, filename
418 418
419 419 return fullfilename, filename
420 420
421 421 def isRadarFolder(folder):
422 422 try:
423 423 year = int(folder[1:5])
424 424 doy = int(folder[5:8])
425 425 except:
426 426 return 0
427 427
428 428 return 1
429 429
430 430 def isRadarFile(file):
431 431 try:
432 432 year = int(file[1:5])
433 433 doy = int(file[5:8])
434 434 set = int(file[8:11])
435 435 except:
436 436 return 0
437 437
438 438 return 1
439 439
440 440 def getDateFromRadarFile(file):
441 441 try:
442 442 year = int(file[1:5])
443 443 doy = int(file[5:8])
444 444 set = int(file[8:11])
445 445 except:
446 446 return None
447 447
448 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 449 return thisDate
450 450
451 451 def getDateFromRadarFolder(folder):
452 452 try:
453 453 year = int(folder[1:5])
454 454 doy = int(folder[5:8])
455 455 except:
456 456 return None
457 457
458 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 459 return thisDate
460 460
461 461 class JRODataIO:
462 462
463 463 c = 3E8
464 464
465 465 isConfig = False
466 466
467 467 basicHeaderObj = None
468 468
469 469 systemHeaderObj = None
470 470
471 471 radarControllerHeaderObj = None
472 472
473 473 processingHeaderObj = None
474 474
475 475 dtype = None
476 476
477 477 pathList = []
478 478
479 479 filenameList = []
480 480
481 481 filename = None
482 482
483 483 ext = None
484 484
485 485 flagIsNewFile = 1
486 486
487 487 flagDiscontinuousBlock = 0
488 488
489 489 flagIsNewBlock = 0
490 490
491 491 fp = None
492 492
493 493 firstHeaderSize = 0
494 494
495 495 basicHeaderSize = 24
496 496
497 497 versionFile = 1103
498 498
499 499 fileSize = None
500 500
501 501 # ippSeconds = None
502 502
503 503 fileSizeByHeader = None
504 504
505 505 fileIndex = None
506 506
507 507 profileIndex = None
508 508
509 509 blockIndex = None
510 510
511 511 nTotalBlocks = None
512 512
513 513 maxTimeStep = 30
514 514
515 515 lastUTTime = None
516 516
517 517 datablock = None
518 518
519 519 dataOut = None
520 520
521 521 blocksize = None
522 522
523 523 getByBlock = False
524 524
525 525 def __init__(self):
526 526
527 527 raise NotImplementedError
528 528
529 529 def run(self):
530 530
531 531 raise NotImplementedError
532 532
533 533 def getDtypeWidth(self):
534 534
535 535 dtype_index = get_dtype_index(self.dtype)
536 536 dtype_width = get_dtype_width(dtype_index)
537 537
538 538 return dtype_width
539 539
540 540 def getAllowedArgs(self):
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544
545
544
545 firstTime = True
546 546 online = 0
547 547
548 548 realtime = 0
549 549
550 550 nReadBlocks = 0
551 551
552 552 delay = 10 #number of seconds waiting a new file
553 553
554 554 nTries = 3 #quantity tries
555 555
556 556 nFiles = 3 #number of files for searching
557 557
558 558 path = None
559 559
560 560 foldercounter = 0
561 561
562 562 flagNoMoreFiles = 0
563 563
564 564 datetimeList = []
565 565
566 566 __isFirstTimeOnline = 1
567 567
568 568 __printInfo = True
569 569
570 570 profileIndex = None
571 571
572 572 nTxs = 1
573 573
574 574 txIndex = None
575 575
576 576 #Added--------------------
577 577
578 578 selBlocksize = None
579 579
580 580 selBlocktime = None
581
582
581
582 onlineWithDate = False
583 583 def __init__(self):
584 584
585 585 """
586 586 This class is used to find data files
587 587
588 588 Example:
589 589 reader = JRODataReader()
590 590 fileList = reader.findDataFiles()
591 591
592 592 """
593 593 pass
594 594
595 595
596 596 def createObjByDefault(self):
597 597 """
598 598
599 599 """
600 600 raise NotImplementedError
601 601
602 602 def getBlockDimension(self):
603 603
604 604 raise NotImplementedError
605 605
606 606 def __searchFilesOffLine(self,
607 607 path,
608 608 startDate=None,
609 609 endDate=None,
610 610 startTime=datetime.time(0,0,0),
611 611 endTime=datetime.time(23,59,59),
612 612 set=None,
613 613 expLabel='',
614 614 ext='.r',
615 615 queue=None,
616 616 cursor=None,
617 617 skip=None,
618 618 walk=True):
619
620 619 self.filenameList = []
621 620 self.datetimeList = []
622 621
623 622 pathList = []
624 623
625 624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
626 625
627 626 if dateList == []:
628 627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
629 628 return None, None
630 629
631 630 if len(dateList) > 1:
632 631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
633 632 else:
634 633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
635 634
636 635 filenameList = []
637 636 datetimeList = []
638 637
639 638 for thisPath in pathList:
640 639 # thisPath = pathList[pathDict[file]]
641 640
642 641 fileList = glob.glob1(thisPath, "*%s" %ext)
643 642 fileList.sort()
644 643
645 644 skippedFileList = []
646 645
647 646 if cursor is not None and skip is not None:
648 647 # if cursor*skip > len(fileList):
649 648 if skip == 0:
650 649 if queue is not None:
651 650 queue.put(len(fileList))
652 651 skippedFileList = []
653 652 else:
654 653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
655 654
656 655 else:
657 656 skippedFileList = fileList
658 657
659 658 for file in skippedFileList:
660 659
661 660 filename = os.path.join(thisPath,file)
662 661
663 662 if not isFileInDateRange(filename, startDate, endDate):
664 663 continue
665 664
666 665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
667 666
668 667 if not(thisDatetime):
669 668 continue
670 669
671 670 filenameList.append(filename)
672 671 datetimeList.append(thisDatetime)
673 672
674 673 if not(filenameList):
675 674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
676 675 return None, None
677 676
678 677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
679 678 print
680 679
681 680 for i in range(len(filenameList)):
682 681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
683 682
684 683 self.filenameList = filenameList
685 self.datetimeList = datetimeList
686
684 self.datetimeList = datetimeList
687 685 return pathList, filenameList
688 686
689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
690
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688
691 689 """
692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
693 devuelve el archivo encontrado ademas de otros datos.
694
695 Input:
696 path : carpeta donde estan contenidos los files que contiene data
697
698 expLabel : Nombre del subexperimento (subfolder)
699
700 ext : extension de los files
701
702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
703
704 Return:
705 directory : eL directorio donde esta el file encontrado
706 filename : el ultimo file de una determinada carpeta
707 year : el anho
708 doy : el numero de dia del anho
709 set : el set del archivo
710
711
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
692
693 Input:
694 path : carpeta donde estan contenidos los files que contiene data
695
696 expLabel : Nombre del subexperimento (subfolder)
697
698 ext : extension de los files
699
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701
702 Return:
703 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
705 year : el anho
706 doy : el numero de dia del anho
707 set : el set del archivo
708
709
712 710 """
711 pathList = None
712 filenameList = None
713 713 if not os.path.isdir(path):
714 714 return None, None, None, None, None, None
715 715
716 716 dirList = []
717 717
718 718 if not walk:
719 719 fullpath = path
720 720 foldercounter = 0
721 721 else:
722 #Filtra solo los directorios
722 # Filtra solo los directorios
723 723 for thisPath in os.listdir(path):
724 724 if not os.path.isdir(os.path.join(path,thisPath)):
725 725 continue
726 726 if not isRadarFolder(thisPath):
727 727 continue
728 728
729 729 dirList.append(thisPath)
730 730
731 731 if not(dirList):
732 732 return None, None, None, None, None, None
733 733
734 734 dirList = sorted( dirList, key=str.lower )
735 735
736 736 doypath = dirList[-1]
737 737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 738 fullpath = os.path.join(path, doypath, expLabel)
739 739
740 740
741 741 print "[Reading] %s folder was found: " %(fullpath )
742 742
743 743 if set == None:
744 744 filename = getlastFileFromPath(fullpath, ext)
745 745 else:
746 746 filename = getFileFromSet(fullpath, ext, set)
747 747
748 748 if not(filename):
749 749 return None, None, None, None, None, None
750 750
751 751 print "[Reading] %s file was found" %(filename)
752 752
753 753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 754 return None, None, None, None, None, None
755 755
756 756 year = int( filename[1:5] )
757 757 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
758 set = int( filename[8:11] )
759 759
760 760 return fullpath, foldercounter, filename, year, doy, set
761 761
762 762 def __setNextFileOffline(self):
763 763
764 764 idFile = self.fileIndex
765 765
766 766 while (True):
767 767 idFile += 1
768 768 if not(idFile < len(self.filenameList)):
769 769 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
770 # print "[Reading] No more Files"
771 771 return 0
772 772
773 773 filename = self.filenameList[idFile]
774 774
775 775 if not(self.__verifyFile(filename)):
776 776 continue
777 777
778 778 fileSize = os.path.getsize(filename)
779 779 fp = open(filename,'rb')
780 780 break
781 781
782 782 self.flagIsNewFile = 1
783 783 self.fileIndex = idFile
784 784 self.filename = filename
785 785 self.fileSize = fileSize
786 786 self.fp = fp
787 787
788 # print "[Reading] Setting the file: %s"%self.filename
788 #print "[Reading] Setting the file: %s"%self.filename
789 789
790 790 return 1
791 791
792 792 def __setNextFileOnline(self):
793 793 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
797
798 Affected:
799 self.flagIsNewFile
800 self.filename
801 self.fileSize
802 self.fp
803 self.set
804 self.flagNoMoreFiles
805
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
810 Excepciones:
811 Si un determinado file no puede ser abierto
812 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
797
798 Affected:
799 self.flagIsNewFile
800 self.filename
801 self.fileSize
802 self.fp
803 self.set
804 self.flagNoMoreFiles
805
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
810 Excepciones:
811 Si un determinado file no puede ser abierto
812 """
813
813 814 nFiles = 0
814 815 fileOk_flag = False
815 816 firstTime_flag = True
816 817
817 818 self.set += 1
818 819
819 820 if self.set > 999:
820 821 self.set = 0
821 822 self.foldercounter += 1
822 823
823 824 #busca el 1er file disponible
824 825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 826 if fullfilename:
826 827 if self.__verifyFile(fullfilename, False):
827 828 fileOk_flag = True
828 829
829 830 #si no encuentra un file entonces espera y vuelve a buscar
830 831 if not(fileOk_flag):
831 832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832 833
833 834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 835 tries = self.nTries
835 836 else:
836 837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837 838
838 839 for nTries in range( tries ):
839 840 if firstTime_flag:
840 841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 842 sleep( self.delay )
842 843 else:
843 844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844 845
845 846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 847 if fullfilename:
847 848 if self.__verifyFile(fullfilename):
848 849 fileOk_flag = True
849 850 break
850 851
851 852 if fileOk_flag:
852 853 break
853 854
854 855 firstTime_flag = False
855 856
856 857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 858 self.set += 1
858 859
859 860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 861 self.set = 0
861 862 self.doy += 1
862 863 self.foldercounter = 0
863 864
864 865 if fileOk_flag:
865 866 self.fileSize = os.path.getsize( fullfilename )
866 867 self.filename = fullfilename
867 868 self.flagIsNewFile = 1
868 869 if self.fp != None: self.fp.close()
869 870 self.fp = open(fullfilename, 'rb')
870 871 self.flagNoMoreFiles = 0
871 872 # print '[Reading] Setting the file: %s' % fullfilename
872 873 else:
873 874 self.fileSize = 0
874 875 self.filename = None
875 876 self.flagIsNewFile = 0
876 877 self.fp = None
877 878 self.flagNoMoreFiles = 1
878 879 # print '[Reading] No more files to read'
879 880
880 881 return fileOk_flag
881 882
882 883 def setNextFile(self):
883 884 if self.fp != None:
884 885 self.fp.close()
885
886 886 if self.online:
887 887 newFile = self.__setNextFileOnline()
888 888 else:
889 889 newFile = self.__setNextFileOffline()
890
891 890 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
892 914 print '[Reading] No more files to read'
893 915 return 0
894 916
895 917 if self.verbose:
896 918 print '[Reading] Setting the file: %s' % self.filename
897 919
898 920 self.__readFirstHeader()
899 921 self.nReadBlocks = 0
900 922 return 1
901 923
902 924 def __waitNewBlock(self):
903 925 """
904 926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
905 927
906 928 Si el modo de lectura es OffLine siempre retorn 0
907 929 """
908 930 if not self.online:
909 931 return 0
910 932
911 933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
912 934 return 0
913 935
914 936 currentPointer = self.fp.tell()
915 937
916 938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917 939
918 940 for nTries in range( self.nTries ):
919 941
920 942 self.fp.close()
921 943 self.fp = open( self.filename, 'rb' )
922 944 self.fp.seek( currentPointer )
923 945
924 946 self.fileSize = os.path.getsize( self.filename )
925 947 currentSize = self.fileSize - currentPointer
926 948
927 949 if ( currentSize >= neededSize ):
928 950 self.basicHeaderObj.read(self.fp)
929 951 return 1
930 952
931 953 if self.fileSize == self.fileSizeByHeader:
932 954 # self.flagEoF = True
933 955 return 0
934 956
935 957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
936 958 sleep( self.delay )
937 959
938 960
939 961 return 0
940 962
941 963 def waitDataBlock(self,pointer_location):
942 964
943 965 currentPointer = pointer_location
944 966
945 967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
946 968
947 969 for nTries in range( self.nTries ):
948 970 self.fp.close()
949 971 self.fp = open( self.filename, 'rb' )
950 972 self.fp.seek( currentPointer )
951 973
952 974 self.fileSize = os.path.getsize( self.filename )
953 975 currentSize = self.fileSize - currentPointer
954 976
955 977 if ( currentSize >= neededSize ):
956 978 return 1
957 979
958 980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
959 981 sleep( self.delay )
960 982
961 983 return 0
962 984
963 985 def __jumpToLastBlock(self):
964 986
965 987 if not(self.__isFirstTimeOnline):
966 988 return
967 989
968 990 csize = self.fileSize - self.fp.tell()
969 991 blocksize = self.processingHeaderObj.blockSize
970 992
971 993 #salta el primer bloque de datos
972 994 if csize > self.processingHeaderObj.blockSize:
973 995 self.fp.seek(self.fp.tell() + blocksize)
974 996 else:
975 997 return
976 998
977 999 csize = self.fileSize - self.fp.tell()
978 1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
979 1001 while True:
980 1002
981 1003 if self.fp.tell()<self.fileSize:
982 1004 self.fp.seek(self.fp.tell() + neededsize)
983 1005 else:
984 1006 self.fp.seek(self.fp.tell() - neededsize)
985 1007 break
986 1008
987 1009 # csize = self.fileSize - self.fp.tell()
988 1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
989 1011 # factor = int(csize/neededsize)
990 1012 # if factor > 0:
991 1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
992 1014
993 1015 self.flagIsNewFile = 0
994 1016 self.__isFirstTimeOnline = 0
995 1017
996 1018 def __setNewBlock(self):
997 1019 #if self.server is None:
998 1020 if self.fp == None:
999 1021 return 0
1000 1022
1001 1023 # if self.online:
1002 1024 # self.__jumpToLastBlock()
1003 1025
1004 1026 if self.flagIsNewFile:
1005 1027 self.lastUTTime = self.basicHeaderObj.utc
1006 1028 return 1
1007 1029
1008 1030 if self.realtime:
1009 1031 self.flagDiscontinuousBlock = 1
1010 1032 if not(self.setNextFile()):
1011 1033 return 0
1012 1034 else:
1013 1035 return 1
1014 1036 #if self.server is None:
1015 1037 currentSize = self.fileSize - self.fp.tell()
1016 1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1017 1039 if (currentSize >= neededSize):
1018 1040 self.basicHeaderObj.read(self.fp)
1019 1041 self.lastUTTime = self.basicHeaderObj.utc
1020 1042 return 1
1021 1043 # else:
1022 1044 # self.basicHeaderObj.read(self.zHeader)
1023 1045 # self.lastUTTime = self.basicHeaderObj.utc
1024 1046 # return 1
1025 1047 if self.__waitNewBlock():
1026 1048 self.lastUTTime = self.basicHeaderObj.utc
1027 1049 return 1
1028 1050 #if self.server is None:
1029 1051 if not(self.setNextFile()):
1030 1052 return 0
1031 1053
1032 1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1033 1055 self.lastUTTime = self.basicHeaderObj.utc
1034 1056
1035 1057 self.flagDiscontinuousBlock = 0
1036 1058
1037 1059 if deltaTime > self.maxTimeStep:
1038 1060 self.flagDiscontinuousBlock = 1
1039 1061
1040 1062 return 1
1041 1063
1042 1064 def readNextBlock(self):
1043 1065
1044 1066 #Skip block out of startTime and endTime
1045 1067 while True:
1046 1068 if not(self.__setNewBlock()):
1047 1069 print 'returning'
1048 1070 return 0
1049
1050 1071 if not(self.readBlock()):
1051 1072 return 0
1052
1053 1073 self.getBasicHeader()
1054
1055 1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1056 1075
1057 1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1058 1077 self.processingHeaderObj.dataBlocksPerFile,
1059 1078 self.dataOut.datatime.ctime())
1060 1079 continue
1061 1080
1062 1081 break
1063 1082
1064 1083 if self.verbose:
1065 1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1066 1085 self.processingHeaderObj.dataBlocksPerFile,
1067 1086 self.dataOut.datatime.ctime())
1068 1087 return 1
1069 1088
1070 1089 def __readFirstHeader(self):
1071 1090
1072 1091 self.basicHeaderObj.read(self.fp)
1073 1092 self.systemHeaderObj.read(self.fp)
1074 1093 self.radarControllerHeaderObj.read(self.fp)
1075 1094 self.processingHeaderObj.read(self.fp)
1076 1095
1077 1096 self.firstHeaderSize = self.basicHeaderObj.size
1078 1097
1079 1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1080 1099 if datatype == 0:
1081 1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1082 1101 elif datatype == 1:
1083 1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1084 1103 elif datatype == 2:
1085 1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1086 1105 elif datatype == 3:
1087 1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1088 1107 elif datatype == 4:
1089 1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1090 1109 elif datatype == 5:
1091 1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1092 1111 else:
1093 1112 raise ValueError, 'Data type was not defined'
1094 1113
1095 1114 self.dtype = datatype_str
1096 1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1097 1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1098 1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1099 1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1100 1119 self.getBlockDimension()
1101 1120
1102 1121 def __verifyFile(self, filename, msgFlag=True):
1103 1122
1104 1123 msg = None
1105 1124
1106 1125 try:
1107 1126 fp = open(filename, 'rb')
1108 1127 except IOError:
1109 1128
1110 1129 if msgFlag:
1111 1130 print "[Reading] File %s can't be opened" % (filename)
1112 1131
1113 1132 return False
1114 1133
1115 1134 currentPosition = fp.tell()
1116 1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1117 1136
1118 1137 if neededSize == 0:
1119 1138 basicHeaderObj = BasicHeader(LOCALTIME)
1120 1139 systemHeaderObj = SystemHeader()
1121 1140 radarControllerHeaderObj = RadarControllerHeader()
1122 1141 processingHeaderObj = ProcessingHeader()
1123 1142
1124 1143 if not( basicHeaderObj.read(fp) ):
1125 1144 fp.close()
1126 1145 return False
1127 1146
1128 1147 if not( systemHeaderObj.read(fp) ):
1129 1148 fp.close()
1130 1149 return False
1131 1150
1132 1151 if not( radarControllerHeaderObj.read(fp) ):
1133 1152 fp.close()
1134 1153 return False
1135 1154
1136 1155 if not( processingHeaderObj.read(fp) ):
1137 1156 fp.close()
1138 1157 return False
1139 1158
1140 1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1141 1160 else:
1142 1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1143 1162
1144 1163 fp.close()
1145 1164
1146 1165 fileSize = os.path.getsize(filename)
1147 1166 currentSize = fileSize - currentPosition
1148 1167
1149 1168 if currentSize < neededSize:
1150 1169 if msgFlag and (msg != None):
1151 1170 print msg
1152 1171 return False
1153 1172
1154 1173 return True
1155 1174
1156 1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1157 1176
1158 1177 path_empty = True
1159 1178
1160 1179 dateList = []
1161 1180 pathList = []
1162 1181
1163 1182 multi_path = path.split(',')
1164 1183
1165 1184 if not walk:
1166 1185
1167 1186 for single_path in multi_path:
1168 1187
1169 1188 if not os.path.isdir(single_path):
1170 1189 continue
1171 1190
1172 1191 fileList = glob.glob1(single_path, "*"+ext)
1173 1192
1174 1193 if not fileList:
1175 1194 continue
1176 1195
1177 1196 path_empty = False
1178 1197
1179 1198 fileList.sort()
1180 1199
1181 1200 for thisFile in fileList:
1182 1201
1183 1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1184 1203 continue
1185 1204
1186 1205 if not isRadarFile(thisFile):
1187 1206 continue
1188 1207
1189 1208 if not isFileInDateRange(thisFile, startDate, endDate):
1190 1209 continue
1191 1210
1192 1211 thisDate = getDateFromRadarFile(thisFile)
1193 1212
1194 1213 if thisDate in dateList:
1195 1214 continue
1196 1215
1197 1216 dateList.append(thisDate)
1198 1217 pathList.append(single_path)
1199 1218
1200 1219 else:
1201 1220 for single_path in multi_path:
1202 1221
1203 1222 if not os.path.isdir(single_path):
1204 1223 continue
1205 1224
1206 1225 dirList = []
1207 1226
1208 1227 for thisPath in os.listdir(single_path):
1209 1228
1210 1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1211 1230 continue
1212 1231
1213 1232 if not isRadarFolder(thisPath):
1214 1233 continue
1215 1234
1216 1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1217 1236 continue
1218 1237
1219 1238 dirList.append(thisPath)
1220 1239
1221 1240 if not dirList:
1222 1241 continue
1223 1242
1224 1243 dirList.sort()
1225 1244
1226 1245 for thisDir in dirList:
1227 1246
1228 1247 datapath = os.path.join(single_path, thisDir, expLabel)
1229 1248 fileList = glob.glob1(datapath, "*"+ext)
1230 1249
1231 1250 if not fileList:
1232 1251 continue
1233 1252
1234 1253 path_empty = False
1235 1254
1236 1255 thisDate = getDateFromRadarFolder(thisDir)
1237 1256
1238 1257 pathList.append(datapath)
1239 1258 dateList.append(thisDate)
1240 1259
1241 1260 dateList.sort()
1242 1261
1243 1262 if walk:
1244 1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1245 1264 else:
1246 1265 pattern_path = multi_path[0]
1247 1266
1248 1267 if path_empty:
1249 1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1250 1269 else:
1251 1270 if not dateList:
1252 1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1253 1272
1254 1273 if include_path:
1255 1274 return dateList, pathList
1256 1275
1257 1276 return dateList
1258 1277
1259 1278 def setup(self,
1260 1279 path=None,
1261 1280 startDate=None,
1262 1281 endDate=None,
1263 1282 startTime=datetime.time(0,0,0),
1264 1283 endTime=datetime.time(23,59,59),
1265 1284 set=None,
1266 1285 expLabel = "",
1267 1286 ext = None,
1268 1287 online = False,
1269 1288 delay = 60,
1270 1289 walk = True,
1271 1290 getblock = False,
1272 1291 nTxs = 1,
1273 1292 realtime=False,
1274 1293 blocksize=None,
1275 1294 blocktime=None,
1276 1295 queue=None,
1277 1296 skip=None,
1278 1297 cursor=None,
1279 1298 warnings=True,
1280 1299 verbose=True,
1281 1300 server=None):
1282 1301 if server is not None:
1283 1302 if 'tcp://' in server:
1284 1303 address = server
1285 1304 else:
1286 1305 address = 'ipc:///tmp/%s' % server
1287 1306 self.server = address
1288 1307 self.context = zmq.Context()
1289 1308 self.receiver = self.context.socket(zmq.PULL)
1290 1309 self.receiver.connect(self.server)
1291 1310 time.sleep(0.5)
1292 1311 print '[Starting] ReceiverData from {}'.format(self.server)
1293 1312 else:
1294 1313 self.server = None
1295 1314 if path == None:
1296 1315 raise ValueError, "[Reading] The path is not valid"
1297 1316
1298 1317 if ext == None:
1299 1318 ext = self.ext
1300 1319
1301 1320 if online:
1302 1321 print "[Reading] Searching files in online mode..."
1303 1322
1304 1323 for nTries in range( self.nTries ):
1305 1324 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1306 1325
1307 1326 if fullpath:
1308 1327 break
1309 1328
1310 1329 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1311 1330 sleep( self.delay )
1312 1331
1313 1332 if not(fullpath):
1314 1333 print "[Reading] There 'isn't any valid file in %s" % path
1315 1334 return
1316 1335
1317 1336 self.year = year
1318 1337 self.doy = doy
1319 1338 self.set = set - 1
1320 1339 self.path = path
1321 1340 self.foldercounter = foldercounter
1322 1341 last_set = None
1323 1342 else:
1324 1343 print "[Reading] Searching files in offline mode ..."
1325 1344 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 1345 startTime=startTime, endTime=endTime,
1327 1346 set=set, expLabel=expLabel, ext=ext,
1328 1347 walk=walk, cursor=cursor,
1329 1348 skip=skip, queue=queue)
1330 1349
1350 <<<<<<< HEAD
1331 1351 if not(pathList):
1332 1352 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1333 1353 # datetime.datetime.combine(startDate,startTime).ctime(),
1334 1354 # datetime.datetime.combine(endDate,endTime).ctime())
1335 1355
1336 1356 # sys.exit(-1)
1337 1357
1338 1358 self.fileIndex = -1
1339 1359 self.pathList = []
1340 1360 self.filenameList = []
1341 1361 return
1342 1362
1363 =======
1364 if path == None:
1365 raise ValueError, "[Reading] The path is not valid"
1366
1367
1368 if ext == None:
1369 ext = self.ext
1370
1371 self.path = path
1372 self.startDate = startDate
1373 self.endDate = endDate
1374 self.startTime = startTime
1375 self.endTime = endTime
1376 self.set = set
1377 self.expLabel = expLabel
1378 self.ext = ext
1379 self.online = online
1380 self.delay = delay
1381 self.walk = walk
1382 self.getblock = getblock
1383 self.nTxs = nTxs
1384 self.realtime = realtime
1385 self.blocksize = blocksize
1386 self.blocktime = blocktime
1387
1388
1389 if self.firstTime is True:
1390 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1391 startTime=startTime, endTime=endTime,
1392 set=set, expLabel=expLabel, ext=ext,
1393 walk=walk)
1394 filenameList = filenameList[:-1]
1395
1396 if pathList is not None and filenameList is not None and online:
1397 self.onlineWithDate = True
1398 online = False
1399 self.fileIndex = -1
1400 self.pathList = pathList
1401 self.filenameList = filenameList
1402 file_name = os.path.basename(filenameList[-1])
1403 basename, ext = os.path.splitext(file_name)
1404 last_set = int(basename[-3:])
1405
1406 if online:
1407 print "[Reading] Searching files in online mode..."
1408
1409 for nTries in range(self.nTries):
1410 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1411 expLabel=expLabel,
1412 ext=ext,
1413 walk=walk,
1414 startDate=startDate,
1415 startTime=startTime,
1416 set=set)
1417
1418 if fullpath:
1419 break
1420 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1421 sleep( self.delay )
1422
1423 if not(fullpath):
1424 print "[Reading] There 'isn't any valid file in %s" % path
1425 return
1426
1427 self.year = year
1428 self.doy = doy
1429 self.set = set - 1
1430 self.path = path
1431 self.foldercounter = foldercounter
1432 last_set = None
1433
1434 else:
1435 print "[Reading] Searching files in offline mode ..."
1436 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1437 startTime=startTime, endTime=endTime,
1438 set=set, expLabel=expLabel, ext=ext,
1439 walk=walk)
1440
1441 if not(pathList):
1442 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1443 # datetime.datetime.combine(startDate,startTime).ctime(),
1444 # datetime.datetime.combine(endDate,endTime).ctime())
1445
1446 # sys.exit(-1)
1447
1448 self.fileIndex = -1
1449 self.pathList = []
1450 self.filenameList = []
1451 return
1452
1453 self.fileIndex = -1
1454 self.pathList = pathList
1455 self.filenameList = filenameList
1456 file_name = os.path.basename(filenameList[-1])
1457 basename, ext = os.path.splitext(file_name)
1458 last_set = int(basename[-3:])
1459
1460
1461 self.online = online
1462 self.realtime = realtime
1463 self.delay = delay
1464 ext = ext.lower()
1465 self.ext = ext
1466 self.getByBlock = getblock
1467 self.nTxs = nTxs
1468 self.startTime = startTime
1469 self.endTime = endTime
1470
1471
1472 #Added-----------------
1473 self.selBlocksize = blocksize
1474 self.selBlocktime = blocktime
1475
1476
1477 if not(self.setNextFile()):
1478 if (startDate!=None) and (endDate!=None):
1479 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1480 elif startDate != None:
1481 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1482 else:
1483 print "[Reading] No files"
1484
1485 >>>>>>> online_data_hour
1343 1486 self.fileIndex = -1
1344 1487 self.pathList = pathList
1345 1488 self.filenameList = filenameList
1346 1489 file_name = os.path.basename(filenameList[-1])
1347 1490 basename, ext = os.path.splitext(file_name)
1348 1491 last_set = int(basename[-3:])
1349 1492
1350 1493 self.online = online
1351 1494 self.realtime = realtime
1352 1495 self.delay = delay
1353 1496 ext = ext.lower()
1354 1497 self.ext = ext
1355 1498 self.getByBlock = getblock
1356 1499 self.nTxs = nTxs
1357 1500 self.startTime = startTime
1358 1501 self.endTime = endTime
1359 1502
1360 1503 #Added-----------------
1361 1504 self.selBlocksize = blocksize
1362 1505 self.selBlocktime = blocktime
1363 1506
1364 1507 # Verbose-----------
1365 1508 self.verbose = verbose
1366 1509 self.warnings = warnings
1367 1510
1511 <<<<<<< HEAD
1368 1512 if not(self.setNextFile()):
1369 1513 if (startDate!=None) and (endDate!=None):
1370 1514 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1371 1515 elif startDate != None:
1372 1516 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1373 1517 else:
1374 1518 print "[Reading] No files"
1375 1519
1376 1520 self.fileIndex = -1
1377 1521 self.pathList = []
1378 1522 self.filenameList = []
1379 1523 return
1380 1524
1381 1525 # self.getBasicHeader()
1382 1526
1383 1527 if last_set != None:
1384 1528 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1385 1529 return
1530 =======
1531 # self.getBasicHeader()
1532
1533 if last_set != None:
1534 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1535 return
1536 >>>>>>> online_data_hour
1386 1537
1387 1538 def getBasicHeader(self):
1388 1539
1389 1540 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1390 1541
1391 1542 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1392 1543
1393 1544 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1394 1545
1395 1546 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1396 1547
1397 1548 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1398 1549
1399 1550 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1400 1551
1401 1552 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1402
1403 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1404
1405
1553
1554 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1555
1556
1406 1557 def getFirstHeader(self):
1407 1558
1408 1559 raise NotImplementedError
1409 1560
1410 1561 def getData(self):
1411 1562
1412 1563 raise NotImplementedError
1413 1564
1414 1565 def hasNotDataInBuffer(self):
1415 1566
1416 1567 raise NotImplementedError
1417 1568
1418 1569 def readBlock(self):
1419 1570
1420 1571 raise NotImplementedError
1421 1572
1422 1573 def isEndProcess(self):
1423 1574
1424 1575 return self.flagNoMoreFiles
1425 1576
1426 1577 def printReadBlocks(self):
1427 1578
1428 1579 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1429 1580
1430 1581 def printTotalBlocks(self):
1431 1582
1432 1583 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1433 1584
1434 1585 def printNumberOfBlock(self):
1435 1586
1436 1587 if self.flagIsNewBlock:
1437 1588 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1438 1589 self.processingHeaderObj.dataBlocksPerFile,
1439 1590 self.dataOut.datatime.ctime())
1440 1591
1441 1592 def printInfo(self):
1442 1593
1443 1594 if self.__printInfo == False:
1444 1595 return
1445 1596
1446 1597 self.basicHeaderObj.printInfo()
1447 1598 self.systemHeaderObj.printInfo()
1448 1599 self.radarControllerHeaderObj.printInfo()
1449 1600 self.processingHeaderObj.printInfo()
1450 1601
1451 1602 self.__printInfo = False
1452 1603
1453 1604
1454 1605 def run(self,
1455 1606 path=None,
1456 1607 startDate=None,
1457 1608 endDate=None,
1458 1609 startTime=datetime.time(0,0,0),
1459 1610 endTime=datetime.time(23,59,59),
1460 1611 set=None,
1461 1612 expLabel = "",
1462 1613 ext = None,
1463 1614 online = False,
1464 1615 delay = 60,
1465 1616 walk = True,
1466 1617 getblock = False,
1467 1618 nTxs = 1,
1468 1619 realtime=False,
1469 1620 blocksize=None,
1470 1621 blocktime=None,
1471 1622 queue=None,
1472 1623 skip=None,
1473 1624 cursor=None,
1474 1625 warnings=True,
1475 1626 server=None,
1476 1627 verbose=True, **kwargs):
1477 1628
1478 1629 if not(self.isConfig):
1630 <<<<<<< HEAD
1479 1631 # self.dataOut = dataOut
1480 1632 self.setup( path=path,
1481 1633 startDate=startDate,
1482 1634 endDate=endDate,
1483 1635 startTime=startTime,
1484 1636 endTime=endTime,
1485 1637 set=set,
1486 1638 expLabel=expLabel,
1487 1639 ext=ext,
1488 1640 online=online,
1489 1641 delay=delay,
1490 1642 walk=walk,
1491 1643 getblock=getblock,
1492 1644 nTxs=nTxs,
1493 1645 realtime=realtime,
1494 1646 blocksize=blocksize,
1495 1647 blocktime=blocktime,
1496 1648 queue=queue,
1497 1649 skip=skip,
1498 1650 cursor=cursor,
1499 1651 warnings=warnings,
1500 1652 server=server,
1501 1653 verbose=verbose)
1654 =======
1655
1656 # self.dataOut = dataOut
1657 self.setup(**kwargs)
1658 >>>>>>> online_data_hour
1502 1659 self.isConfig = True
1503 1660 if server is None:
1504 1661 self.getData()
1505 1662 else:
1506 1663 self.getFromServer()
1507 1664
1508 1665 class JRODataWriter(JRODataIO):
1509 1666
1510 1667 """
1511 1668 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1512 1669 de los datos siempre se realiza por bloques.
1513 1670 """
1514 1671
1515 1672 blockIndex = 0
1516 1673
1517 1674 path = None
1518 1675
1519 1676 setFile = None
1520 1677
1521 1678 profilesPerBlock = None
1522 1679
1523 1680 blocksPerFile = None
1524 1681
1525 1682 nWriteBlocks = 0
1526 1683
1527 1684 fileDate = None
1528 1685
1529 1686 def __init__(self, dataOut=None):
1530 1687 raise NotImplementedError
1531 1688
1532 1689
1533 1690 def hasAllDataInBuffer(self):
1534 1691 raise NotImplementedError
1535 1692
1536 1693
1537 1694 def setBlockDimension(self):
1538 1695 raise NotImplementedError
1539 1696
1540 1697
1541 1698 def writeBlock(self):
1542 1699 raise NotImplementedError
1543 1700
1544 1701
1545 1702 def putData(self):
1546 1703 raise NotImplementedError
1547 1704
1548 1705
1549 1706 def getProcessFlags(self):
1550 1707
1551 1708 processFlags = 0
1552 1709
1553 1710 dtype_index = get_dtype_index(self.dtype)
1554 1711 procflag_dtype = get_procflag_dtype(dtype_index)
1555 1712
1556 1713 processFlags += procflag_dtype
1557 1714
1558 1715 if self.dataOut.flagDecodeData:
1559 1716 processFlags += PROCFLAG.DECODE_DATA
1560 1717
1561 1718 if self.dataOut.flagDeflipData:
1562 1719 processFlags += PROCFLAG.DEFLIP_DATA
1563 1720
1564 1721 if self.dataOut.code is not None:
1565 1722 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1566 1723
1567 1724 if self.dataOut.nCohInt > 1:
1568 1725 processFlags += PROCFLAG.COHERENT_INTEGRATION
1569 1726
1570 1727 if self.dataOut.type == "Spectra":
1571 1728 if self.dataOut.nIncohInt > 1:
1572 1729 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1573 1730
1574 1731 if self.dataOut.data_dc is not None:
1575 1732 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1576 1733
1577 1734 if self.dataOut.flagShiftFFT:
1578 1735 processFlags += PROCFLAG.SHIFT_FFT_DATA
1579 1736
1580 1737 return processFlags
1581 1738
1582 1739 def setBasicHeader(self):
1583 1740
1584 1741 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1585 1742 self.basicHeaderObj.version = self.versionFile
1586 1743 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1587 1744
1588 1745 utc = numpy.floor(self.dataOut.utctime)
1589 1746 milisecond = (self.dataOut.utctime - utc)* 1000.0
1590 1747
1591 1748 self.basicHeaderObj.utc = utc
1592 1749 self.basicHeaderObj.miliSecond = milisecond
1593 1750 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1594 1751 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1595 1752 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1596 1753
1597 1754 def setFirstHeader(self):
1598 1755 """
1599 1756 Obtiene una copia del First Header
1600 1757
1601 1758 Affected:
1602 1759
1603 1760 self.basicHeaderObj
1604 1761 self.systemHeaderObj
1605 1762 self.radarControllerHeaderObj
1606 1763 self.processingHeaderObj self.
1607 1764
1608 1765 Return:
1609 1766 None
1610 1767 """
1611 1768
1612 1769 raise NotImplementedError
1613 1770
1614 1771 def __writeFirstHeader(self):
1615 1772 """
1616 1773 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1617 1774
1618 1775 Affected:
1619 1776 __dataType
1620 1777
1621 1778 Return:
1622 1779 None
1623 1780 """
1781 <<<<<<< HEAD
1624 1782
1625 1783 # CALCULAR PARAMETROS
1626 1784
1785 =======
1786
1787 # CALCULAR PARAMETROS
1788
1789 >>>>>>> online_data_hour
1627 1790 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1628 1791 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1629 1792
1630 1793 self.basicHeaderObj.write(self.fp)
1631 1794 self.systemHeaderObj.write(self.fp)
1632 1795 self.radarControllerHeaderObj.write(self.fp)
1633 1796 self.processingHeaderObj.write(self.fp)
1634 1797
1635 1798 def __setNewBlock(self):
1636 1799 """
1637 1800 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1638 1801
1639 1802 Return:
1640 1803 0 : si no pudo escribir nada
1641 1804 1 : Si escribio el Basic el First Header
1642 1805 """
1643 1806 if self.fp == None:
1644 1807 self.setNextFile()
1645 1808
1646 1809 if self.flagIsNewFile:
1647 1810 return 1
1648 1811
1649 1812 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1650 1813 self.basicHeaderObj.write(self.fp)
1651 1814 return 1
1652 1815
1653 1816 if not( self.setNextFile() ):
1654 1817 return 0
1655 1818
1656 1819 return 1
1657 1820
1658 1821
1659 1822 def writeNextBlock(self):
1660 1823 """
1661 1824 Selecciona el bloque siguiente de datos y los escribe en un file
1662 1825
1663 1826 Return:
1664 1827 0 : Si no hizo pudo escribir el bloque de datos
1665 1828 1 : Si no pudo escribir el bloque de datos
1666 1829 """
1667 1830 if not( self.__setNewBlock() ):
1668 1831 return 0
1669 1832
1670 1833 self.writeBlock()
1671 1834
1672 1835 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1673 1836 self.processingHeaderObj.dataBlocksPerFile)
1674 1837
1675 1838 return 1
1676 1839
1677 1840 def setNextFile(self):
1678 1841 """
1679 1842 Determina el siguiente file que sera escrito
1680 1843
1681 1844 Affected:
1682 1845 self.filename
1683 1846 self.subfolder
1684 1847 self.fp
1685 1848 self.setFile
1686 1849 self.flagIsNewFile
1687 1850
1688 1851 Return:
1689 1852 0 : Si el archivo no puede ser escrito
1690 1853 1 : Si el archivo esta listo para ser escrito
1691 1854 """
1692 1855 ext = self.ext
1693 1856 path = self.path
1694 1857
1695 1858 if self.fp != None:
1696 1859 self.fp.close()
1697 1860
1698 1861 timeTuple = time.localtime( self.dataOut.utctime)
1699 1862 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1700 1863
1701 1864 fullpath = os.path.join( path, subfolder )
1702 1865 setFile = self.setFile
1703 1866
1704 1867 if not( os.path.exists(fullpath) ):
1705 1868 os.mkdir(fullpath)
1706 1869 setFile = -1 #inicializo mi contador de seteo
1707 1870 else:
1708 1871 filesList = os.listdir( fullpath )
1709 1872 if len( filesList ) > 0:
1710 1873 filesList = sorted( filesList, key=str.lower )
1711 1874 filen = filesList[-1]
1712 1875 # el filename debera tener el siguiente formato
1713 1876 # 0 1234 567 89A BCDE (hex)
1714 1877 # x YYYY DDD SSS .ext
1715 1878 if isNumber( filen[8:11] ):
1716 1879 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1717 1880 else:
1718 1881 setFile = -1
1719 1882 else:
1720 1883 setFile = -1 #inicializo mi contador de seteo
1721 1884
1722 1885 setFile += 1
1723 1886
1724 1887 #If this is a new day it resets some values
1725 1888 if self.dataOut.datatime.date() > self.fileDate:
1726 1889 setFile = 0
1727 1890 self.nTotalBlocks = 0
1728 1891
1729 1892 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1730 1893
1731 1894 filename = os.path.join( path, subfolder, filen )
1732 1895
1733 1896 fp = open( filename,'wb' )
1734 1897
1735 1898 self.blockIndex = 0
1736 1899
1737 1900 #guardando atributos
1738 1901 self.filename = filename
1739 1902 self.subfolder = subfolder
1740 1903 self.fp = fp
1741 1904 self.setFile = setFile
1742 1905 self.flagIsNewFile = 1
1743 1906 self.fileDate = self.dataOut.datatime.date()
1744 1907
1745 1908 self.setFirstHeader()
1746 1909
1747 1910 print '[Writing] Opening file: %s'%self.filename
1748 1911
1749 1912 self.__writeFirstHeader()
1750 1913
1751 1914 return 1
1752 1915
1753 1916 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1754 1917 """
1755 1918 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1756 1919
1757 1920 Inputs:
1758 1921 path : directory where data will be saved
1759 1922 profilesPerBlock : number of profiles per block
1760 1923 set : initial file set
1761 1924 datatype : An integer number that defines data type:
1762 1925 0 : int8 (1 byte)
1763 1926 1 : int16 (2 bytes)
1764 1927 2 : int32 (4 bytes)
1765 1928 3 : int64 (8 bytes)
1766 1929 4 : float32 (4 bytes)
1767 1930 5 : double64 (8 bytes)
1768 1931
1769 1932 Return:
1770 1933 0 : Si no realizo un buen seteo
1771 1934 1 : Si realizo un buen seteo
1772 1935 """
1773 1936
1774 1937 if ext == None:
1775 1938 ext = self.ext
1776 1939
1777 1940 self.ext = ext.lower()
1778 1941
1779 1942 self.path = path
1780 1943
1781 1944 if set is None:
1782 1945 self.setFile = -1
1783 1946 else:
1784 1947 self.setFile = set - 1
1785 1948
1786 1949 self.blocksPerFile = blocksPerFile
1787 1950
1788 1951 self.profilesPerBlock = profilesPerBlock
1789 1952
1790 1953 self.dataOut = dataOut
1791 1954 self.fileDate = self.dataOut.datatime.date()
1792 1955 #By default
1793 1956 self.dtype = self.dataOut.dtype
1794 1957
1795 1958 if datatype is not None:
1796 1959 self.dtype = get_numpy_dtype(datatype)
1797 1960
1798 1961 if not(self.setNextFile()):
1799 1962 print "[Writing] There isn't a next file"
1800 1963 return 0
1801 1964
1802 1965 self.setBlockDimension()
1803 1966
1804 1967 return 1
1805 1968
1806 1969 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1807 1970
1808 1971 if not(self.isConfig):
1809 1972
1810 1973 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1811 1974 self.isConfig = True
1812 1975
1813 1976 self.putData()
Show file before | Show file after | Hide comments
@@ -1,737 +1,824
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 13 import zmq
14 14 import tempfile
15 15 from StringIO import StringIO
16 16 # from _sha import blocksize
17 17
18 18 class VoltageReader(JRODataReader, ProcessingUnit):
19 19 """
20 20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 21 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
22 22 perfiles*alturas*canales) son almacenados en la variable "buffer".
23 23
24 24 perfiles * alturas * canales
25 25
26 26 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
27 27 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
28 28 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
29 29 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
30 30
31 31 Example:
32 32
33 33 dpath = "/home/myuser/data"
34 34
35 35 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
36 36
37 37 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
38 38
39 39 readerObj = VoltageReader()
40 40
41 41 readerObj.setup(dpath, startTime, endTime)
42 42
43 43 while(True):
44 44
45 45 #to get one profile
46 46 profile = readerObj.getData()
47 47
48 48 #print the profile
49 49 print profile
50 50
51 51 #If you want to see all datablock
52 52 print readerObj.datablock
53 53
54 54 if readerObj.flagNoMoreFiles:
55 55 break
56 56
57 57 """
58 58
59 59 ext = ".r"
60 60
61 61 optchar = "D"
62 62 dataOut = None
63 63
64 64 def __init__(self, **kwargs):
65 65 """
66 66 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
67 67
68 68 Input:
69 69 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
70 70 almacenar un perfil de datos cada vez que se haga un requerimiento
71 71 (getData). El perfil sera obtenido a partir del buffer de datos,
72 72 si el buffer esta vacio se hara un nuevo proceso de lectura de un
73 73 bloque de datos.
74 74 Si este parametro no es pasado se creara uno internamente.
75 75
76 76 Variables afectadas:
77 77 self.dataOut
78 78
79 79 Return:
80 80 None
81 81 """
82 82
83 83 ProcessingUnit.__init__(self, **kwargs)
84 84
85 85 self.isConfig = False
86 86
87 87 self.datablock = None
88 88
89 89 self.utc = 0
90 90
91 91 self.ext = ".r"
92 92
93 93 self.optchar = "D"
94 94
95 95 self.basicHeaderObj = BasicHeader(LOCALTIME)
96 96
97 97 self.systemHeaderObj = SystemHeader()
98 98
99 99 self.radarControllerHeaderObj = RadarControllerHeader()
100 100
101 101 self.processingHeaderObj = ProcessingHeader()
102 102
103 103 self.online = 0
104 104
105 105 self.fp = None
106 106
107 107 self.idFile = None
108 108
109 109 self.dtype = None
110 110
111 111 self.fileSizeByHeader = None
112 112
113 113 self.filenameList = []
114 114
115 115 self.filename = None
116 116
117 117 self.fileSize = None
118 118
119 119 self.firstHeaderSize = 0
120 120
121 121 self.basicHeaderSize = 24
122 122
123 123 self.pathList = []
124 124
125 125 self.filenameList = []
126 126
127 127 self.lastUTTime = 0
128 128
129 129 self.maxTimeStep = 30
130 130
131 131 self.flagNoMoreFiles = 0
132 132
133 133 self.set = 0
134 134
135 135 self.path = None
136 136
137 137 self.profileIndex = 2**32-1
138 138
139 139 self.delay = 3 #seconds
140 140
141 141 self.nTries = 3 #quantity tries
142 142
143 143 self.nFiles = 3 #number of files for searching
144 144
145 145 self.nReadBlocks = 0
146 146
147 147 self.flagIsNewFile = 1
148 148
149 149 self.__isFirstTimeOnline = 1
150 150
151 151 # self.ippSeconds = 0
152 152
153 153 self.flagDiscontinuousBlock = 0
154 154
155 155 self.flagIsNewBlock = 0
156 156
157 157 self.nTotalBlocks = 0
158 158
159 159 self.blocksize = 0
160 160
161 161 self.dataOut = self.createObjByDefault()
162 162
163 163 self.nTxs = 1
164 164
165 165 self.txIndex = 0
166 166
167 167 def createObjByDefault(self):
168 168
169 169 dataObj = Voltage()
170 170
171 171 return dataObj
172 172
173 173 def __hasNotDataInBuffer(self):
174 174
175 175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
176 176 return 1
177 177
178 178 return 0
179 179
180 180
181 181 def getBlockDimension(self):
182 182 """
183 <<<<<<< HEAD
183 184 Obtiene la cantidad de puntos a leer por cada bloque de datos
184 185
185 186 Affected:
186 187 self.blocksize
187
188 Return:
189 None
188 =======
189 Obtiene la cantidad de puntos a leer por cada bloque de datos
190
191 Affected:
192 self.blocksize
193 >>>>>>> online_data_hour
194
195 Return:
196 None
190 197 """
191 198 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
192 199 self.blocksize = pts2read
193 200
194 201
195 202
196 203 def readBlock(self):
197 204 """
205 <<<<<<< HEAD
198 206 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
199 207 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
200 208 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
201 209 es seteado a 0
202 210
203 211 Inputs:
204 212 None
205 213
206 214 Return:
207 215 None
208 216
209 217 Affected:
210 218 self.profileIndex
211 219 self.datablock
212 220 self.flagIsNewFile
213 221 self.flagIsNewBlock
214 222 self.nTotalBlocks
215 223
216 224 Exceptions:
217 225 Si un bloque leido no es un bloque valido
226 =======
227 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
228 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
229 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
230 es seteado a 0
231
232 Inputs:
233 None
234
235 Return:
236 None
237
238 Affected:
239 self.profileIndex
240 self.datablock
241 self.flagIsNewFile
242 self.flagIsNewBlock
243 self.nTotalBlocks
244
245 Exceptions:
246 Si un bloque leido no es un bloque valido
247 >>>>>>> online_data_hour
218 248 """
219 249
220 250 # if self.server is not None:
221 251 # self.zBlock = self.receiver.recv()
222 252 # self.zHeader = self.zBlock[:24]
223 253 # self.zDataBlock = self.zBlock[24:]
224 254 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
225 255 # self.processingHeaderObj.profilesPerBlock = 240
226 256 # self.processingHeaderObj.nHeights = 248
227 257 # self.systemHeaderObj.nChannels
228 258 # else:
229 259 current_pointer_location = self.fp.tell()
230 260 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
231 261
232 262 try:
233 263 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
234 264 except:
235 265 #print "The read block (%3d) has not enough data" %self.nReadBlocks
236 266
237 267 if self.waitDataBlock(pointer_location=current_pointer_location):
238 268 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
239 269 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
240 270 # return 0
241 271
242 272 #Dimensions : nChannels, nProfiles, nSamples
243 273
244 274 junk = numpy.transpose(junk, (2,0,1))
245 275 self.datablock = junk['real'] + junk['imag']*1j
246 276
247 277 self.profileIndex = 0
248 278
249 279 self.flagIsNewFile = 0
250 280 self.flagIsNewBlock = 1
251 281
252 282 self.nTotalBlocks += 1
253 283 self.nReadBlocks += 1
254 284
255 285 return 1
256 286
257 287 def getFirstHeader(self):
258 288
259 289 self.getBasicHeader()
260 290
261 291 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
262 292
263 293 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
264 294
265 295 if self.nTxs > 1:
266 296 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
267 297
268 298 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
269 299
270 300 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
271 301 #
272 302 # if self.radarControllerHeaderObj.code is not None:
273 303 #
274 304 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
275 305 #
276 306 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
277 307 #
278 308 # self.dataOut.code = self.radarControllerHeaderObj.code
279 309
280 310 self.dataOut.dtype = self.dtype
281 311
282 312 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
283 313
284 314 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
285 315
286 316 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
287 317
288 318 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
289 319
290 320 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
291 321
292 322 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
293 323
294 324 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
295 325
296 326 def reshapeData(self):
297 327
298 328 if self.nTxs < 0:
299 329 return
300 330
301 331 if self.nTxs == 1:
302 332 return
303 333
304 334 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
305 335 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
306 336
307 337 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
308 338 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
309 339
310 340 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
311 341
312 342 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
313 343 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
314 344 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
315 345
316 346 return
317 347
318 348 def readFirstHeaderFromServer(self):
319 349
320 350 self.getFirstHeader()
321 351
322 352 self.firstHeaderSize = self.basicHeaderObj.size
323 353
324 354 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
325 355 if datatype == 0:
326 356 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
327 357 elif datatype == 1:
328 358 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
329 359 elif datatype == 2:
330 360 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
331 361 elif datatype == 3:
332 362 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
333 363 elif datatype == 4:
334 364 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
335 365 elif datatype == 5:
336 366 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
337 367 else:
338 368 raise ValueError, 'Data type was not defined'
339 369
340 370 self.dtype = datatype_str
341 371 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
342 372 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
343 373 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
344 374 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
345 375 self.getBlockDimension()
346 376
347 377
348 378 def getFromServer(self):
349 379 self.flagDiscontinuousBlock = 0
350 380 self.profileIndex = 0
351 381 self.flagIsNewBlock = 1
352 382 self.dataOut.flagNoData = False
353 383 self.nTotalBlocks += 1
354 384 self.nReadBlocks += 1
355 385 self.blockPointer = 0
356 386
357 387 block = self.receiver.recv()
358 388
359 389 self.basicHeaderObj.read(block[self.blockPointer:])
360 390 self.blockPointer += self.basicHeaderObj.length
361 391 self.systemHeaderObj.read(block[self.blockPointer:])
362 392 self.blockPointer += self.systemHeaderObj.length
363 393 self.radarControllerHeaderObj.read(block[self.blockPointer:])
364 394 self.blockPointer += self.radarControllerHeaderObj.length
365 395 self.processingHeaderObj.read(block[self.blockPointer:])
366 396 self.blockPointer += self.processingHeaderObj.length
367 397 self.readFirstHeaderFromServer()
368 398
369 399 timestamp = self.basicHeaderObj.get_datatime()
370 400 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
371 401 current_pointer_location = self.blockPointer
372 402 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
373 403
374 404 try:
375 405 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
376 406 except:
377 407 #print "The read block (%3d) has not enough data" %self.nReadBlocks
378 408 if self.waitDataBlock(pointer_location=current_pointer_location):
379 409 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
380 410 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
381 411 # return 0
382 412
383 413 #Dimensions : nChannels, nProfiles, nSamples
384 414
385 415 junk = numpy.transpose(junk, (2,0,1))
386 416 self.datablock = junk['real'] + junk['imag'] * 1j
387 417 self.profileIndex = 0
388 418 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
389 419 if self.selBlocktime != None:
390 420 if self.dataOut.nCohInt is not None:
391 421 nCohInt = self.dataOut.nCohInt
392 422 else:
393 423 nCohInt = 1
394 424 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
395 425 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
396 426 datasize = self.dataOut.data.shape[1]
397 427 if datasize < self.selBlocksize:
398 428 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
399 429 buffer[:,:datasize,:] = self.dataOut.data
400 430 self.dataOut.data = buffer
401 431 self.profileIndex = blockIndex
402 432
403 433 self.dataOut.flagDataAsBlock = True
404 434 self.flagIsNewBlock = 1
405 435 self.dataOut.realtime = self.online
406 436
407 437 return self.dataOut.data
408 438
409 439 def getData(self):
410 440 """
441 <<<<<<< HEAD
411 442 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
412 443 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
413 444 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
414 445 "readNextBlock"
415 446
416 447 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
417 448
418 449 Return:
419 450
420 451 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
421 452 es igual al total de perfiles leidos desde el archivo.
422 453
423 454 Si self.getByBlock == False:
424 455
425 456 self.dataOut.data = buffer[:, thisProfile, :]
426 457
427 458 shape = [nChannels, nHeis]
428 459
429 460 Si self.getByBlock == True:
430 461
431 462 self.dataOut.data = buffer[:, :, :]
432 463
433 464 shape = [nChannels, nProfiles, nHeis]
434 465
435 466 Variables afectadas:
436 467 self.dataOut
437 468 self.profileIndex
438 469
439 470 Affected:
440 471 self.dataOut
441 472 self.profileIndex
442 473 self.flagDiscontinuousBlock
443 474 self.flagIsNewBlock
475 =======
476 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
477 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
478 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
479 "readNextBlock"
480
481 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
482
483 Return:
484
485 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
486 es igual al total de perfiles leidos desde el archivo.
487
488 Si self.getByBlock == False:
489
490 self.dataOut.data = buffer[:, thisProfile, :]
491
492 shape = [nChannels, nHeis]
493
494 Si self.getByBlock == True:
495
496 self.dataOut.data = buffer[:, :, :]
497
498 shape = [nChannels, nProfiles, nHeis]
499
500 Variables afectadas:
501 self.dataOut
502 self.profileIndex
503
504 Affected:
505 self.dataOut
506 self.profileIndex
507 self.flagDiscontinuousBlock
508 self.flagIsNewBlock
509 >>>>>>> online_data_hour
444 510 """
445 511 if self.flagNoMoreFiles:
446 512 self.dataOut.flagNoData = True
447 513 print 'Process finished'
448 514 return 0
449 515 self.flagDiscontinuousBlock = 0
450 516 self.flagIsNewBlock = 0
451 517 if self.__hasNotDataInBuffer():
452 518 if not( self.readNextBlock() ):
453 519 return 0
454 520
455 521 self.getFirstHeader()
456 522
457 523 self.reshapeData()
458 524 if self.datablock is None:
459 525 self.dataOut.flagNoData = True
460 526 return 0
461 527
462 528 if not self.getByBlock:
463 529
464 530 """
531 <<<<<<< HEAD
465 532 Return profile by profile
466 533
467 534 If nTxs > 1 then one profile is divided by nTxs and number of total
468 535 blocks is increased by nTxs (nProfiles *= nTxs)
536 =======
537 Return profile by profile
538
539 If nTxs > 1 then one profile is divided by nTxs and number of total
540 blocks is increased by nTxs (nProfiles *= nTxs)
541 >>>>>>> online_data_hour
469 542 """
470 543 self.dataOut.flagDataAsBlock = False
471 544 self.dataOut.data = self.datablock[:,self.profileIndex,:]
472 545 self.dataOut.profileIndex = self.profileIndex
473 546
474 547 self.profileIndex += 1
548 <<<<<<< HEAD
475 549
476 550 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
477 551 # """
478 552 # Return all block
479 553 # """
480 554 # self.dataOut.flagDataAsBlock = True
481 555 # self.dataOut.data = self.datablock
482 556 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
483 557 #
484 558 # self.profileIndex = self.dataOut.nProfiles
485 559
560 =======
561
562 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
563 # """
564 # Return all block
565 # """
566 # self.dataOut.flagDataAsBlock = True
567 # self.dataOut.data = self.datablock
568 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
569 #
570 # self.profileIndex = self.dataOut.nProfiles
571
572 >>>>>>> online_data_hour
486 573 else:
487 574 """
488 Return a block
575 Return a block
489 576 """
490 577 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
491 578 if self.selBlocktime != None:
492 579 if self.dataOut.nCohInt is not None:
493 580 nCohInt = self.dataOut.nCohInt
494 581 else:
495 582 nCohInt = 1
496 583 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
497 584
498 585 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
499 586 self.profileIndex += self.selBlocksize
500 587 datasize = self.dataOut.data.shape[1]
501 588
502 589 if datasize < self.selBlocksize:
503 590 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
504 591 buffer[:,:datasize,:] = self.dataOut.data
505 592
506 593 while datasize < self.selBlocksize: #Not enough profiles to fill the block
507 594 if not( self.readNextBlock() ):
508 595 return 0
509 596 self.getFirstHeader()
510 597 self.reshapeData()
511 598 if self.datablock is None:
512 599 self.dataOut.flagNoData = True
513 600 return 0
514 601 #stack data
515 602 blockIndex = self.selBlocksize - datasize
516 603 datablock1 = self.datablock[:,:blockIndex,:]
517 604
518 605 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
519 606 datasize += datablock1.shape[1]
520 607
521 608 self.dataOut.data = buffer
522 609 self.profileIndex = blockIndex
523 610
524 611 self.dataOut.flagDataAsBlock = True
525 612 self.dataOut.nProfiles = self.dataOut.data.shape[1]
526 613
527 614 self.dataOut.flagNoData = False
528 615
529 616 self.getBasicHeader()
530 617
531 618 self.dataOut.realtime = self.online
532 619
533 620 return self.dataOut.data
534 621
535 622 class VoltageWriter(JRODataWriter, Operation):
536 623 """
537 624 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
538 625 de los datos siempre se realiza por bloques.
539 626 """
540 627
541 628 ext = ".r"
542 629
543 630 optchar = "D"
544 631
545 632 shapeBuffer = None
546 633
547 634
548 635 def __init__(self, **kwargs):
549 636 """
550 637 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
551 638
552 639 Affected:
553 640 self.dataOut
554 641
555 642 Return: None
556 643 """
557 644 Operation.__init__(self, **kwargs)
558 645
559 646 self.nTotalBlocks = 0
560 647
561 648 self.profileIndex = 0
562 649
563 650 self.isConfig = False
564 651
565 652 self.fp = None
566 653
567 654 self.flagIsNewFile = 1
568 655
569 656 self.blockIndex = 0
570 657
571 658 self.flagIsNewBlock = 0
572 659
573 660 self.setFile = None
574 661
575 662 self.dtype = None
576 663
577 664 self.path = None
578 665
579 666 self.filename = None
580 667
581 668 self.basicHeaderObj = BasicHeader(LOCALTIME)
582 669
583 670 self.systemHeaderObj = SystemHeader()
584 671
585 672 self.radarControllerHeaderObj = RadarControllerHeader()
586 673
587 674 self.processingHeaderObj = ProcessingHeader()
588 675
589 676 def hasAllDataInBuffer(self):
590 677 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
591 678 return 1
592 679 return 0
593 680
594 681
595 682 def setBlockDimension(self):
596 683 """
597 684 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
598 685
599 686 Affected:
600 687 self.shape_spc_Buffer
601 688 self.shape_cspc_Buffer
602 689 self.shape_dc_Buffer
603 690
604 691 Return: None
605 692 """
606 693 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
607 694 self.processingHeaderObj.nHeights,
608 695 self.systemHeaderObj.nChannels)
609 696
610 697 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
611 698 self.processingHeaderObj.profilesPerBlock,
612 699 self.processingHeaderObj.nHeights),
613 700 dtype=numpy.dtype('complex64'))
614 701
615 702 def writeBlock(self):
616 703 """
617 704 Escribe el buffer en el file designado
618 705
619 706 Affected:
620 707 self.profileIndex
621 708 self.flagIsNewFile
622 709 self.flagIsNewBlock
623 710 self.nTotalBlocks
624 711 self.blockIndex
625 712
626 713 Return: None
627 714 """
628 715 data = numpy.zeros( self.shapeBuffer, self.dtype )
629 716
630 717 junk = numpy.transpose(self.datablock, (1,2,0))
631 718
632 719 data['real'] = junk.real
633 720 data['imag'] = junk.imag
634 721
635 722 data = data.reshape( (-1) )
636 723
637 724 data.tofile( self.fp )
638 725
639 726 self.datablock.fill(0)
640 727
641 728 self.profileIndex = 0
642 729 self.flagIsNewFile = 0
643 730 self.flagIsNewBlock = 1
644 731
645 732 self.blockIndex += 1
646 733 self.nTotalBlocks += 1
647 734
648 735 # print "[Writing] Block = %04d" %self.blockIndex
649 736
650 737 def putData(self):
651 738 """
652 739 Setea un bloque de datos y luego los escribe en un file
653 740
654 741 Affected:
655 742 self.flagIsNewBlock
656 743 self.profileIndex
657 744
658 745 Return:
659 746 0 : Si no hay data o no hay mas files que puedan escribirse
660 747 1 : Si se escribio la data de un bloque en un file
661 748 """
662 749 if self.dataOut.flagNoData:
663 750 return 0
664 751
665 752 self.flagIsNewBlock = 0
666 753
667 754 if self.dataOut.flagDiscontinuousBlock:
668 755 self.datablock.fill(0)
669 756 self.profileIndex = 0
670 757 self.setNextFile()
671 758
672 759 if self.profileIndex == 0:
673 760 self.setBasicHeader()
674 761
675 762 self.datablock[:,self.profileIndex,:] = self.dataOut.data
676 763
677 764 self.profileIndex += 1
678 765
679 766 if self.hasAllDataInBuffer():
680 767 #if self.flagIsNewFile:
681 768 self.writeNextBlock()
682 769 # self.setFirstHeader()
683 770
684 771 return 1
685 772
686 773 def __getBlockSize(self):
687 774 '''
688 775 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
689 776 '''
690 777
691 778 dtype_width = self.getDtypeWidth()
692 779
693 780 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
694 781
695 782 return blocksize
696 783
697 784 def setFirstHeader(self):
698 785
699 786 """
700 787 Obtiene una copia del First Header
701 788
702 789 Affected:
703 790 self.systemHeaderObj
704 791 self.radarControllerHeaderObj
705 792 self.dtype
706 793
707 794 Return:
708 795 None
709 796 """
710 797
711 798 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
712 799 self.systemHeaderObj.nChannels = self.dataOut.nChannels
713 800 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
714 801
715 802 self.processingHeaderObj.dtype = 0 # Voltage
716 803 self.processingHeaderObj.blockSize = self.__getBlockSize()
717 804 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
718 805 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
719 806 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
720 807 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
721 808 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
722 809 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
723 810
724 811 if self.dataOut.code is not None:
725 812 self.processingHeaderObj.code = self.dataOut.code
726 813 self.processingHeaderObj.nCode = self.dataOut.nCode
727 814 self.processingHeaderObj.nBaud = self.dataOut.nBaud
728 815
729 816 if self.processingHeaderObj.nWindows != 0:
730 817 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
731 818 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
732 819 self.processingHeaderObj.nHeights = self.dataOut.nHeights
733 820 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
734 821
735 822 self.processingHeaderObj.processFlags = self.getProcessFlags()
736 823
737 824 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -1,1007 +1,1021
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13 13
14 14 import time
15 15
16 16 import threading
17 17 Thread = threading.Thread
18 18
19 19 # try:
20 20 # from gevent import sleep
21 21 # except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25
26 26 class Remote(Thread):
27 27 """
28 28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 29 used to upload or download files remotely.
30 30
31 31 Non-standard Python modules used:
32 32 None
33 33
34 34 Written by:
35 35 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 36 """
37 37
38 38 server = None
39 39 username = None
40 40 password = None
41 41 remotefolder = None
42 42
43 43 period = 60
44 44 fileList = []
45 45 bussy = False
46 46
47 47 def __init__(self, server, username, password, remotefolder, period=60):
48 48
49 49 Thread.__init__(self)
50 50
51 51 self.setDaemon(True)
52 52
53 53 self.status = 0
54 54
55 55 self.__server = server
56 56 self.__username = username
57 57 self.__password = password
58 58 self.__remotefolder = remotefolder
59 59
60 60 self.period = period
61 61
62 62 self.fileList = []
63 63 self.bussy = False
64 64
65 65 self.stopFlag = False
66 66
67 67 print "[Remote Server] Opening server: %s" %self.__server
68 68 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
69 69 print "[Remote Server] %s server was opened successfully" %self.__server
70 70
71 71 self.close()
72 72
73 73 self.mutex = threading.Lock()
74 74
75 75 def stop(self):
76 76
77 77 self.stopFlag = True
78 78 self.join(10)
79 79
80 80 def open(self):
81 81 """
82 82 Connect to server and create a connection class (FTP or SSH) to remote server.
83 83 """
84 84 raise NotImplementedError, "Implement this method in child class"
85 85
86 86 def close(self):
87 87 """
88 88 Close connection to server
89 89 """
90 90 raise NotImplementedError, "Implement this method in child class"
91 91
92 92 def mkdir(self, remotefolder):
93 93 """
94 94 Create a folder remotely
95 95 """
96 96 raise NotImplementedError, "Implement this method in child class"
97 97
98 98 def cd(self, remotefolder):
99 99 """
100 100 Change working directory in remote server
101 101 """
102 102 raise NotImplementedError, "Implement this method in child class"
103 103
104 104 def download(self, filename, localfolder=None):
105 105 """
106 106 Download a file from server to local host
107 107 """
108 108 raise NotImplementedError, "Implement this method in child class"
109 109
110 110 def sendFile(self, fullfilename):
111 111 """
112 112 sendFile method is used to upload a local file to the current directory in remote server
113 113
114 114 Inputs:
115 115 fullfilename - full path name of local file to store in remote directory
116 116
117 117 Returns:
118 118 0 in error case else 1
119 119 """
120 120 raise NotImplementedError, "Implement this method in child class"
121 121
122 122 def upload(self, fullfilename, remotefolder=None):
123 123 """
124 124 upload method is used to upload a local file to remote directory. This method changes
125 125 working directory before sending a file.
126 126
127 127 Inputs:
128 128 fullfilename - full path name of local file to store in remote directory
129 129
130 130 remotefolder - remote directory
131 131
132 132 Returns:
133 133 0 in error case else 1
134 134 """
135 135 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
136 136
137 137 if not self.status:
138 138 return 0
139 139
140 140 if remotefolder == None:
141 141 remotefolder = self.remotefolder
142 142
143 143 if not self.cd(remotefolder):
144 144 return 0
145 145
146 146 if not self.sendFile(fullfilename):
147 147 print "[Remote Server] Error uploading file %s" %fullfilename
148 148 return 0
149 149
150 150 print "[Remote Server] upload finished successfully"
151 151
152 152 return 1
153 153
154 154 def delete(self, filename):
155 155 """
156 156 Remove a file from remote server
157 157 """
158 158 pass
159 159
160 160 def updateFileList(self, fileList):
161 161 """
162 162 Remove a file from remote server
163 163 """
164 164
165 165 if fileList == self.fileList:
166 166 return 0
167 167
168 168 self.mutex.acquire()
169 <<<<<<< HEAD
169 170 # init = time.time()
170 171 #
171 172 # while(self.bussy):
172 173 # sleep(0.1)
173 174 # if time.time() - init > 2*self.period:
174 175 # return 0
176 =======
177 # init = time.time()
178 #
179 # while(self.bussy):
180 # sleep(0.1)
181 # if time.time() - init > 2*self.period:
182 # return 0
183 >>>>>>> online_data_hour
175 184
176 185 self.fileList = fileList
177 186 self.mutex.release()
178 187 return 1
179 188
180 189 def run(self):
181 190
182 191 if not self.status:
183 192 print "Finishing FTP service"
184 193 return
185 194
186 195 if not self.cd(self.remotefolder):
187 196 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
188 197
189 198 while True:
190 199
191 200 for i in range(self.period):
192 201 if self.stopFlag:
193 202 break
194 203 sleep(1)
195 204
196 205 if self.stopFlag:
197 206 break
198 207
208 <<<<<<< HEAD
199 209 # self.bussy = True
210 =======
211 # self.bussy = True
212 >>>>>>> online_data_hour
200 213 self.mutex.acquire()
201 214
202 215 print "[Remote Server] Opening %s" %self.__server
203 216 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
204 217 self.mutex.release()
205 218 continue
206 219
207 220 for thisFile in self.fileList:
208 221 self.upload(thisFile, self.remotefolder)
209 222
210 223 print "[Remote Server] Closing %s" %self.__server
211 224 self.close()
212 225
213 226 self.mutex.release()
214 227 # self.bussy = False
215 228
216 229 print "[Remote Server] Thread stopped successfully"
217 230
218 231 class FTPClient(Remote):
219 232
220 233 __ftpClientObj = None
221 234
222 235 def __init__(self, server, username, password, remotefolder, period=60):
223 236 """
224 237 """
225 238 Remote.__init__(self, server, username, password, remotefolder, period)
226 239
227 240 def open(self, server, username, password, remotefolder):
228 241
229 242 """
230 243 This method is used to set FTP parameters and establish a connection to remote server
231 244
232 245 Inputs:
233 246 server - remote server IP Address
234 247
235 248 username - remote server Username
236 249
237 250 password - remote server password
238 251
239 252 remotefolder - remote server current working directory
240 253
241 254 Return:
242 255 Boolean - Returns 1 if a connection has been established, 0 otherwise
243 256
244 257 Affects:
245 258 self.status - in case of error or fail connection this parameter is set to 0 else 1
246 259
247 260 """
248 261
249 262 if server == None:
250 263 raise ValueError, "FTP server should be defined"
251 264
252 265 if username == None:
253 266 raise ValueError, "FTP username should be defined"
254 267
255 268 if password == None:
256 269 raise ValueError, "FTP password should be defined"
257 270
258 271 if remotefolder == None:
259 272 raise ValueError, "FTP remote folder should be defined"
260 273
261 274 try:
262 275 ftpClientObj = ftplib.FTP(server)
263 276 except ftplib.all_errors, e:
264 277 print "[FTP Server]: FTP server connection fail: %s" %server
265 278 print "[FTP Server]:", e
266 279 self.status = 0
267 280 return 0
268 281
269 282 try:
270 283 ftpClientObj.login(username, password)
271 284 except ftplib.all_errors:
272 285 print "[FTP Server]: FTP username or password are incorrect"
273 286 self.status = 0
274 287 return 0
275 288
276 289 if remotefolder == None:
277 290 remotefolder = ftpClientObj.pwd()
278 291 else:
279 292 try:
280 293 ftpClientObj.cwd(remotefolder)
281 294 except ftplib.all_errors:
282 295 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
283 296 remotefolder = ftpClientObj.pwd()
284 297
285 298 self.server = server
286 299 self.username = username
287 300 self.password = password
288 301 self.remotefolder = remotefolder
289 302 self.__ftpClientObj = ftpClientObj
290 303 self.status = 1
291 304
292 305 return 1
293 306
294 307 def close(self):
295 308 """
296 309 Close connection to remote server
297 310 """
298 311 if not self.status:
299 312 return 0
300 313
301 314 self.__ftpClientObj.close()
302 315
303 316 def mkdir(self, remotefolder):
304 317 """
305 318 mkdir is used to make a new directory in remote server
306 319
307 320 Input:
308 321 remotefolder - directory name
309 322
310 323 Return:
311 324 0 in error case else 1
312 325 """
313 326 if not self.status:
314 327 return 0
315 328
316 329 try:
317 330 self.__ftpClientObj.mkd(dirname)
318 331 except ftplib.all_errors:
319 332 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
320 333 return 0
321 334
322 335 return 1
323 336
324 337 def cd(self, remotefolder):
325 338 """
326 339 cd is used to change remote working directory on server
327 340
328 341 Input:
329 342 remotefolder - current working directory
330 343
331 344 Affects:
332 345 self.remotefolder
333 346
334 347 Return:
335 348 0 in case of error else 1
336 349 """
337 350 if not self.status:
338 351 return 0
339 352
340 353 if remotefolder == self.remotefolder:
341 354 return 1
342 355
343 356 try:
344 357 self.__ftpClientObj.cwd(remotefolder)
345 358 except ftplib.all_errors:
346 359 print '[FTP Server]: Error changing to %s' %remotefolder
347 360 print '[FTP Server]: Trying to create remote folder'
348 361
349 362 if not self.mkdir(remotefolder):
350 363 print '[FTP Server]: Remote folder could not be created'
351 364 return 0
352 365
353 366 try:
354 367 self.__ftpClientObj.cwd(remotefolder)
355 368 except ftplib.all_errors:
356 369 return 0
357 370
358 371 self.remotefolder = remotefolder
359 372
360 373 return 1
361 374
362 375 def sendFile(self, fullfilename):
363 376
364 377 if not self.status:
365 378 return 0
366 379
367 380 fp = open(fullfilename, 'rb')
368 381
369 382 filename = os.path.basename(fullfilename)
370 383
371 384 command = "STOR %s" %filename
372 385
373 386 try:
374 387 self.__ftpClientObj.storbinary(command, fp)
375 388 except ftplib.all_errors, e:
376 389 print "[FTP Server]:", e
377 390 return 0
378 391
379 392 try:
380 393 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
381 394 except ftplib.all_errors, e:
382 395 print "[FTP Server]:", e
383 396
384 397 fp.close()
385 398
386 399 return 1
387 400
388 401 class SSHClient(Remote):
389 402
390 403 __sshClientObj = None
391 404 __scpClientObj = None
392 405
393 406 def __init__(self, server, username, password, remotefolder, period=60):
394 407 """
395 408 """
396 409 Remote.__init__(self, server, username, password, remotefolder, period)
397 410
398 411 def open(self, server, username, password, remotefolder, port=22):
399 412
400 413 """
401 414 This method is used to set SSH parameters and establish a connection to a remote server
402 415
403 416 Inputs:
404 417 server - remote server IP Address
405 418
406 419 username - remote server Username
407 420
408 421 password - remote server password
409 422
410 423 remotefolder - remote server current working directory
411 424
412 425 Return: void
413 426
414 427 Affects:
415 428 self.status - in case of error or fail connection this parameter is set to 0 else 1
416 429
417 430 """
418 431 import socket
419 432
420 433 if server == None:
421 434 raise ValueError, "SSH server should be defined"
422 435
423 436 if username == None:
424 437 raise ValueError, "SSH username should be defined"
425 438
426 439 if password == None:
427 440 raise ValueError, "SSH password should be defined"
428 441
429 442 if remotefolder == None:
430 443 raise ValueError, "SSH remote folder should be defined"
431 444
432 445 sshClientObj = paramiko.SSHClient()
433 446
434 447 sshClientObj.load_system_host_keys()
435 448 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
436 449
437 450 self.status = 0
438 451 try:
439 452 sshClientObj.connect(server, username=username, password=password, port=port)
440 453 except paramiko.AuthenticationException, e:
441 454 # print "SSH username or password are incorrect: %s"
442 455 print "[SSH Server]:", e
443 456 return 0
444 457 except SSHException, e:
445 458 print "[SSH Server]:", e
446 459 return 0
447 460 except socket.error:
448 461 self.status = 0
449 462 print "[SSH Server]:", e
450 463 return 0
451 464
452 465 self.status = 1
453 466 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
454 467
455 468 if remotefolder == None:
456 469 remotefolder = self.pwd()
457 470
458 471 self.server = server
459 472 self.username = username
460 473 self.password = password
461 474 self.__sshClientObj = sshClientObj
462 475 self.__scpClientObj = scpClientObj
463 476 self.status = 1
464 477
465 478 if not self.cd(remotefolder):
466 479 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
467 480 return 0
468 481
469 482 self.remotefolder = remotefolder
470 483
471 484 return 1
472 485
473 486 def close(self):
474 487 """
475 488 Close connection to remote server
476 489 """
477 490 if not self.status:
478 491 return 0
479 492
480 493 self.__scpClientObj.close()
481 494 self.__sshClientObj.close()
482 495
483 496 def __execute(self, command):
484 497 """
485 498 __execute a command on remote server
486 499
487 500 Input:
488 501 command - Exmaple 'ls -l'
489 502
490 503 Return:
491 504 0 in error case else 1
492 505 """
493 506 if not self.status:
494 507 return 0
495 508
496 509 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
497 510
498 511 result = stderr.readlines()
499 512 if len(result) > 1:
500 513 return 0
501 514
502 515 result = stdout.readlines()
503 516 if len(result) > 1:
504 517 return result[0][:-1]
505 518
506 519 return 1
507 520
508 521 def mkdir(self, remotefolder):
509 522 """
510 523 mkdir is used to make a new directory in remote server
511 524
512 525 Input:
513 526 remotefolder - directory name
514 527
515 528 Return:
516 529 0 in error case else 1
517 530 """
518 531
519 532 command = 'mkdir %s' %remotefolder
520 533
521 534 return self.__execute(command)
522 535
523 536 def pwd(self):
524 537
525 538 command = 'pwd'
526 539
527 540 return self.__execute(command)
528 541
529 542 def cd(self, remotefolder):
530 543 """
531 544 cd is used to change remote working directory on server
532 545
533 546 Input:
534 547 remotefolder - current working directory
535 548
536 549 Affects:
537 550 self.remotefolder
538 551
539 552 Return:
540 553 0 in case of error else 1
541 554 """
542 555 if not self.status:
543 556 return 0
544 557
545 558 if remotefolder == self.remotefolder:
546 559 return 1
547 560
548 561 chk_command = "cd %s; pwd" %remotefolder
549 562 mkdir_command = "mkdir %s" %remotefolder
550 563
551 564 if not self.__execute(chk_command):
552 565 if not self.__execute(mkdir_command):
553 566 self.remotefolder = None
554 567 return 0
555 568
556 569 self.remotefolder = remotefolder
557 570
558 571 return 1
559 572
560 573 def sendFile(self, fullfilename):
561 574
562 575 if not self.status:
563 576 return 0
564 577
565 578 try:
566 579 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
567 580 except scp.ScpError, e:
568 581 print "[SSH Server]", str(e)
569 582 return 0
570 583
571 584 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
572 585 command = 'chmod 775 %s' %remotefile
573 586
574 587 return self.__execute(command)
575 588
576 589 class SendToServer(ProcessingUnit):
577 590
578 591 def __init__(self, **kwargs):
579 592
580 593 ProcessingUnit.__init__(self, **kwargs)
581 594
582 595 self.isConfig = False
583 596 self.clientObj = None
584 597
585 598 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
586 599
587 600 self.clientObj = None
588 601 self.localfolder = localfolder
589 602 self.ext = ext
590 603 self.period = period
591 604
592 605 if str.lower(protocol) == 'ftp':
593 606 self.clientObj = FTPClient(server, username, password, remotefolder, period)
594 607
595 608 if str.lower(protocol) == 'ssh':
596 609 self.clientObj = SSHClient(server, username, password, remotefolder, period)
597 610
598 611 if not self.clientObj:
599 612 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
600 613
601 614 self.clientObj.start()
602 615
603 616 def findFiles(self):
604 617
605 618 if not type(self.localfolder) == list:
606 619 folderList = [self.localfolder]
607 620 else:
608 621 folderList = self.localfolder
609 622
610 623 #Remove duplicate items
611 624 folderList = list(set(folderList))
612 625
613 626 fullfilenameList = []
614 627
615 628 for thisFolder in folderList:
616 629
617 630 print "[Remote Server]: Searching files on %s" %thisFolder
618 631
619 632 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
620 633
621 634 if len(filenameList) < 1:
635
622 636 continue
623 637
624 638 for thisFile in filenameList:
625 639 fullfilename = os.path.join(thisFolder, thisFile)
626 640
627 641 if fullfilename in fullfilenameList:
628 642 continue
629 643
630 644 #Only files modified in the last 30 minutes are considered
631 645 if os.path.getmtime(fullfilename) < time.time() - 30*60:
632 646 continue
633 647
634 648 fullfilenameList.append(fullfilename)
635 649
636 650 return fullfilenameList
637 651
638 652 def run(self, **kwargs):
639 653 if not self.isConfig:
640 654 self.init = time.time()
641 655 self.setup(**kwargs)
642 656 self.isConfig = True
643 657
644 658 if not self.clientObj.is_alive():
645 659 print "[Remote Server]: Restarting connection "
646 660 self.setup(**kwargs)
647 661
648 662 if time.time() - self.init >= self.period:
649 663 fullfilenameList = self.findFiles()
650 664
651 665 if self.clientObj.updateFileList(fullfilenameList):
652 666 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
653 667 self.init = time.time()
654 668
655 669 def close(self):
656 670 print "[Remote Server] Stopping thread"
657 671 self.clientObj.stop()
658 672
659 673
660 674 class FTP(object):
661 675 """
662 676 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
663 677
664 678 Non-standard Python modules used: None
665 679
666 680 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
667 681 """
668 682
669 683 def __init__(self,server = None, username=None, password=None, remotefolder=None):
670 684 """
671 685 This method is used to setting parameters for FTP and establishing connection to remote server
672 686
673 687 Inputs:
674 688 server - remote server IP Address
675 689
676 690 username - remote server Username
677 691
678 692 password - remote server password
679 693
680 694 remotefolder - remote server current working directory
681 695
682 696 Return: void
683 697
684 698 Affects:
685 699 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
686 700
687 701 self.folderList - sub-folder list of remote folder
688 702
689 703 self.fileList - file list of remote folder
690 704
691 705
692 706 """
693 707
694 708 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
695 709 server, username, password, remotefolder = self.parmsByDefault()
696 710
697 711 self.server = server
698 712 self.username = username
699 713 self.password = password
700 714 self.remotefolder = remotefolder
701 715 self.file = None
702 716 self.ftp = None
703 717 self.status = 0
704 718
705 719 try:
706 720 self.ftp = ftplib.FTP(self.server)
707 721 self.ftp.login(self.username,self.password)
708 722 self.ftp.cwd(self.remotefolder)
709 723 # print 'Connect to FTP Server: Successfully'
710 724
711 725 except ftplib.all_errors:
712 726 print 'Error FTP Service'
713 727 self.status = 1
714 728 return
715 729
716 730
717 731
718 732 self.dirList = []
719 733
720 734 try:
721 735 self.dirList = self.ftp.nlst()
722 736
723 737 except ftplib.error_perm, resp:
724 738 if str(resp) == "550 No files found":
725 739 print "no files in this directory"
726 740 self.status = 1
727 741 return
728 742
729 743 except ftplib.all_errors:
730 744 print 'Error Displaying Dir-Files'
731 745 self.status = 1
732 746 return
733 747
734 748 self.fileList = []
735 749 self.folderList = []
736 750 #only for test
737 751 for f in self.dirList:
738 752 name, ext = os.path.splitext(f)
739 753 if ext != '':
740 754 self.fileList.append(f)
741 755 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
742 756
743 757 def parmsByDefault(self):
744 758 server = 'jro-app.igp.gob.pe'
745 759 username = 'wmaster'
746 760 password = 'mst2010vhf'
747 761 remotefolder = '/home/wmaster/graficos'
748 762
749 763 return server, username, password, remotefolder
750 764
751 765
752 766 def mkd(self,dirname):
753 767 """
754 768 mkd is used to make directory in remote server
755 769
756 770 Input:
757 771 dirname - directory name
758 772
759 773 Return:
760 774 1 in error case else 0
761 775 """
762 776 try:
763 777 self.ftp.mkd(dirname)
764 778 except:
765 779 print 'Error creating remote folder:%s'%dirname
766 780 return 1
767 781
768 782 return 0
769 783
770 784
771 785 def delete(self,filename):
772 786 """
773 787 delete is used to delete file in current working directory of remote server
774 788
775 789 Input:
776 790 filename - filename to delete in remote folder
777 791
778 792 Return:
779 793 1 in error case else 0
780 794 """
781 795
782 796 try:
783 797 self.ftp.delete(filename)
784 798 except:
785 799 print 'Error deleting remote file:%s'%filename
786 800 return 1
787 801
788 802 return 0
789 803
790 804 def download(self,filename,localfolder):
791 805 """
792 806 download is used to downloading file from remote folder into local folder
793 807
794 808 Inputs:
795 809 filename - filename to donwload
796 810
797 811 localfolder - directory local to store filename
798 812
799 813 Returns:
800 814 self.status - 1 in error case else 0
801 815 """
802 816
803 817 self.status = 0
804 818
805 819
806 820 if not(filename in self.fileList):
807 821 print 'filename:%s not exists'%filename
808 822 self.status = 1
809 823 return self.status
810 824
811 825 newfilename = os.path.join(localfolder,filename)
812 826
813 827 self.file = open(newfilename, 'wb')
814 828
815 829 try:
816 830 print 'Download: ' + filename
817 831 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
818 832 print 'Download Complete'
819 833 except ftplib.all_errors:
820 834 print 'Error Downloading ' + filename
821 835 self.status = 1
822 836 return self.status
823 837
824 838 self.file.close()
825 839
826 840 return self.status
827 841
828 842
829 843 def __handleDownload(self,block):
830 844 """
831 845 __handleDownload is used to handle writing file
832 846 """
833 847 self.file.write(block)
834 848
835 849
836 850 def upload(self,filename,remotefolder=None):
837 851 """
838 852 upload is used to uploading local file to remote directory
839 853
840 854 Inputs:
841 855 filename - full path name of local file to store in remote directory
842 856
843 857 remotefolder - remote directory
844 858
845 859 Returns:
846 860 self.status - 1 in error case else 0
847 861 """
848 862
849 863 if remotefolder == None:
850 864 remotefolder = self.remotefolder
851 865
852 866 self.status = 0
853 867
854 868 try:
855 869 self.ftp.cwd(remotefolder)
856 870
857 871 self.file = open(filename, 'rb')
858 872
859 873 (head, tail) = os.path.split(filename)
860 874
861 875 command = "STOR " + tail
862 876
863 877 print 'Uploading: ' + tail
864 878 self.ftp.storbinary(command, self.file)
865 879 print 'Upload Completed'
866 880
867 881 except ftplib.all_errors:
868 882 print 'Error Uploading ' + tail
869 883 self.status = 1
870 884 return self.status
871 885
872 886 self.file.close()
873 887
874 888 #back to initial directory in __init__()
875 889 self.ftp.cwd(self.remotefolder)
876 890
877 891 return self.status
878 892
879 893
880 894 def dir(self,remotefolder):
881 895 """
882 896 dir is used to change working directory of remote server and get folder and file list
883 897
884 898 Input:
885 899 remotefolder - current working directory
886 900
887 901 Affects:
888 902 self.fileList - file list of working directory
889 903
890 904 Return:
891 905 infoList - list with filenames and size of file in bytes
892 906
893 907 self.folderList - folder list
894 908 """
895 909
896 910 self.remotefolder = remotefolder
897 911 print 'Change to ' + self.remotefolder
898 912 try:
899 913 self.ftp.cwd(remotefolder)
900 914 except ftplib.all_errors:
901 915 print 'Error Change to ' + self.remotefolder
902 916 infoList = None
903 917 self.folderList = None
904 918 return infoList,self.folderList
905 919
906 920 self.dirList = []
907 921
908 922 try:
909 923 self.dirList = self.ftp.nlst()
910 924
911 925 except ftplib.error_perm, resp:
912 926 if str(resp) == "550 No files found":
913 927 print "no files in this directory"
914 928 infoList = None
915 929 self.folderList = None
916 930 return infoList,self.folderList
917 931 except ftplib.all_errors:
918 932 print 'Error Displaying Dir-Files'
919 933 infoList = None
920 934 self.folderList = None
921 935 return infoList,self.folderList
922 936
923 937 infoList = []
924 938 self.fileList = []
925 939 self.folderList = []
926 940 for f in self.dirList:
927 941 name,ext = os.path.splitext(f)
928 942 if ext != '':
929 943 self.fileList.append(f)
930 944 value = (f,self.ftp.size(f))
931 945 infoList.append(value)
932 946
933 947 if ext == '':
934 948 self.folderList.append(f)
935 949
936 950 return infoList,self.folderList
937 951
938 952
939 953 def close(self):
940 954 """
941 955 close is used to close and end FTP connection
942 956
943 957 Inputs: None
944 958
945 959 Return: void
946 960
947 961 """
948 962 self.ftp.close()
949 963
950 964 class SendByFTP(Operation):
951 965
952 966 def __init__(self, **kwargs):
953 967 Operation.__init__(self, **kwargs)
954 968 self.status = 1
955 969 self.counter = 0
956 970
957 971 def error_print(self, ValueError):
958 972
959 973 print ValueError, 'Error FTP'
960 974 print "don't worry the program is running..."
961 975
962 976 def worker_ftp(self, server, username, password, remotefolder, filenameList):
963 977
964 978 self.ftpClientObj = FTP(server, username, password, remotefolder)
965 979 for filename in filenameList:
966 980 self.ftpClientObj.upload(filename)
967 981 self.ftpClientObj.close()
968 982
969 983 def ftp_thread(self, server, username, password, remotefolder):
970 984 if not(self.status):
971 985 return
972 986
973 987 import multiprocessing
974 988
975 989 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
976 990 p.start()
977 991
978 992 p.join(3)
979 993
980 994 if p.is_alive():
981 995 p.terminate()
982 996 p.join()
983 997 print 'killing ftp process...'
984 998 self.status = 0
985 999 return
986 1000
987 1001 self.status = 1
988 1002 return
989 1003
990 1004 def filterByExt(self, ext, localfolder):
991 1005 fnameList = glob.glob1(localfolder,ext)
992 1006 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
993 1007
994 1008 if len(self.filenameList) == 0:
995 1009 self.status = 0
996 1010
997 1011 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
998 1012
999 1013 self.counter += 1
1000 1014 if self.counter >= period:
1001 1015 self.filterByExt(ext, localfolder)
1002 1016
1003 1017 self.ftp_thread(server, username, password, remotefolder)
1004 1018
1005 1019 self.counter = 0
1006 1020
1007 1021 self.status = 1
Show file before | Show file after | Hide comments
@@ -1,85 +1,114
1 1
2 2 import os, sys
3 3
4 4 from schainpy.controller import Project
5 5
6 6 controllerObj = Project()
7 7 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
8 8
9 9 #-------------------------------------- Setup -----------------------------------------
10 10 #Verificar estas variables
11 11
12 12 #Path para los archivos
13 13 # path = '/mnt/jars/2016_08/NOCHE'
14 14 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
15 15 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
16 16 path = '/home/nanosat/data/jasmet'
17 17
18 18 #Path para los graficos
19 19 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
20 20
21 21 #Path para los archivos HDF5 de meteoros
22 22 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
23 23
24 24 #Fechas para busqueda de archivos
25 25 startDate = '2010/08/29'
26 26 endDate = '2017/09/11'
27 27 #Horas para busqueda de archivos
28 28 startTime = '00:00:00'
29 29 endTime = '23:59:59'
30 30
31 31
32 32 #------------------------------ Voltage Reading Unit ----------------------------------
33 33
34 34 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
35 35 path=path,
36 36 startDate=startDate,
37 37 endDate=endDate,
38 38 startTime=startTime,
39 39 endTime=endTime,
40 40 online=0,
41 41 delay=30,
42 42 walk=1,
43 43 getblock=1,
44 44 blocktime=100)
45 45
46 46 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
47 47
48 48 #-------------------------- Voltage Processing Unit ------------------------------------
49 49
50 50 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
51 51
52 52 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
53 53 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
54 54
55 55 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
56 56 opObj01.addParameter(name='frequency', value='30.e6', format='float')
57 57
58 opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
59 opObj01.addParameter(name='topLim', value='73', format='int')
60 opObj01.addParameter(name='botLim', value='71', format='int')
58 # opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
59 # opObj01.addParameter(name='topLim', value='73', format='int')
60 # opObj01.addParameter(name='botLim', value='71', format='int')
61 61
62 62 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
63 63
64 64 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
65 65 opObj03.addParameter(name='n', value='2', format='int')
66 66
67 procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId())
68 opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
69 opObj11.addParameter(name='id', value='237', format='int')
70 opObj11.addParameter(name='xmin', value='9.0', format='float')
71 opObj11.addParameter(name='xmax', value='16.0', format='float')
72 opObj11.addParameter(name='zmin', value='15.0', format='float')
73 opObj11.addParameter(name='zmax', value='50.0', format='float')
74
67 75 #--------------------------- Parameters Processing Unit ------------------------------------
68 76
77 <<<<<<< HEAD
69 78 procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
70 79 #
71 80 opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
72 81 opObj10.addParameter(name='azimuth', value='45', format='float')
73 82 opObj10.addParameter(name='hmin', value='60', format='float')
74 83 opObj10.addParameter(name='hmax', value='120', format='float')
75 84
76 85 opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
77 86 opObj12.addParameter(name='path', value=pathfile)
78 87 opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
79 88 opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
80 89 opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
81 90 opObj12.addParameter(name='mode',value='2',format='int')
82 91
83 92 #--------------------------------------------------------------------------------------------------
84 93
85 94 controllerObj.start()
95 =======
96 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
97 # #
98 # opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
99 # opObj10.addParameter(name='azimuth', value='45', format='float')
100 # opObj10.addParameter(name='hmin', value='60', format='float')
101 # opObj10.addParameter(name='hmax', value='120', format='float')
102
103 # opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
104 # opObj12.addParameter(name='path', value=pathfile)
105 # opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
106 # opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
107 # opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
108 # opObj12.addParameter(name='mode',value='2',format='int')
109
110 #--------------------------------------------------------------------------------------------------
111 controllerObj.createObjects()
112 controllerObj.connectObjects()
113 controllerObj.run()
114 >>>>>>> online_data_hour
General Comments 0
You need to be logged in to leave comments. Login now