##// END OF EJS Templates
schain-jc
RSS
cambios para online offline
José Chávez -
r1019:42311705b9c8
parent child
Show More
Side by Side Unified
Context file:
r1019:42311705b9c8 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1730 +1,1786
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isTimeInRange(thisTime, startTime, endTime):
87 87
88 88 if endTime >= startTime:
89 89 if (thisTime < startTime) or (thisTime > endTime):
90 90 return 0
91 91
92 92 return 1
93 93 else:
94 94 if (thisTime < startTime) and (thisTime > endTime):
95 95 return 0
96 96
97 97 return 1
98 98
99 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 100 """
101 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 102
103 103 Inputs:
104 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 105
106 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 107
108 108 endDate : fecha final del rango seleccionado en formato datetime.date
109 109
110 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 111
112 112 endTime : tiempo final del rango seleccionado en formato datetime.time
113 113
114 114 Return:
115 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 116 fecha especificado, de lo contrario retorna False.
117 117
118 118 Excepciones:
119 119 Si el archivo no existe o no puede ser abierto
120 120 Si la cabecera no puede ser leida.
121 121
122 122 """
123 123
124 124
125 125 try:
126 126 fp = open(filename,'rb')
127 127 except IOError:
128 128 print "The file %s can't be opened" %(filename)
129 129 return None
130 130
131 131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 132 systemHeaderObj = SystemHeader()
133 133 radarControllerHeaderObj = RadarControllerHeader()
134 134 processingHeaderObj = ProcessingHeader()
135 135
136 136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 137
138 138 sts = firstBasicHeaderObj.read(fp)
139 139
140 140 if not(sts):
141 141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 142 return None
143 143
144 144 if not systemHeaderObj.read(fp):
145 145 return None
146 146
147 147 if not radarControllerHeaderObj.read(fp):
148 148 return None
149 149
150 150 if not processingHeaderObj.read(fp):
151 151 return None
152 152
153 153 filesize = os.path.getsize(filename)
154 154
155 155 offset = processingHeaderObj.blockSize + 24 #header size
156 156
157 157 if filesize <= offset:
158 158 print "[Reading] %s: This file has not enough data" %filename
159 159 return None
160 160
161 161 fp.seek(-offset, 2)
162 162
163 163 sts = lastBasicHeaderObj.read(fp)
164 164
165 165 fp.close()
166 166
167 167 thisDatetime = lastBasicHeaderObj.datatime
168 168 thisTime_last_block = thisDatetime.time()
169 169
170 170 thisDatetime = firstBasicHeaderObj.datatime
171 171 thisDate = thisDatetime.date()
172 172 thisTime_first_block = thisDatetime.time()
173 173
174 174 #General case
175 175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 176 #-----------o----------------------------o-----------
177 177 # startTime endTime
178 178
179 179 if endTime >= startTime:
180 180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 181 return None
182 182
183 183 return thisDatetime
184 184
185 185 #If endTime < startTime then endTime belongs to the next day
186 186
187 187
188 188 #<<<<<<<<<<<o o>>>>>>>>>>>
189 189 #-----------o----------------------------o-----------
190 190 # endTime startTime
191 191
192 192 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 193 return None
194 194
195 195 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 196 return None
197 197
198 198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 199 return None
200 200
201 201 return thisDatetime
202 202
203 203 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 204 """
205 205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 206
207 207 Inputs:
208 208 folder : nombre completo del directorio.
209 209 Su formato deberia ser "/path_root/?YYYYDDD"
210 210
211 211 siendo:
212 212 YYYY : Anio (ejemplo 2015)
213 213 DDD : Dia del anio (ejemplo 305)
214 214
215 215 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 216
217 217 endDate : fecha final del rango seleccionado en formato datetime.date
218 218
219 219 Return:
220 220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 221 fecha especificado, de lo contrario retorna False.
222 222 Excepciones:
223 223 Si el directorio no tiene el formato adecuado
224 224 """
225 225
226 226 basename = os.path.basename(folder)
227 227
228 228 if not isRadarFolder(basename):
229 229 print "The folder %s has not the rigth format" %folder
230 230 return 0
231 231
232 232 if startDate and endDate:
233 233 thisDate = getDateFromRadarFolder(basename)
234 234
235 235 if thisDate < startDate:
236 236 return 0
237 237
238 238 if thisDate > endDate:
239 239 return 0
240 240
241 241 return 1
242 242
243 243 def isFileInDateRange(filename, startDate=None, endDate=None):
244 244 """
245 245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 246
247 247 Inputs:
248 248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 249
250 250 Su formato deberia ser "?YYYYDDDsss"
251 251
252 252 siendo:
253 253 YYYY : Anio (ejemplo 2015)
254 254 DDD : Dia del anio (ejemplo 305)
255 255 sss : set
256 256
257 257 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 258
259 259 endDate : fecha final del rango seleccionado en formato datetime.date
260 260
261 261 Return:
262 262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 263 fecha especificado, de lo contrario retorna False.
264 264 Excepciones:
265 265 Si el archivo no tiene el formato adecuado
266 266 """
267 267
268 268 basename = os.path.basename(filename)
269 269
270 270 if not isRadarFile(basename):
271 271 print "The filename %s has not the rigth format" %filename
272 272 return 0
273 273
274 274 if startDate and endDate:
275 275 thisDate = getDateFromRadarFile(basename)
276 276
277 277 if thisDate < startDate:
278 278 return 0
279 279
280 280 if thisDate > endDate:
281 281 return 0
282 282
283 283 return 1
284 284
285 285 def getFileFromSet(path, ext, set):
286 286 validFilelist = []
287 287 fileList = os.listdir(path)
288 288
289 289 # 0 1234 567 89A BCDE
290 290 # H YYYY DDD SSS .ext
291 291
292 292 for thisFile in fileList:
293 293 try:
294 294 year = int(thisFile[1:5])
295 295 doy = int(thisFile[5:8])
296 296 except:
297 297 continue
298 298
299 299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 300 continue
301 301
302 302 validFilelist.append(thisFile)
303 303
304 304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 305
306 306 if len(myfile)!= 0:
307 307 return myfile[0]
308 308 else:
309 309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 310 print 'the filename %s does not exist'%filename
311 311 print '...going to the last file: '
312 312
313 313 if validFilelist:
314 314 validFilelist = sorted( validFilelist, key=str.lower )
315 315 return validFilelist[-1]
316 316
317 317 return None
318 318
319 319 def getlastFileFromPath(path, ext):
320 320 """
321 321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 323
324 324 Input:
325 325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 326 ext : extension de los files contenidos en una carpeta
327 327
328 328 Return:
329 329 El ultimo file de una determinada carpeta, no se considera el path.
330 330 """
331 331 validFilelist = []
332 332 fileList = os.listdir(path)
333 333
334 334 # 0 1234 567 89A BCDE
335 335 # H YYYY DDD SSS .ext
336 336
337 337 for thisFile in fileList:
338 338
339 339 year = thisFile[1:5]
340 340 if not isNumber(year):
341 341 continue
342 342
343 343 doy = thisFile[5:8]
344 344 if not isNumber(doy):
345 345 continue
346 346
347 347 year = int(year)
348 348 doy = int(doy)
349 349
350 350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 351 continue
352 352
353 353 validFilelist.append(thisFile)
354 354
355 355 if validFilelist:
356 356 validFilelist = sorted( validFilelist, key=str.lower )
357 357 return validFilelist[-1]
358 358
359 359 return None
360 360
361 361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 362 """
363 363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 365 el path exacto de un determinado file.
366 366
367 367 Example :
368 368 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 369
370 370 Entonces la funcion prueba con las siguientes combinaciones
371 371 .../.../y2009307367.ext
372 372 .../.../Y2009307367.ext
373 373 .../.../x2009307/y2009307367.ext
374 374 .../.../x2009307/Y2009307367.ext
375 375 .../.../X2009307/y2009307367.ext
376 376 .../.../X2009307/Y2009307367.ext
377 377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 378
379 379 Return:
380 380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 382 para el filename
383 383 """
384 384 fullfilename = None
385 385 find_flag = False
386 386 filename = None
387 387
388 388 prefixDirList = [None,'d','D']
389 389 if ext.lower() == ".r": #voltage
390 390 prefixFileList = ['d','D']
391 391 elif ext.lower() == ".pdata": #spectra
392 392 prefixFileList = ['p','P']
393 393 else:
394 394 return None, filename
395 395
396 396 #barrido por las combinaciones posibles
397 397 for prefixDir in prefixDirList:
398 398 thispath = path
399 399 if prefixDir != None:
400 400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 401 if foldercounter == 0:
402 402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 403 else:
404 404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 408
409 409 if os.path.exists( fullfilename ): #verifico que exista
410 410 find_flag = True
411 411 break
412 412 if find_flag:
413 413 break
414 414
415 415 if not(find_flag):
416 416 return None, filename
417 417
418 418 return fullfilename, filename
419 419
420 420 def isRadarFolder(folder):
421 421 try:
422 422 year = int(folder[1:5])
423 423 doy = int(folder[5:8])
424 424 except:
425 425 return 0
426 426
427 427 return 1
428 428
429 429 def isRadarFile(file):
430 430 try:
431 431 year = int(file[1:5])
432 432 doy = int(file[5:8])
433 433 set = int(file[8:11])
434 434 except:
435 435 return 0
436 436
437 437 return 1
438 438
439 439 def getDateFromRadarFile(file):
440 440 try:
441 441 year = int(file[1:5])
442 442 doy = int(file[5:8])
443 443 set = int(file[8:11])
444 444 except:
445 445 return None
446 446
447 447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 448 return thisDate
449 449
450 450 def getDateFromRadarFolder(folder):
451 451 try:
452 452 year = int(folder[1:5])
453 453 doy = int(folder[5:8])
454 454 except:
455 455 return None
456 456
457 457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 458 return thisDate
459 459
460 460 class JRODataIO:
461 461
462 462 c = 3E8
463 463
464 464 isConfig = False
465 465
466 466 basicHeaderObj = None
467 467
468 468 systemHeaderObj = None
469 469
470 470 radarControllerHeaderObj = None
471 471
472 472 processingHeaderObj = None
473 473
474 474 dtype = None
475 475
476 476 pathList = []
477 477
478 478 filenameList = []
479 479
480 480 filename = None
481 481
482 482 ext = None
483 483
484 484 flagIsNewFile = 1
485 485
486 486 flagDiscontinuousBlock = 0
487 487
488 488 flagIsNewBlock = 0
489 489
490 490 fp = None
491 491
492 492 firstHeaderSize = 0
493 493
494 494 basicHeaderSize = 24
495 495
496 496 versionFile = 1103
497 497
498 498 fileSize = None
499 499
500 500 # ippSeconds = None
501 501
502 502 fileSizeByHeader = None
503 503
504 504 fileIndex = None
505 505
506 506 profileIndex = None
507 507
508 508 blockIndex = None
509 509
510 510 nTotalBlocks = None
511 511
512 512 maxTimeStep = 30
513 513
514 514 lastUTTime = None
515 515
516 516 datablock = None
517 517
518 518 dataOut = None
519 519
520 520 blocksize = None
521 521
522 522 getByBlock = False
523 523
524 524 def __init__(self):
525 525
526 526 raise NotImplementedError
527 527
528 528 def run(self):
529 529
530 530 raise NotImplementedError
531 531
532 532 def getDtypeWidth(self):
533 533
534 534 dtype_index = get_dtype_index(self.dtype)
535 535 dtype_width = get_dtype_width(dtype_index)
536 536
537 537 return dtype_width
538 538
539 539 class JRODataReader(JRODataIO):
540 540
541
541 firstTime = True
542 542 online = 0
543 543
544 544 realtime = 0
545 545
546 546 nReadBlocks = 0
547 547
548 548 delay = 10 #number of seconds waiting a new file
549 549
550 550 nTries = 3 #quantity tries
551 551
552 552 nFiles = 3 #number of files for searching
553 553
554 554 path = None
555 555
556 556 foldercounter = 0
557 557
558 558 flagNoMoreFiles = 0
559 559
560 560 datetimeList = []
561 561
562 562 __isFirstTimeOnline = 1
563 563
564 564 __printInfo = True
565 565
566 566 profileIndex = None
567 567
568 568 nTxs = 1
569 569
570 570 txIndex = None
571 571
572 572 #Added--------------------
573 573
574 574 selBlocksize = None
575 575
576 576 selBlocktime = None
577 577
578
578 onlineWithDate = False
579 579 def __init__(self):
580 580
581 581 """
582 582 This class is used to find data files
583 583
584 584 Example:
585 585 reader = JRODataReader()
586 586 fileList = reader.findDataFiles()
587 587
588 588 """
589 589 pass
590 590
591 591
592 592 def createObjByDefault(self):
593 593 """
594 594
595 595 """
596 596 raise NotImplementedError
597 597
598 598 def getBlockDimension(self):
599 599
600 600 raise NotImplementedError
601 601
602 602 def __searchFilesOffLine(self,
603 603 path,
604 604 startDate=None,
605 605 endDate=None,
606 606 startTime=datetime.time(0,0,0),
607 607 endTime=datetime.time(23,59,59),
608 608 set=None,
609 609 expLabel='',
610 610 ext='.r',
611 611 walk=True):
612
613 612 self.filenameList = []
614 613 self.datetimeList = []
615 614
616 615 pathList = []
617 616
618 617 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
619 618
620 619 if dateList == []:
621 620 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
622 621 return None, None
623 622
624 623 if len(dateList) > 1:
625 624 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
626 625 else:
627 626 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
628 627
629 628 filenameList = []
630 629 datetimeList = []
631 630
632 631 for thisPath in pathList:
633 632 # thisPath = pathList[pathDict[file]]
634 633
635 634 fileList = glob.glob1(thisPath, "*%s" %ext)
636 635 fileList.sort()
637 636
638 637 for file in fileList:
639 638
640 639 filename = os.path.join(thisPath,file)
641 640
642 641 if not isFileInDateRange(filename, startDate, endDate):
643 642 continue
644 643
645 644 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
646 645
647 646 if not(thisDatetime):
648 647 continue
649 648
650 649 filenameList.append(filename)
651 650 datetimeList.append(thisDatetime)
652 651
653 652 if not(filenameList):
654 653 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
655 654 return None, None
656 655
657 656 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
658 657 print
659 658
660 659 for i in range(len(filenameList)):
661 660 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
662 661
663 662 self.filenameList = filenameList
664 663 self.datetimeList = datetimeList
665 664
665
666 666 return pathList, filenameList
667 667
668 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None, startDate=None, endDate=None):
668 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
669 669
670 670 """
671 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
672 devuelve el archivo encontrado ademas de otros datos.
673
674 Input:
675 path : carpeta donde estan contenidos los files que contiene data
676
677 expLabel : Nombre del subexperimento (subfolder)
678
679 ext : extension de los files
680
681 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
682
683 Return:
684 directory : eL directorio donde esta el file encontrado
685 filename : el ultimo file de una determinada carpeta
686 year : el anho
687 doy : el numero de dia del anho
688 set : el set del archivo
671 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
672 devuelve el archivo encontrado ademas de otros datos.
689 673
674 Input:
675 path : carpeta donde estan contenidos los files que contiene data
676
677 expLabel : Nombre del subexperimento (subfolder)
678
679 ext : extension de los files
680
681 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
682
683 Return:
684 directory : eL directorio donde esta el file encontrado
685 filename : el ultimo file de una determinada carpeta
686 year : el anho
687 doy : el numero de dia del anho
688 set : el set del archivo
689
690 690
691 691 """
692 pathList = None
693 filenameList = None
692 694 if not os.path.isdir(path):
693 695 return None, None, None, None, None, None
694 696
695 697 dirList = []
696 698
697 699 if not walk:
698 700 fullpath = path
699 701 foldercounter = 0
700 702 else:
701 703 # Filtra solo los directorios
702 704 for thisPath in os.listdir(path):
703 705 if not os.path.isdir(os.path.join(path,thisPath)):
704 706 continue
705 707 if not isRadarFolder(thisPath):
706 708 continue
707 709
708 710 dirList.append(thisPath)
709 711
710 712 if not(dirList):
711 713 return None, None, None, None, None, None
712 714
713 715 dirList = sorted( dirList, key=str.lower )
714 716
715 717 doypath = dirList[-1]
716 718 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
717 719 fullpath = os.path.join(path, doypath, expLabel)
718 720
719 721
720 722 print "[Reading] %s folder was found: " %(fullpath )
721 723
722 724 if set == None:
723 725 filename = getlastFileFromPath(fullpath, ext)
724 726 else:
725 727 filename = getFileFromSet(fullpath, ext, set)
726 728
727 729 if not(filename):
728 730 return None, None, None, None, None, None
729 731
730 732 print "[Reading] %s file was found" %(filename)
731 733
732 734 if not(self.__verifyFile(os.path.join(fullpath, filename))):
733 735 return None, None, None, None, None, None
734 736
735 737 year = int( filename[1:5] )
736 738 doy = int( filename[5:8] )
737 739 set = int( filename[8:11] )
738
740
739 741 return fullpath, foldercounter, filename, year, doy, set
740 742
741 743 def __setNextFileOffline(self):
742 744
743 745 idFile = self.fileIndex
744 746
745 747 while (True):
746 748 idFile += 1
747 749 if not(idFile < len(self.filenameList)):
748 750 self.flagNoMoreFiles = 1
749 # print "[Reading] No more Files"
751 # print "[Reading] No more Files"
750 752 return 0
751 753
752 754 filename = self.filenameList[idFile]
753 755
754 756 if not(self.__verifyFile(filename)):
755 757 continue
756 758
757 759 fileSize = os.path.getsize(filename)
758 760 fp = open(filename,'rb')
759 761 break
760 762
761 763 self.flagIsNewFile = 1
762 764 self.fileIndex = idFile
763 765 self.filename = filename
764 766 self.fileSize = fileSize
765 767 self.fp = fp
766 768
767 # print "[Reading] Setting the file: %s"%self.filename
769 #print "[Reading] Setting the file: %s"%self.filename
768 770
769 771 return 1
770 772
771 773 def __setNextFileOnline(self):
772 774 """
773 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
774 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
775 siguientes.
775 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
776 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
777 siguientes.
778
779 Affected:
780 self.flagIsNewFile
781 self.filename
782 self.fileSize
783 self.fp
784 self.set
785 self.flagNoMoreFiles
786
787 Return:
788 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
789 1 : si el file fue abierto con exito y esta listo a ser leido
776 790
777 Affected:
778 self.flagIsNewFile
779 self.filename
780 self.fileSize
781 self.fp
782 self.set
783 self.flagNoMoreFiles
784
785 Return:
786 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
787 1 : si el file fue abierto con exito y esta listo a ser leido
791 Excepciones:
792 Si un determinado file no puede ser abierto
793 """
788 794
789 Excepciones:
790 Si un determinado file no puede ser abierto
791 """
792 795 nFiles = 0
793 796 fileOk_flag = False
794 797 firstTime_flag = True
795 798
796 799 self.set += 1
797 800
798 801 if self.set > 999:
799 802 self.set = 0
800 803 self.foldercounter += 1
801 804
802 805 #busca el 1er file disponible
803 806 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
804 807 if fullfilename:
805 808 if self.__verifyFile(fullfilename, False):
806 809 fileOk_flag = True
807 810
808 811 #si no encuentra un file entonces espera y vuelve a buscar
809 812 if not(fileOk_flag):
810 813 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
811 814
812 815 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
813 816 tries = self.nTries
814 817 else:
815 818 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
816 819
817 820 for nTries in range( tries ):
818 821 if firstTime_flag:
819 822 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
820 823 sleep( self.delay )
821 824 else:
822 825 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
823 826
824 827 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 828 if fullfilename:
826 829 if self.__verifyFile(fullfilename):
827 830 fileOk_flag = True
828 831 break
829 832
830 833 if fileOk_flag:
831 834 break
832 835
833 836 firstTime_flag = False
834 837
835 838 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
836 839 self.set += 1
837 840
838 841 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
839 842 self.set = 0
840 843 self.doy += 1
841 844 self.foldercounter = 0
842 845
843 846 if fileOk_flag:
844 847 self.fileSize = os.path.getsize( fullfilename )
845 848 self.filename = fullfilename
846 849 self.flagIsNewFile = 1
847 850 if self.fp != None: self.fp.close()
848 851 self.fp = open(fullfilename, 'rb')
849 852 self.flagNoMoreFiles = 0
850 853 # print '[Reading] Setting the file: %s' % fullfilename
851 854 else:
852 855 self.fileSize = 0
853 856 self.filename = None
854 857 self.flagIsNewFile = 0
855 858 self.fp = None
856 859 self.flagNoMoreFiles = 1
857 860 # print '[Reading] No more files to read'
858 861
859 862 return fileOk_flag
860 863
861 864 def setNextFile(self):
862 865 if self.fp != None:
863 866 self.fp.close()
864
865 867 if self.online:
866 868 newFile = self.__setNextFileOnline()
867 869 else:
868 870 newFile = self.__setNextFileOffline()
869
870 871 if not(newFile):
872 if self.onlineWithDate is True:
873 self.onlineWithDate=False
874 self.online = True
875 self.firstTime = False
876 self.setup(
877 path=self.path,
878 startDate=self.startDate,
879 endDate=self.endDate,
880 startTime=self.startTime ,
881 endTime=self.endTime,
882 set=self.set,
883 expLabel=self.expLabel,
884 ext=self.ext,
885 online=self.online,
886 delay=self.delay,
887 walk=self.walk,
888 getblock=self.getblock,
889 nTxs=self.nTxs,
890 realtime=self.realtime,
891 blocksize=self.blocksize,
892 blocktime=self.blocktime
893 )
894 return 1
871 895 print '[Reading] No more files to read'
872 896 return 0
873 897
874 898 print '[Reading] Setting the file: %s' % self.filename
875 899
876 900 self.__readFirstHeader()
877 901 self.nReadBlocks = 0
878 902 return 1
879 903
880 904 def __waitNewBlock(self):
881 905 """
882 906 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
883 907
884 908 Si el modo de lectura es OffLine siempre retorn 0
885 909 """
886 910 if not self.online:
887 911 return 0
888 912
889 913 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
890 914 return 0
891 915
892 916 currentPointer = self.fp.tell()
893 917
894 918 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
895 919
896 920 for nTries in range( self.nTries ):
897 921
898 922 self.fp.close()
899 923 self.fp = open( self.filename, 'rb' )
900 924 self.fp.seek( currentPointer )
901 925
902 926 self.fileSize = os.path.getsize( self.filename )
903 927 currentSize = self.fileSize - currentPointer
904 928
905 929 if ( currentSize >= neededSize ):
906 930 self.basicHeaderObj.read(self.fp)
907 931 return 1
908 932
909 933 if self.fileSize == self.fileSizeByHeader:
910 934 # self.flagEoF = True
911 935 return 0
912 936
913 937 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
914 938 sleep( self.delay )
915 939
916 940
917 941 return 0
918 942
919 943 def waitDataBlock(self,pointer_location):
920 944
921 945 currentPointer = pointer_location
922 946
923 947 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
924 948
925 949 for nTries in range( self.nTries ):
926 950 self.fp.close()
927 951 self.fp = open( self.filename, 'rb' )
928 952 self.fp.seek( currentPointer )
929 953
930 954 self.fileSize = os.path.getsize( self.filename )
931 955 currentSize = self.fileSize - currentPointer
932 956
933 957 if ( currentSize >= neededSize ):
934 958 return 1
935 959
936 960 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
937 961 sleep( self.delay )
938 962
939 963 return 0
940 964
941 965 def __jumpToLastBlock(self):
942 966
943 967 if not(self.__isFirstTimeOnline):
944 968 return
945 969
946 970 csize = self.fileSize - self.fp.tell()
947 971 blocksize = self.processingHeaderObj.blockSize
948 972
949 973 #salta el primer bloque de datos
950 974 if csize > self.processingHeaderObj.blockSize:
951 975 self.fp.seek(self.fp.tell() + blocksize)
952 976 else:
953 977 return
954 978
955 979 csize = self.fileSize - self.fp.tell()
956 980 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
957 981 while True:
958 982
959 983 if self.fp.tell()<self.fileSize:
960 984 self.fp.seek(self.fp.tell() + neededsize)
961 985 else:
962 986 self.fp.seek(self.fp.tell() - neededsize)
963 987 break
964 988
965 989 # csize = self.fileSize - self.fp.tell()
966 990 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
967 991 # factor = int(csize/neededsize)
968 992 # if factor > 0:
969 993 # self.fp.seek(self.fp.tell() + factor*neededsize)
970 994
971 995 self.flagIsNewFile = 0
972 996 self.__isFirstTimeOnline = 0
973 997
974 998 def __setNewBlock(self):
975 999
976 1000 if self.fp == None:
977 1001 return 0
978 1002
979 1003 # if self.online:
980 1004 # self.__jumpToLastBlock()
981 1005
982 1006 if self.flagIsNewFile:
983 1007 self.lastUTTime = self.basicHeaderObj.utc
984 1008 return 1
985 1009
986 1010 if self.realtime:
987 1011 self.flagDiscontinuousBlock = 1
988 1012 if not(self.setNextFile()):
989 1013 return 0
990 1014 else:
991 1015 return 1
992 1016
993 1017 currentSize = self.fileSize - self.fp.tell()
994 1018 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 1019
996 1020 if (currentSize >= neededSize):
997 1021 self.basicHeaderObj.read(self.fp)
998 1022 self.lastUTTime = self.basicHeaderObj.utc
999 1023 return 1
1000 1024
1001 1025 if self.__waitNewBlock():
1002 1026 self.lastUTTime = self.basicHeaderObj.utc
1003 1027 return 1
1004 1028
1005 1029 if not(self.setNextFile()):
1006 1030 return 0
1007 1031
1008 1032 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1009 1033 self.lastUTTime = self.basicHeaderObj.utc
1010 1034
1011 1035 self.flagDiscontinuousBlock = 0
1012 1036
1013 1037 if deltaTime > self.maxTimeStep:
1014 1038 self.flagDiscontinuousBlock = 1
1015 1039
1016 1040 return 1
1017 1041
1018 1042 def readNextBlock(self):
1019 1043
1020 1044 #Skip block out of startTime and endTime
1021 1045 while True:
1022 1046 if not(self.__setNewBlock()):
1023 1047 return 0
1024
1025 1048 if not(self.readBlock()):
1026 1049 return 0
1027
1028 1050 self.getBasicHeader()
1029
1030 1051 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1031 1052
1032 1053 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1033 1054 self.processingHeaderObj.dataBlocksPerFile,
1034 1055 self.dataOut.datatime.ctime())
1035 1056 continue
1036 1057
1037 1058 break
1038
1039 1059 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1040 1060 self.processingHeaderObj.dataBlocksPerFile,
1041 1061 self.dataOut.datatime.ctime())
1042 1062 return 1
1043 1063
1044 1064 def __readFirstHeader(self):
1045 1065
1046 1066 self.basicHeaderObj.read(self.fp)
1047 1067 self.systemHeaderObj.read(self.fp)
1048 1068 self.radarControllerHeaderObj.read(self.fp)
1049 1069 self.processingHeaderObj.read(self.fp)
1050 1070
1051 1071 self.firstHeaderSize = self.basicHeaderObj.size
1052 1072
1053 1073 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1054 1074 if datatype == 0:
1055 1075 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1056 1076 elif datatype == 1:
1057 1077 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1058 1078 elif datatype == 2:
1059 1079 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1060 1080 elif datatype == 3:
1061 1081 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1062 1082 elif datatype == 4:
1063 1083 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1064 1084 elif datatype == 5:
1065 1085 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1066 1086 else:
1067 1087 raise ValueError, 'Data type was not defined'
1068 1088
1069 1089 self.dtype = datatype_str
1070 1090 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1071 1091 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1072 1092 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1073 1093 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1074 1094 self.getBlockDimension()
1075 1095
1076 1096 def __verifyFile(self, filename, msgFlag=True):
1077 1097
1078 1098 msg = None
1079 1099
1080 1100 try:
1081 1101 fp = open(filename, 'rb')
1082 1102 except IOError:
1083 1103
1084 1104 if msgFlag:
1085 1105 print "[Reading] File %s can't be opened" % (filename)
1086 1106
1087 1107 return False
1088 1108
1089 1109 currentPosition = fp.tell()
1090 1110 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1091 1111
1092 1112 if neededSize == 0:
1093 1113 basicHeaderObj = BasicHeader(LOCALTIME)
1094 1114 systemHeaderObj = SystemHeader()
1095 1115 radarControllerHeaderObj = RadarControllerHeader()
1096 1116 processingHeaderObj = ProcessingHeader()
1097 1117
1098 1118 if not( basicHeaderObj.read(fp) ):
1099 1119 fp.close()
1100 1120 return False
1101 1121
1102 1122 if not( systemHeaderObj.read(fp) ):
1103 1123 fp.close()
1104 1124 return False
1105 1125
1106 1126 if not( radarControllerHeaderObj.read(fp) ):
1107 1127 fp.close()
1108 1128 return False
1109 1129
1110 1130 if not( processingHeaderObj.read(fp) ):
1111 1131 fp.close()
1112 1132 return False
1113 1133
1114 1134 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1115 1135 else:
1116 1136 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1117 1137
1118 1138 fp.close()
1119 1139
1120 1140 fileSize = os.path.getsize(filename)
1121 1141 currentSize = fileSize - currentPosition
1122 1142
1123 1143 if currentSize < neededSize:
1124 1144 if msgFlag and (msg != None):
1125 1145 print msg
1126 1146 return False
1127 1147
1128 1148 return True
1129 1149
1130 1150 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1131 1151
1132 1152 path_empty = True
1133 1153
1134 1154 dateList = []
1135 1155 pathList = []
1136 1156
1137 1157 multi_path = path.split(',')
1138 1158
1139 1159 if not walk:
1140 1160
1141 1161 for single_path in multi_path:
1142 1162
1143 1163 if not os.path.isdir(single_path):
1144 1164 continue
1145 1165
1146 1166 fileList = glob.glob1(single_path, "*"+ext)
1147 1167
1148 1168 if not fileList:
1149 1169 continue
1150 1170
1151 1171 path_empty = False
1152 1172
1153 1173 fileList.sort()
1154 1174
1155 1175 for thisFile in fileList:
1156 1176
1157 1177 if not os.path.isfile(os.path.join(single_path, thisFile)):
1158 1178 continue
1159 1179
1160 1180 if not isRadarFile(thisFile):
1161 1181 continue
1162 1182
1163 1183 if not isFileInDateRange(thisFile, startDate, endDate):
1164 1184 continue
1165 1185
1166 1186 thisDate = getDateFromRadarFile(thisFile)
1167 1187
1168 1188 if thisDate in dateList:
1169 1189 continue
1170 1190
1171 1191 dateList.append(thisDate)
1172 1192 pathList.append(single_path)
1173 1193
1174 1194 else:
1175 1195 for single_path in multi_path:
1176 1196
1177 1197 if not os.path.isdir(single_path):
1178 1198 continue
1179 1199
1180 1200 dirList = []
1181 1201
1182 1202 for thisPath in os.listdir(single_path):
1183 1203
1184 1204 if not os.path.isdir(os.path.join(single_path,thisPath)):
1185 1205 continue
1186 1206
1187 1207 if not isRadarFolder(thisPath):
1188 1208 continue
1189 1209
1190 1210 if not isFolderInDateRange(thisPath, startDate, endDate):
1191 1211 continue
1192 1212
1193 1213 dirList.append(thisPath)
1194 1214
1195 1215 if not dirList:
1196 1216 continue
1197 1217
1198 1218 dirList.sort()
1199 1219
1200 1220 for thisDir in dirList:
1201 1221
1202 1222 datapath = os.path.join(single_path, thisDir, expLabel)
1203 1223 fileList = glob.glob1(datapath, "*"+ext)
1204 1224
1205 1225 if not fileList:
1206 1226 continue
1207 1227
1208 1228 path_empty = False
1209 1229
1210 1230 thisDate = getDateFromRadarFolder(thisDir)
1211 1231
1212 1232 pathList.append(datapath)
1213 1233 dateList.append(thisDate)
1214 1234
1215 1235 dateList.sort()
1216 1236
1217 1237 if walk:
1218 1238 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1219 1239 else:
1220 1240 pattern_path = multi_path[0]
1221 1241
1222 1242 if path_empty:
1223 1243 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1224 1244 else:
1225 1245 if not dateList:
1226 1246 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1227 1247
1228 1248 if include_path:
1229 1249 return dateList, pathList
1230 1250
1231 1251 return dateList
1232 1252
1233 1253 def setup(self,
1234 1254 path=None,
1235 1255 startDate=None,
1236 1256 endDate=None,
1237 1257 startTime=datetime.time(0,0,0),
1238 1258 endTime=datetime.time(23,59,59),
1239 1259 set=None,
1240 1260 expLabel = "",
1241 1261 ext = None,
1242 1262 online = False,
1243 1263 delay = 60,
1244 1264 walk = True,
1245 1265 getblock = False,
1246 1266 nTxs = 1,
1247 1267 realtime=False,
1248 1268 blocksize=None,
1249 1269 blocktime=None):
1250 1270
1251 1271 if path == None:
1252 1272 raise ValueError, "[Reading] The path is not valid"
1273
1253 1274
1254 1275 if ext == None:
1255 1276 ext = self.ext
1277
1278 self.path = path
1279 self.startDate = startDate
1280 self.endDate = endDate
1281 self.startTime = startTime
1282 self.endTime = endTime
1283 self.set = set
1284 self.expLabel = expLabel
1285 self.ext = ext
1286 self.online = online
1287 self.delay = delay
1288 self.walk = walk
1289 self.getblock = getblock
1290 self.nTxs = nTxs
1291 self.realtime = realtime
1292 self.blocksize = blocksize
1293 self.blocktime = blocktime
1294
1295
1296 if self.firstTime is True:
1297 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1298 startTime=startTime, endTime=endTime,
1299 set=set, expLabel=expLabel, ext=ext,
1300 walk=walk)
1301 filenameList = filenameList[:-1]
1302
1303 if pathList is not None and filenameList is not None and online:
1304 self.onlineWithDate = True
1305 online = False
1306 self.fileIndex = -1
1307 self.pathList = pathList
1308 self.filenameList = filenameList
1309 file_name = os.path.basename(filenameList[-1])
1310 basename, ext = os.path.splitext(file_name)
1311 last_set = int(basename[-3:])
1256 1312
1257 1313 if online:
1258 1314 print "[Reading] Searching files in online mode..."
1259 1315
1260 for nTries in range( self.nTries ):
1316 for nTries in range(self.nTries):
1261 1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1262 1318 expLabel=expLabel,
1263 1319 ext=ext,
1264 1320 walk=walk,
1265 1321 startDate=startDate,
1266 endDate=endDate,
1267 startTime=startTime, endTime=endTime,
1322 startTime=startTime,
1268 1323 set=set)
1269
1324
1270 1325 if fullpath:
1271 1326 break
1272
1273 1327 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1274 1328 sleep( self.delay )
1275 1329
1276 1330 if not(fullpath):
1277 1331 print "[Reading] There 'isn't any valid file in %s" % path
1278 1332 return
1279 1333
1280 1334 self.year = year
1281 1335 self.doy = doy
1282 1336 self.set = set - 1
1283 1337 self.path = path
1284 1338 self.foldercounter = foldercounter
1285 1339 last_set = None
1286
1340
1287 1341 else:
1288 1342 print "[Reading] Searching files in offline mode ..."
1289 1343 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1290 1344 startTime=startTime, endTime=endTime,
1291 1345 set=set, expLabel=expLabel, ext=ext,
1292 1346 walk=walk)
1293 1347
1294 1348 if not(pathList):
1295 1349 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1296 1350 # datetime.datetime.combine(startDate,startTime).ctime(),
1297 1351 # datetime.datetime.combine(endDate,endTime).ctime())
1298 1352
1299 1353 # sys.exit(-1)
1300 1354
1301 1355 self.fileIndex = -1
1302 1356 self.pathList = []
1303 1357 self.filenameList = []
1304 1358 return
1305 1359
1306 1360 self.fileIndex = -1
1307 1361 self.pathList = pathList
1308 1362 self.filenameList = filenameList
1309 1363 file_name = os.path.basename(filenameList[-1])
1310 1364 basename, ext = os.path.splitext(file_name)
1311 1365 last_set = int(basename[-3:])
1312
1366
1367
1313 1368 self.online = online
1314 1369 self.realtime = realtime
1315 1370 self.delay = delay
1316 1371 ext = ext.lower()
1317 1372 self.ext = ext
1318 1373 self.getByBlock = getblock
1319 1374 self.nTxs = nTxs
1320 1375 self.startTime = startTime
1321 1376 self.endTime = endTime
1377
1322 1378
1323 1379 #Added-----------------
1324 1380 self.selBlocksize = blocksize
1325 1381 self.selBlocktime = blocktime
1326 1382
1327 1383
1328 1384 if not(self.setNextFile()):
1329 1385 if (startDate!=None) and (endDate!=None):
1330 1386 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1331 1387 elif startDate != None:
1332 1388 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1333 1389 else:
1334 1390 print "[Reading] No files"
1335 1391
1336 1392 self.fileIndex = -1
1337 1393 self.pathList = []
1338 1394 self.filenameList = []
1339 1395 return
1340 1396
1341 1397 # self.getBasicHeader()
1342
1398
1343 1399 if last_set != None:
1344 1400 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1345 1401 return
1346 1402
1347 1403 def getBasicHeader(self):
1348 1404
1349 1405 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1350 1406
1351 1407 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1352 1408
1353 1409 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1354 1410
1355 1411 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1356 1412
1357 1413 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1358 1414
1359 1415 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1360 1416
1361 1417 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1362 1418
1363 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1419 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1364 1420
1365 1421
1366 1422 def getFirstHeader(self):
1367 1423
1368 1424 raise NotImplementedError
1369 1425
1370 1426 def getData(self):
1371 1427
1372 1428 raise NotImplementedError
1373 1429
1374 1430 def hasNotDataInBuffer(self):
1375 1431
1376 1432 raise NotImplementedError
1377 1433
1378 1434 def readBlock(self):
1379 1435
1380 1436 raise NotImplementedError
1381 1437
1382 1438 def isEndProcess(self):
1383 1439
1384 1440 return self.flagNoMoreFiles
1385 1441
1386 1442 def printReadBlocks(self):
1387 1443
1388 1444 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1389 1445
1390 1446 def printTotalBlocks(self):
1391 1447
1392 1448 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1393 1449
1394 1450 def printNumberOfBlock(self):
1395 1451
1396 1452 if self.flagIsNewBlock:
1397 1453 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1398 1454 self.processingHeaderObj.dataBlocksPerFile,
1399 1455 self.dataOut.datatime.ctime())
1400 1456
1401 1457 def printInfo(self):
1402 1458
1403 1459 if self.__printInfo == False:
1404 1460 return
1405 1461
1406 1462 self.basicHeaderObj.printInfo()
1407 1463 self.systemHeaderObj.printInfo()
1408 1464 self.radarControllerHeaderObj.printInfo()
1409 1465 self.processingHeaderObj.printInfo()
1410 1466
1411 1467 self.__printInfo = False
1412 1468
1413 1469
1414 1470 def run(self, **kwargs):
1415 1471
1416 1472 if not(self.isConfig):
1417 1473
1418 # self.dataOut = dataOut
1474 # self.dataOut = dataOut
1419 1475 self.setup(**kwargs)
1420 1476 self.isConfig = True
1421 1477
1422 1478 self.getData()
1423 1479
1424 1480 class JRODataWriter(JRODataIO):
1425 1481
1426 1482 """
1427 1483 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1428 1484 de los datos siempre se realiza por bloques.
1429 1485 """
1430 1486
1431 1487 blockIndex = 0
1432 1488
1433 1489 path = None
1434 1490
1435 1491 setFile = None
1436 1492
1437 1493 profilesPerBlock = None
1438 1494
1439 1495 blocksPerFile = None
1440 1496
1441 1497 nWriteBlocks = 0
1442 1498
1443 1499 fileDate = None
1444 1500
1445 1501 def __init__(self, dataOut=None):
1446 1502 raise NotImplementedError
1447 1503
1448 1504
1449 1505 def hasAllDataInBuffer(self):
1450 1506 raise NotImplementedError
1451 1507
1452 1508
1453 1509 def setBlockDimension(self):
1454 1510 raise NotImplementedError
1455 1511
1456 1512
1457 1513 def writeBlock(self):
1458 1514 raise NotImplementedError
1459 1515
1460 1516
1461 1517 def putData(self):
1462 1518 raise NotImplementedError
1463 1519
1464 1520
1465 1521 def getProcessFlags(self):
1466 1522
1467 1523 processFlags = 0
1468 1524
1469 1525 dtype_index = get_dtype_index(self.dtype)
1470 1526 procflag_dtype = get_procflag_dtype(dtype_index)
1471 1527
1472 1528 processFlags += procflag_dtype
1473 1529
1474 1530 if self.dataOut.flagDecodeData:
1475 1531 processFlags += PROCFLAG.DECODE_DATA
1476 1532
1477 1533 if self.dataOut.flagDeflipData:
1478 1534 processFlags += PROCFLAG.DEFLIP_DATA
1479 1535
1480 1536 if self.dataOut.code is not None:
1481 1537 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1482 1538
1483 1539 if self.dataOut.nCohInt > 1:
1484 1540 processFlags += PROCFLAG.COHERENT_INTEGRATION
1485 1541
1486 1542 if self.dataOut.type == "Spectra":
1487 1543 if self.dataOut.nIncohInt > 1:
1488 1544 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1489 1545
1490 1546 if self.dataOut.data_dc is not None:
1491 1547 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1492 1548
1493 1549 if self.dataOut.flagShiftFFT:
1494 1550 processFlags += PROCFLAG.SHIFT_FFT_DATA
1495 1551
1496 1552 return processFlags
1497 1553
1498 1554 def setBasicHeader(self):
1499 1555
1500 1556 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1501 1557 self.basicHeaderObj.version = self.versionFile
1502 1558 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1503 1559
1504 1560 utc = numpy.floor(self.dataOut.utctime)
1505 1561 milisecond = (self.dataOut.utctime - utc)* 1000.0
1506 1562
1507 1563 self.basicHeaderObj.utc = utc
1508 1564 self.basicHeaderObj.miliSecond = milisecond
1509 1565 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1510 1566 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1511 1567 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1512 1568
1513 1569 def setFirstHeader(self):
1514 1570 """
1515 1571 Obtiene una copia del First Header
1516 1572
1517 1573 Affected:
1518 1574
1519 1575 self.basicHeaderObj
1520 1576 self.systemHeaderObj
1521 1577 self.radarControllerHeaderObj
1522 1578 self.processingHeaderObj self.
1523 1579
1524 1580 Return:
1525 1581 None
1526 1582 """
1527 1583
1528 1584 raise NotImplementedError
1529 1585
1530 1586 def __writeFirstHeader(self):
1531 1587 """
1532 1588 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1533 1589
1534 1590 Affected:
1535 1591 __dataType
1536 1592
1537 1593 Return:
1538 1594 None
1539 1595 """
1540 1596
1541 # CALCULAR PARAMETROS
1597 # CALCULAR PARAMETROS
1542 1598
1543 1599 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1544 1600 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1545 1601
1546 1602 self.basicHeaderObj.write(self.fp)
1547 1603 self.systemHeaderObj.write(self.fp)
1548 1604 self.radarControllerHeaderObj.write(self.fp)
1549 1605 self.processingHeaderObj.write(self.fp)
1550 1606
1551 1607 def __setNewBlock(self):
1552 1608 """
1553 1609 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1554 1610
1555 1611 Return:
1556 1612 0 : si no pudo escribir nada
1557 1613 1 : Si escribio el Basic el First Header
1558 1614 """
1559 1615 if self.fp == None:
1560 1616 self.setNextFile()
1561 1617
1562 1618 if self.flagIsNewFile:
1563 1619 return 1
1564 1620
1565 1621 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1566 1622 self.basicHeaderObj.write(self.fp)
1567 1623 return 1
1568 1624
1569 1625 if not( self.setNextFile() ):
1570 1626 return 0
1571 1627
1572 1628 return 1
1573 1629
1574 1630
1575 1631 def writeNextBlock(self):
1576 1632 """
1577 1633 Selecciona el bloque siguiente de datos y los escribe en un file
1578 1634
1579 1635 Return:
1580 1636 0 : Si no hizo pudo escribir el bloque de datos
1581 1637 1 : Si no pudo escribir el bloque de datos
1582 1638 """
1583 1639 if not( self.__setNewBlock() ):
1584 1640 return 0
1585 1641
1586 1642 self.writeBlock()
1587 1643
1588 1644 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1589 1645 self.processingHeaderObj.dataBlocksPerFile)
1590 1646
1591 1647 return 1
1592 1648
1593 1649 def setNextFile(self):
1594 1650 """
1595 1651 Determina el siguiente file que sera escrito
1596 1652
1597 1653 Affected:
1598 1654 self.filename
1599 1655 self.subfolder
1600 1656 self.fp
1601 1657 self.setFile
1602 1658 self.flagIsNewFile
1603 1659
1604 1660 Return:
1605 1661 0 : Si el archivo no puede ser escrito
1606 1662 1 : Si el archivo esta listo para ser escrito
1607 1663 """
1608 1664 ext = self.ext
1609 1665 path = self.path
1610 1666
1611 1667 if self.fp != None:
1612 1668 self.fp.close()
1613 1669
1614 1670 timeTuple = time.localtime( self.dataOut.utctime)
1615 1671 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1616 1672
1617 1673 fullpath = os.path.join( path, subfolder )
1618 1674 setFile = self.setFile
1619 1675
1620 1676 if not( os.path.exists(fullpath) ):
1621 1677 os.mkdir(fullpath)
1622 1678 setFile = -1 #inicializo mi contador de seteo
1623 1679 else:
1624 1680 filesList = os.listdir( fullpath )
1625 1681 if len( filesList ) > 0:
1626 1682 filesList = sorted( filesList, key=str.lower )
1627 1683 filen = filesList[-1]
1628 1684 # el filename debera tener el siguiente formato
1629 1685 # 0 1234 567 89A BCDE (hex)
1630 1686 # x YYYY DDD SSS .ext
1631 1687 if isNumber( filen[8:11] ):
1632 1688 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1633 1689 else:
1634 1690 setFile = -1
1635 1691 else:
1636 1692 setFile = -1 #inicializo mi contador de seteo
1637 1693
1638 1694 setFile += 1
1639 1695
1640 1696 #If this is a new day it resets some values
1641 1697 if self.dataOut.datatime.date() > self.fileDate:
1642 1698 setFile = 0
1643 1699 self.nTotalBlocks = 0
1644 1700
1645 1701 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1646 1702
1647 1703 filename = os.path.join( path, subfolder, filen )
1648 1704
1649 1705 fp = open( filename,'wb' )
1650 1706
1651 1707 self.blockIndex = 0
1652 1708
1653 1709 #guardando atributos
1654 1710 self.filename = filename
1655 1711 self.subfolder = subfolder
1656 1712 self.fp = fp
1657 1713 self.setFile = setFile
1658 1714 self.flagIsNewFile = 1
1659 1715 self.fileDate = self.dataOut.datatime.date()
1660 1716
1661 1717 self.setFirstHeader()
1662 1718
1663 1719 print '[Writing] Opening file: %s'%self.filename
1664 1720
1665 1721 self.__writeFirstHeader()
1666 1722
1667 1723 return 1
1668 1724
1669 1725 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1670 1726 """
1671 1727 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1672 1728
1673 1729 Inputs:
1674 1730 path : directory where data will be saved
1675 1731 profilesPerBlock : number of profiles per block
1676 1732 set : initial file set
1677 1733 datatype : An integer number that defines data type:
1678 1734 0 : int8 (1 byte)
1679 1735 1 : int16 (2 bytes)
1680 1736 2 : int32 (4 bytes)
1681 1737 3 : int64 (8 bytes)
1682 1738 4 : float32 (4 bytes)
1683 1739 5 : double64 (8 bytes)
1684 1740
1685 1741 Return:
1686 1742 0 : Si no realizo un buen seteo
1687 1743 1 : Si realizo un buen seteo
1688 1744 """
1689 1745
1690 1746 if ext == None:
1691 1747 ext = self.ext
1692 1748
1693 1749 self.ext = ext.lower()
1694 1750
1695 1751 self.path = path
1696 1752
1697 1753 if set is None:
1698 1754 self.setFile = -1
1699 1755 else:
1700 1756 self.setFile = set - 1
1701 1757
1702 1758 self.blocksPerFile = blocksPerFile
1703 1759
1704 1760 self.profilesPerBlock = profilesPerBlock
1705 1761
1706 1762 self.dataOut = dataOut
1707 1763 self.fileDate = self.dataOut.datatime.date()
1708 1764 #By default
1709 1765 self.dtype = self.dataOut.dtype
1710 1766
1711 1767 if datatype is not None:
1712 1768 self.dtype = get_numpy_dtype(datatype)
1713 1769
1714 1770 if not(self.setNextFile()):
1715 1771 print "[Writing] There isn't a next file"
1716 1772 return 0
1717 1773
1718 1774 self.setBlockDimension()
1719 1775
1720 1776 return 1
1721 1777
1722 1778 def run(self, dataOut, **kwargs):
1723 1779
1724 1780 if not(self.isConfig):
1725 1781
1726 1782 self.setup(dataOut, **kwargs)
1727 1783 self.isConfig = True
1728 1784
1729 1785 self.putData()
1730 1786
Show file before | Show file after | Hide comments
@@ -1,637 +1,637
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 13 # from _sha import blocksize
14 14
15 15 class VoltageReader(JRODataReader, ProcessingUnit):
16 16 """
17 17 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
18 18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
19 19 perfiles*alturas*canales) son almacenados en la variable "buffer".
20 20
21 21 perfiles * alturas * canales
22 22
23 23 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
24 24 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
25 25 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
26 26 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
27 27
28 28 Example:
29 29
30 30 dpath = "/home/myuser/data"
31 31
32 32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33 33
34 34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35 35
36 36 readerObj = VoltageReader()
37 37
38 38 readerObj.setup(dpath, startTime, endTime)
39 39
40 40 while(True):
41 41
42 42 #to get one profile
43 43 profile = readerObj.getData()
44 44
45 45 #print the profile
46 46 print profile
47 47
48 48 #If you want to see all datablock
49 49 print readerObj.datablock
50 50
51 51 if readerObj.flagNoMoreFiles:
52 52 break
53 53
54 54 """
55 55
56 56 ext = ".r"
57 57
58 58 optchar = "D"
59 59 dataOut = None
60 60
61 61 def __init__(self):
62 62 """
63 63 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
64 64
65 65 Input:
66 66 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
67 67 almacenar un perfil de datos cada vez que se haga un requerimiento
68 68 (getData). El perfil sera obtenido a partir del buffer de datos,
69 69 si el buffer esta vacio se hara un nuevo proceso de lectura de un
70 70 bloque de datos.
71 71 Si este parametro no es pasado se creara uno internamente.
72 72
73 73 Variables afectadas:
74 74 self.dataOut
75 75
76 76 Return:
77 77 None
78 78 """
79 79
80 80 ProcessingUnit.__init__(self)
81 81
82 82 self.isConfig = False
83 83
84 84 self.datablock = None
85 85
86 86 self.utc = 0
87 87
88 88 self.ext = ".r"
89 89
90 90 self.optchar = "D"
91 91
92 92 self.basicHeaderObj = BasicHeader(LOCALTIME)
93 93
94 94 self.systemHeaderObj = SystemHeader()
95 95
96 96 self.radarControllerHeaderObj = RadarControllerHeader()
97 97
98 98 self.processingHeaderObj = ProcessingHeader()
99 99
100 100 self.online = 0
101 101
102 102 self.fp = None
103 103
104 104 self.idFile = None
105 105
106 106 self.dtype = None
107 107
108 108 self.fileSizeByHeader = None
109 109
110 110 self.filenameList = []
111 111
112 112 self.filename = None
113 113
114 114 self.fileSize = None
115 115
116 116 self.firstHeaderSize = 0
117 117
118 118 self.basicHeaderSize = 24
119 119
120 120 self.pathList = []
121 121
122 122 self.filenameList = []
123 123
124 124 self.lastUTTime = 0
125 125
126 126 self.maxTimeStep = 30
127 127
128 128 self.flagNoMoreFiles = 0
129 129
130 130 self.set = 0
131 131
132 132 self.path = None
133 133
134 134 self.profileIndex = 2**32-1
135 135
136 136 self.delay = 3 #seconds
137 137
138 138 self.nTries = 3 #quantity tries
139 139
140 140 self.nFiles = 3 #number of files for searching
141 141
142 142 self.nReadBlocks = 0
143 143
144 144 self.flagIsNewFile = 1
145 145
146 146 self.__isFirstTimeOnline = 1
147 147
148 148 # self.ippSeconds = 0
149 149
150 150 self.flagDiscontinuousBlock = 0
151 151
152 152 self.flagIsNewBlock = 0
153 153
154 154 self.nTotalBlocks = 0
155 155
156 156 self.blocksize = 0
157 157
158 158 self.dataOut = self.createObjByDefault()
159 159
160 160 self.nTxs = 1
161 161
162 162 self.txIndex = 0
163 163
164 164 def createObjByDefault(self):
165 165
166 166 dataObj = Voltage()
167 167
168 168 return dataObj
169 169
170 170 def __hasNotDataInBuffer(self):
171 171
172 172 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
173 173 return 1
174 174
175 175 return 0
176 176
177 177
178 178 def getBlockDimension(self):
179 179 """
180 Obtiene la cantidad de puntos a leer por cada bloque de datos
181
182 Affected:
183 self.blocksize
180 Obtiene la cantidad de puntos a leer por cada bloque de datos
181
182 Affected:
183 self.blocksize
184 184
185 Return:
186 None
185 Return:
186 None
187 187 """
188 188 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
189 189 self.blocksize = pts2read
190 190
191 191
192 192 def readBlock(self):
193 193 """
194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
197 es seteado a 0
198
199 Inputs:
200 None
194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
197 es seteado a 0
201 198
202 Return:
203 None
204
205 Affected:
206 self.profileIndex
207 self.datablock
208 self.flagIsNewFile
209 self.flagIsNewBlock
210 self.nTotalBlocks
199 Inputs:
200 None
201
202 Return:
203 None
211 204
212 Exceptions:
213 Si un bloque leido no es un bloque valido
205 Affected:
206 self.profileIndex
207 self.datablock
208 self.flagIsNewFile
209 self.flagIsNewBlock
210 self.nTotalBlocks
211
212 Exceptions:
213 Si un bloque leido no es un bloque valido
214 214 """
215 215 current_pointer_location = self.fp.tell()
216 216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217 217
218 218 try:
219 219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
220 220 except:
221 221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222 222
223 223 if self.waitDataBlock(pointer_location=current_pointer_location):
224 224 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
225 225 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
226 226 # return 0
227 227
228 228 #Dimensions : nChannels, nProfiles, nSamples
229 229
230 230 junk = numpy.transpose(junk, (2,0,1))
231 231 self.datablock = junk['real'] + junk['imag']*1j
232 232
233 233 self.profileIndex = 0
234 234
235 235 self.flagIsNewFile = 0
236 236 self.flagIsNewBlock = 1
237 237
238 238 self.nTotalBlocks += 1
239 239 self.nReadBlocks += 1
240 240
241 241 return 1
242 242
243 243 def getFirstHeader(self):
244 244
245 245 self.getBasicHeader()
246 246
247 247 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
248 248
249 249 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
250 250
251 251 if self.nTxs > 1:
252 252 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
253 253
254 254 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
255 255
256 256 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
257 257 #
258 258 # if self.radarControllerHeaderObj.code is not None:
259 259 #
260 260 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
261 261 #
262 262 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
263 263 #
264 264 # self.dataOut.code = self.radarControllerHeaderObj.code
265 265
266 266 self.dataOut.dtype = self.dtype
267 267
268 268 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
269 269
270 270 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
271 271
272 272 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
273 273
274 274 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
275 275
276 276 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
277 277
278 278 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
279 279
280 280 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
281 281
282 282 def reshapeData(self):
283 283
284 284 if self.nTxs < 0:
285 285 return
286 286
287 287 if self.nTxs == 1:
288 288 return
289 289
290 290 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
291 291 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
292 292
293 293 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
294 294 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
295 295
296 296 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
297 297
298 298 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
299 299 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
300 300 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
301 301
302 302 return
303 303
304 304 def getData(self):
305 305 """
306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
309 "readNextBlock"
310
311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
312
313 Return:
314
315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
316 es igual al total de perfiles leidos desde el archivo.
317
318 Si self.getByBlock == False:
306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
309 "readNextBlock"
319 310
320 self.dataOut.data = buffer[:, thisProfile, :]
321
322 shape = [nChannels, nHeis]
323
324 Si self.getByBlock == True:
311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
325 312
326 self.dataOut.data = buffer[:, :, :]
327
328 shape = [nChannels, nProfiles, nHeis]
313 Return:
329 314
330 Variables afectadas:
331 self.dataOut
332 self.profileIndex
315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
316 es igual al total de perfiles leidos desde el archivo.
333 317
334 Affected:
335 self.dataOut
336 self.profileIndex
337 self.flagDiscontinuousBlock
338 self.flagIsNewBlock
318 Si self.getByBlock == False:
319
320 self.dataOut.data = buffer[:, thisProfile, :]
321
322 shape = [nChannels, nHeis]
323
324 Si self.getByBlock == True:
325
326 self.dataOut.data = buffer[:, :, :]
327
328 shape = [nChannels, nProfiles, nHeis]
329
330 Variables afectadas:
331 self.dataOut
332 self.profileIndex
333
334 Affected:
335 self.dataOut
336 self.profileIndex
337 self.flagDiscontinuousBlock
338 self.flagIsNewBlock
339 339 """
340 340
341 341 if self.flagNoMoreFiles:
342 342 self.dataOut.flagNoData = True
343 343 print 'Process finished'
344 344 return 0
345 345
346 346 self.flagDiscontinuousBlock = 0
347 347 self.flagIsNewBlock = 0
348 348
349 349 if self.__hasNotDataInBuffer():
350 350
351 351 if not( self.readNextBlock() ):
352 352 return 0
353 353
354 354 self.getFirstHeader()
355 355
356 356 self.reshapeData()
357 357
358 358 if self.datablock is None:
359 359 self.dataOut.flagNoData = True
360 360 return 0
361 361
362 362 if not self.getByBlock:
363 363
364 364 """
365 Return profile by profile
366
367 If nTxs > 1 then one profile is divided by nTxs and number of total
368 blocks is increased by nTxs (nProfiles *= nTxs)
365 Return profile by profile
366
367 If nTxs > 1 then one profile is divided by nTxs and number of total
368 blocks is increased by nTxs (nProfiles *= nTxs)
369 369 """
370 370 self.dataOut.flagDataAsBlock = False
371 371 self.dataOut.data = self.datablock[:,self.profileIndex,:]
372 372 self.dataOut.profileIndex = self.profileIndex
373 373
374 374 self.profileIndex += 1
375 375
376 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
377 # """
378 # Return all block
379 # """
380 # self.dataOut.flagDataAsBlock = True
381 # self.dataOut.data = self.datablock
382 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
383 #
384 # self.profileIndex = self.dataOut.nProfiles
376 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
377 # """
378 # Return all block
379 # """
380 # self.dataOut.flagDataAsBlock = True
381 # self.dataOut.data = self.datablock
382 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
383 #
384 # self.profileIndex = self.dataOut.nProfiles
385 385
386 386 else:
387 387 """
388 Return a block
388 Return a block
389 389 """
390 390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 391 if self.selBlocktime != None:
392 392 if self.dataOut.nCohInt is not None:
393 393 nCohInt = self.dataOut.nCohInt
394 394 else:
395 nCohInt = 1
395 nCohInt = 1
396 396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397 397
398 398 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
399 399 self.profileIndex += self.selBlocksize
400 400 datasize = self.dataOut.data.shape[1]
401 401
402 402 if datasize < self.selBlocksize:
403 403 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
404 404 buffer[:,:datasize,:] = self.dataOut.data
405 405
406 406 while datasize < self.selBlocksize: #Not enough profiles to fill the block
407 407 if not( self.readNextBlock() ):
408 408 return 0
409 409 self.getFirstHeader()
410 410 self.reshapeData()
411 411 if self.datablock is None:
412 412 self.dataOut.flagNoData = True
413 413 return 0
414 414 #stack data
415 415 blockIndex = self.selBlocksize - datasize
416 416 datablock1 = self.datablock[:,:blockIndex,:]
417 417
418 418 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
419 419 datasize += datablock1.shape[1]
420 420
421 421 self.dataOut.data = buffer
422 422 self.profileIndex = blockIndex
423 423
424 424 self.dataOut.flagDataAsBlock = True
425 425 self.dataOut.nProfiles = self.dataOut.data.shape[1]
426 426
427 427 self.dataOut.flagNoData = False
428 428
429 429 self.getBasicHeader()
430 430
431 431 self.dataOut.realtime = self.online
432
432
433 433 return self.dataOut.data
434 434
435 435 class VoltageWriter(JRODataWriter, Operation):
436 436 """
437 437 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
438 438 de los datos siempre se realiza por bloques.
439 439 """
440 440
441 441 ext = ".r"
442 442
443 443 optchar = "D"
444 444
445 445 shapeBuffer = None
446 446
447 447
448 448 def __init__(self):
449 449 """
450 450 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
451 451
452 452 Affected:
453 453 self.dataOut
454 454
455 455 Return: None
456 456 """
457 457 Operation.__init__(self)
458 458
459 459 self.nTotalBlocks = 0
460 460
461 461 self.profileIndex = 0
462 462
463 463 self.isConfig = False
464 464
465 465 self.fp = None
466 466
467 467 self.flagIsNewFile = 1
468 468
469 469 self.blockIndex = 0
470 470
471 471 self.flagIsNewBlock = 0
472 472
473 473 self.setFile = None
474 474
475 475 self.dtype = None
476 476
477 477 self.path = None
478 478
479 479 self.filename = None
480 480
481 481 self.basicHeaderObj = BasicHeader(LOCALTIME)
482 482
483 483 self.systemHeaderObj = SystemHeader()
484 484
485 485 self.radarControllerHeaderObj = RadarControllerHeader()
486 486
487 487 self.processingHeaderObj = ProcessingHeader()
488 488
489 489 def hasAllDataInBuffer(self):
490 490 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
491 491 return 1
492 492 return 0
493 493
494 494
495 495 def setBlockDimension(self):
496 496 """
497 497 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
498 498
499 499 Affected:
500 500 self.shape_spc_Buffer
501 501 self.shape_cspc_Buffer
502 502 self.shape_dc_Buffer
503 503
504 504 Return: None
505 505 """
506 506 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
507 507 self.processingHeaderObj.nHeights,
508 508 self.systemHeaderObj.nChannels)
509 509
510 510 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
511 511 self.processingHeaderObj.profilesPerBlock,
512 512 self.processingHeaderObj.nHeights),
513 513 dtype=numpy.dtype('complex64'))
514 514
515 515 def writeBlock(self):
516 516 """
517 517 Escribe el buffer en el file designado
518 518
519 519 Affected:
520 520 self.profileIndex
521 521 self.flagIsNewFile
522 522 self.flagIsNewBlock
523 523 self.nTotalBlocks
524 524 self.blockIndex
525 525
526 526 Return: None
527 527 """
528 528 data = numpy.zeros( self.shapeBuffer, self.dtype )
529 529
530 530 junk = numpy.transpose(self.datablock, (1,2,0))
531 531
532 532 data['real'] = junk.real
533 533 data['imag'] = junk.imag
534 534
535 535 data = data.reshape( (-1) )
536 536
537 537 data.tofile( self.fp )
538 538
539 539 self.datablock.fill(0)
540 540
541 541 self.profileIndex = 0
542 542 self.flagIsNewFile = 0
543 543 self.flagIsNewBlock = 1
544 544
545 545 self.blockIndex += 1
546 546 self.nTotalBlocks += 1
547 547
548 548 # print "[Writing] Block = %04d" %self.blockIndex
549 549
550 550 def putData(self):
551 551 """
552 552 Setea un bloque de datos y luego los escribe en un file
553 553
554 554 Affected:
555 555 self.flagIsNewBlock
556 556 self.profileIndex
557 557
558 558 Return:
559 559 0 : Si no hay data o no hay mas files que puedan escribirse
560 560 1 : Si se escribio la data de un bloque en un file
561 561 """
562 562 if self.dataOut.flagNoData:
563 563 return 0
564 564
565 565 self.flagIsNewBlock = 0
566 566
567 567 if self.dataOut.flagDiscontinuousBlock:
568 568 self.datablock.fill(0)
569 569 self.profileIndex = 0
570 570 self.setNextFile()
571 571
572 572 if self.profileIndex == 0:
573 573 self.setBasicHeader()
574 574
575 575 self.datablock[:,self.profileIndex,:] = self.dataOut.data
576 576
577 577 self.profileIndex += 1
578 578
579 579 if self.hasAllDataInBuffer():
580 580 #if self.flagIsNewFile:
581 581 self.writeNextBlock()
582 582 # self.setFirstHeader()
583 583
584 584 return 1
585 585
586 586 def __getBlockSize(self):
587 587 '''
588 588 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
589 589 '''
590 590
591 591 dtype_width = self.getDtypeWidth()
592 592
593 593 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
594 594
595 595 return blocksize
596 596
597 597 def setFirstHeader(self):
598 598
599 599 """
600 600 Obtiene una copia del First Header
601 601
602 602 Affected:
603 603 self.systemHeaderObj
604 604 self.radarControllerHeaderObj
605 605 self.dtype
606 606
607 607 Return:
608 608 None
609 609 """
610 610
611 611 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
612 612 self.systemHeaderObj.nChannels = self.dataOut.nChannels
613 613 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
614 614
615 615 self.processingHeaderObj.dtype = 0 # Voltage
616 616 self.processingHeaderObj.blockSize = self.__getBlockSize()
617 617 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
618 618 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
619 619 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
620 620 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
621 621 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
622 622 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
623 623
624 624 if self.dataOut.code is not None:
625 625 self.processingHeaderObj.code = self.dataOut.code
626 626 self.processingHeaderObj.nCode = self.dataOut.nCode
627 627 self.processingHeaderObj.nBaud = self.dataOut.nBaud
628 628
629 629 if self.processingHeaderObj.nWindows != 0:
630 630 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
631 631 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
632 632 self.processingHeaderObj.nHeights = self.dataOut.nHeights
633 633 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
634 634
635 635 self.processingHeaderObj.processFlags = self.getProcessFlags()
636 636
637 637 self.setBasicHeader() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1012 +1,1013
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13 13
14 14 import time
15 15
16 16 import threading
17 17 Thread = threading.Thread
18 18
19 19 # try:
20 20 # from gevent import sleep
21 21 # except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25
26 26 class Remote(Thread):
27 27 """
28 28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 29 used to upload or download files remotely.
30 30
31 31 Non-standard Python modules used:
32 32 None
33 33
34 34 Written by:
35 35
36 36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37 37
38 38 """
39 39
40 40 server = None
41 41 username = None
42 42 password = None
43 43 remotefolder = None
44 44
45 45 period = 60
46 46 fileList = []
47 47 bussy = False
48 48
49 49 def __init__(self, server, username, password, remotefolder, period=60):
50 50
51 51 Thread.__init__(self)
52 52
53 53 self.setDaemon(True)
54 54
55 55 self.status = 0
56 56
57 57 self.__server = server
58 58 self.__username = username
59 59 self.__password = password
60 60 self.__remotefolder = remotefolder
61 61
62 62 self.period = period
63 63
64 64 self.fileList = []
65 65 self.bussy = False
66 66
67 67 self.stopFlag = False
68 68
69 69 print "[Remote Server] Opening server: %s" %self.__server
70 70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 71 print "[Remote Server] %s server was opened successfully" %self.__server
72 72
73 73 self.close()
74 74
75 75 self.mutex = threading.Lock()
76 76
77 77 def stop(self):
78 78
79 79 self.stopFlag = True
80 80 self.join(10)
81 81
82 82 def open(self):
83 83 """
84 84 Connect to server and create a connection class (FTP or SSH) to remote server.
85 85 """
86 86 raise NotImplementedError, "Implement this method in child class"
87 87
88 88 def close(self):
89 89 """
90 90 Close connection to server
91 91 """
92 92 raise NotImplementedError, "Implement this method in child class"
93 93
94 94 def mkdir(self, remotefolder):
95 95 """
96 96 Create a folder remotely
97 97 """
98 98 raise NotImplementedError, "Implement this method in child class"
99 99
100 100 def cd(self, remotefolder):
101 101 """
102 102 Change working directory in remote server
103 103 """
104 104 raise NotImplementedError, "Implement this method in child class"
105 105
106 106 def download(self, filename, localfolder=None):
107 107 """
108 108 Download a file from server to local host
109 109 """
110 110 raise NotImplementedError, "Implement this method in child class"
111 111
112 112 def sendFile(self, fullfilename):
113 113 """
114 114 sendFile method is used to upload a local file to the current directory in remote server
115 115
116 116 Inputs:
117 117 fullfilename - full path name of local file to store in remote directory
118 118
119 119 Returns:
120 120 0 in error case else 1
121 121 """
122 122 raise NotImplementedError, "Implement this method in child class"
123 123
124 124 def upload(self, fullfilename, remotefolder=None):
125 125 """
126 126 upload method is used to upload a local file to remote directory. This method changes
127 127 working directory before sending a file.
128 128
129 129 Inputs:
130 130 fullfilename - full path name of local file to store in remote directory
131 131
132 132 remotefolder - remote directory
133 133
134 134 Returns:
135 135 0 in error case else 1
136 136 """
137 137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
138 138
139 139 if not self.status:
140 140 return 0
141 141
142 142 if remotefolder == None:
143 143 remotefolder = self.remotefolder
144 144
145 145 if not self.cd(remotefolder):
146 146 return 0
147 147
148 148 if not self.sendFile(fullfilename):
149 149 print "[Remote Server] Error uploading file %s" %fullfilename
150 150 return 0
151 151
152 152 print "[Remote Server] upload finished successfully"
153 153
154 154 return 1
155 155
156 156 def delete(self, filename):
157 157 """
158 158 Remove a file from remote server
159 159 """
160 160 pass
161 161
162 162 def updateFileList(self, fileList):
163 163 """
164 164 Remove a file from remote server
165 165 """
166 166
167 167 if fileList == self.fileList:
168 168 return 0
169 169
170 170 self.mutex.acquire()
171 # init = time.time()
172 #
173 # while(self.bussy):
174 # sleep(0.1)
175 # if time.time() - init > 2*self.period:
176 # return 0
177
171 # init = time.time()
172 #
173 # while(self.bussy):
174 # sleep(0.1)
175 # if time.time() - init > 2*self.period:
176 # return 0
177
178 178 self.fileList = fileList
179 179 self.mutex.release()
180 180 return 1
181 181
182 182 def run(self):
183 183
184 184 if not self.status:
185 185 print "Finishing FTP service"
186 186 return
187 187
188 188 if not self.cd(self.remotefolder):
189 189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
190 190
191 191 while True:
192 192
193 193 for i in range(self.period):
194 194 if self.stopFlag:
195 195 break
196 196 sleep(1)
197 197
198 198 if self.stopFlag:
199 199 break
200 200
201 # self.bussy = True
201 # self.bussy = True
202 202 self.mutex.acquire()
203 203
204 204 print "[Remote Server] Opening %s" %self.__server
205 205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 206 self.mutex.release()
207 207 continue
208 208
209 209 for thisFile in self.fileList:
210 210 self.upload(thisFile, self.remotefolder)
211 211
212 212 print "[Remote Server] Closing %s" %self.__server
213 213 self.close()
214 214
215 215 self.mutex.release()
216 216 # self.bussy = False
217 217
218 218 print "[Remote Server] Thread stopped successfully"
219 219
220 220 class FTPClient(Remote):
221 221
222 222 __ftpClientObj = None
223 223
224 224 def __init__(self, server, username, password, remotefolder, period=60):
225 225 """
226 226 """
227 227 Remote.__init__(self, server, username, password, remotefolder, period)
228 228
229 229 def open(self, server, username, password, remotefolder):
230 230
231 231 """
232 232 This method is used to set FTP parameters and establish a connection to remote server
233 233
234 234 Inputs:
235 235 server - remote server IP Address
236 236
237 237 username - remote server Username
238 238
239 239 password - remote server password
240 240
241 241 remotefolder - remote server current working directory
242 242
243 243 Return:
244 244 Boolean - Returns 1 if a connection has been established, 0 otherwise
245 245
246 246 Affects:
247 247 self.status - in case of error or fail connection this parameter is set to 0 else 1
248 248
249 249 """
250 250
251 251 if server == None:
252 252 raise ValueError, "FTP server should be defined"
253 253
254 254 if username == None:
255 255 raise ValueError, "FTP username should be defined"
256 256
257 257 if password == None:
258 258 raise ValueError, "FTP password should be defined"
259 259
260 260 if remotefolder == None:
261 261 raise ValueError, "FTP remote folder should be defined"
262 262
263 263 try:
264 264 ftpClientObj = ftplib.FTP(server)
265 265 except ftplib.all_errors, e:
266 266 print "[FTP Server]: FTP server connection fail: %s" %server
267 267 print "[FTP Server]:", e
268 268 self.status = 0
269 269 return 0
270 270
271 271 try:
272 272 ftpClientObj.login(username, password)
273 273 except ftplib.all_errors:
274 274 print "[FTP Server]: FTP username or password are incorrect"
275 275 self.status = 0
276 276 return 0
277 277
278 278 if remotefolder == None:
279 279 remotefolder = ftpClientObj.pwd()
280 280 else:
281 281 try:
282 282 ftpClientObj.cwd(remotefolder)
283 283 except ftplib.all_errors:
284 284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
285 285 remotefolder = ftpClientObj.pwd()
286 286
287 287 self.server = server
288 288 self.username = username
289 289 self.password = password
290 290 self.remotefolder = remotefolder
291 291 self.__ftpClientObj = ftpClientObj
292 292 self.status = 1
293 293
294 294 return 1
295 295
296 296 def close(self):
297 297 """
298 298 Close connection to remote server
299 299 """
300 300 if not self.status:
301 301 return 0
302 302
303 303 self.__ftpClientObj.close()
304 304
305 305 def mkdir(self, remotefolder):
306 306 """
307 307 mkdir is used to make a new directory in remote server
308 308
309 309 Input:
310 310 remotefolder - directory name
311 311
312 312 Return:
313 313 0 in error case else 1
314 314 """
315 315 if not self.status:
316 316 return 0
317 317
318 318 try:
319 319 self.__ftpClientObj.mkd(dirname)
320 320 except ftplib.all_errors:
321 321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
322 322 return 0
323 323
324 324 return 1
325 325
326 326 def cd(self, remotefolder):
327 327 """
328 328 cd is used to change remote working directory on server
329 329
330 330 Input:
331 331 remotefolder - current working directory
332 332
333 333 Affects:
334 334 self.remotefolder
335 335
336 336 Return:
337 337 0 in case of error else 1
338 338 """
339 339 if not self.status:
340 340 return 0
341 341
342 342 if remotefolder == self.remotefolder:
343 343 return 1
344 344
345 345 try:
346 346 self.__ftpClientObj.cwd(remotefolder)
347 347 except ftplib.all_errors:
348 348 print '[FTP Server]: Error changing to %s' %remotefolder
349 349 print '[FTP Server]: Trying to create remote folder'
350 350
351 351 if not self.mkdir(remotefolder):
352 352 print '[FTP Server]: Remote folder could not be created'
353 353 return 0
354 354
355 355 try:
356 356 self.__ftpClientObj.cwd(remotefolder)
357 357 except ftplib.all_errors:
358 358 return 0
359 359
360 360 self.remotefolder = remotefolder
361 361
362 362 return 1
363 363
364 364 def sendFile(self, fullfilename):
365 365
366 366 if not self.status:
367 367 return 0
368 368
369 369 fp = open(fullfilename, 'rb')
370 370
371 371 filename = os.path.basename(fullfilename)
372 372
373 373 command = "STOR %s" %filename
374 374
375 375 try:
376 376 self.__ftpClientObj.storbinary(command, fp)
377 377 except ftplib.all_errors, e:
378 378 print "[FTP Server]:", e
379 379 return 0
380 380
381 381 try:
382 382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 383 except ftplib.all_errors, e:
384 384 print "[FTP Server]:", e
385 385
386 386 fp.close()
387 387
388 388 return 1
389 389
390 390 class SSHClient(Remote):
391 391
392 392 __sshClientObj = None
393 393 __scpClientObj = None
394 394
395 395 def __init__(self, server, username, password, remotefolder, period=60):
396 396 """
397 397 """
398 398 Remote.__init__(self, server, username, password, remotefolder, period)
399 399
400 400 def open(self, server, username, password, remotefolder, port=22):
401 401
402 402 """
403 403 This method is used to set SSH parameters and establish a connection to a remote server
404 404
405 405 Inputs:
406 406 server - remote server IP Address
407 407
408 408 username - remote server Username
409 409
410 410 password - remote server password
411 411
412 412 remotefolder - remote server current working directory
413 413
414 414 Return: void
415 415
416 416 Affects:
417 417 self.status - in case of error or fail connection this parameter is set to 0 else 1
418 418
419 419 """
420 420 import socket
421 421
422 422 if server == None:
423 423 raise ValueError, "SSH server should be defined"
424 424
425 425 if username == None:
426 426 raise ValueError, "SSH username should be defined"
427 427
428 428 if password == None:
429 429 raise ValueError, "SSH password should be defined"
430 430
431 431 if remotefolder == None:
432 432 raise ValueError, "SSH remote folder should be defined"
433 433
434 434 sshClientObj = paramiko.SSHClient()
435 435
436 436 sshClientObj.load_system_host_keys()
437 437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
438 438
439 439 self.status = 0
440 440 try:
441 441 sshClientObj.connect(server, username=username, password=password, port=port)
442 442 except paramiko.AuthenticationException, e:
443 443 # print "SSH username or password are incorrect: %s"
444 444 print "[SSH Server]:", e
445 445 return 0
446 446 except SSHException, e:
447 447 print "[SSH Server]:", e
448 448 return 0
449 449 except socket.error:
450 450 self.status = 0
451 451 print "[SSH Server]:", e
452 452 return 0
453 453
454 454 self.status = 1
455 455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
456 456
457 457 if remotefolder == None:
458 458 remotefolder = self.pwd()
459 459
460 460 self.server = server
461 461 self.username = username
462 462 self.password = password
463 463 self.__sshClientObj = sshClientObj
464 464 self.__scpClientObj = scpClientObj
465 465 self.status = 1
466 466
467 467 if not self.cd(remotefolder):
468 468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
469 469 return 0
470 470
471 471 self.remotefolder = remotefolder
472 472
473 473 return 1
474 474
475 475 def close(self):
476 476 """
477 477 Close connection to remote server
478 478 """
479 479 if not self.status:
480 480 return 0
481 481
482 482 self.__scpClientObj.close()
483 483 self.__sshClientObj.close()
484 484
485 485 def __execute(self, command):
486 486 """
487 487 __execute a command on remote server
488 488
489 489 Input:
490 490 command - Exmaple 'ls -l'
491 491
492 492 Return:
493 493 0 in error case else 1
494 494 """
495 495 if not self.status:
496 496 return 0
497 497
498 498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
499 499
500 500 result = stderr.readlines()
501 501 if len(result) > 1:
502 502 return 0
503 503
504 504 result = stdout.readlines()
505 505 if len(result) > 1:
506 506 return result[0][:-1]
507 507
508 508 return 1
509 509
510 510 def mkdir(self, remotefolder):
511 511 """
512 512 mkdir is used to make a new directory in remote server
513 513
514 514 Input:
515 515 remotefolder - directory name
516 516
517 517 Return:
518 518 0 in error case else 1
519 519 """
520 520
521 521 command = 'mkdir %s' %remotefolder
522 522
523 523 return self.__execute(command)
524 524
525 525 def pwd(self):
526 526
527 527 command = 'pwd'
528 528
529 529 return self.__execute(command)
530 530
531 531 def cd(self, remotefolder):
532 532 """
533 533 cd is used to change remote working directory on server
534 534
535 535 Input:
536 536 remotefolder - current working directory
537 537
538 538 Affects:
539 539 self.remotefolder
540 540
541 541 Return:
542 542 0 in case of error else 1
543 543 """
544 544 if not self.status:
545 545 return 0
546 546
547 547 if remotefolder == self.remotefolder:
548 548 return 1
549 549
550 550 chk_command = "cd %s; pwd" %remotefolder
551 551 mkdir_command = "mkdir %s" %remotefolder
552 552
553 553 if not self.__execute(chk_command):
554 554 if not self.__execute(mkdir_command):
555 555 self.remotefolder = None
556 556 return 0
557 557
558 558 self.remotefolder = remotefolder
559 559
560 560 return 1
561 561
562 562 def sendFile(self, fullfilename):
563 563
564 564 if not self.status:
565 565 return 0
566 566
567 567 try:
568 568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
569 569 except scp.ScpError, e:
570 570 print "[SSH Server]", str(e)
571 571 return 0
572 572
573 573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
574 574 command = 'chmod 775 %s' %remotefile
575 575
576 576 return self.__execute(command)
577 577
578 578 class SendToServer(ProcessingUnit):
579 579
580 580 def __init__(self):
581 581
582 582 ProcessingUnit.__init__(self)
583 583
584 584 self.isConfig = False
585 585 self.clientObj = None
586 586
587 587
588 588
589 589 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
590 590
591 591 self.clientObj = None
592 592 self.localfolder = localfolder
593 593 self.ext = ext
594 594 self.period = period
595 595
596 596 if str.lower(protocol) == 'ftp':
597 597 self.clientObj = FTPClient(server, username, password, remotefolder, period)
598 598
599 599 if str.lower(protocol) == 'ssh':
600 600 self.clientObj = SSHClient(server, username, password, remotefolder, period)
601 601
602 602 if not self.clientObj:
603 603 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
604 604
605 605 self.clientObj.start()
606 606
607 607 def findFiles(self):
608 608
609 609 if not type(self.localfolder) == list:
610 610 folderList = [self.localfolder]
611 611 else:
612 612 folderList = self.localfolder
613 613
614 614 #Remove duplicate items
615 615 folderList = list(set(folderList))
616 616
617 617 fullfilenameList = []
618 618
619 619 for thisFolder in folderList:
620 620
621 621 print "[Remote Server]: Searching files on %s" %thisFolder
622 622
623 623 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
624 624
625 625 if len(filenameList) < 1:
626
626 627 continue
627 628
628 629 for thisFile in filenameList:
629 630 fullfilename = os.path.join(thisFolder, thisFile)
630 631
631 632 if fullfilename in fullfilenameList:
632 633 continue
633 634
634 635 #Only files modified in the last 30 minutes are considered
635 636 if os.path.getmtime(fullfilename) < time.time() - 30*60:
636 637 continue
637 638
638 639 fullfilenameList.append(fullfilename)
639 640
640 641 return fullfilenameList
641 642
642 643 def run(self, **kwargs):
643 644 if not self.isConfig:
644 645 self.init = time.time()
645 646 self.setup(**kwargs)
646 647 self.isConfig = True
647 648
648 649 if not self.clientObj.is_alive():
649 650 print "[Remote Server]: Restarting connection "
650 651 self.setup(**kwargs)
651 652
652 653 if time.time() - self.init >= self.period:
653 654 fullfilenameList = self.findFiles()
654 655
655 656 if self.clientObj.updateFileList(fullfilenameList):
656 657 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
657 658 self.init = time.time()
658 659
659 660 def close(self):
660 661 print "[Remote Server] Stopping thread"
661 662 self.clientObj.stop()
662 663
663 664
664 665 class FTP(object):
665 666 """
666 667 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
667 668
668 669 Non-standard Python modules used: None
669 670
670 671 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
671 672 """
672 673
673 674 def __init__(self,server = None, username=None, password=None, remotefolder=None):
674 675 """
675 676 This method is used to setting parameters for FTP and establishing connection to remote server
676 677
677 678 Inputs:
678 679 server - remote server IP Address
679 680
680 681 username - remote server Username
681 682
682 683 password - remote server password
683 684
684 685 remotefolder - remote server current working directory
685 686
686 687 Return: void
687 688
688 689 Affects:
689 690 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
690 691
691 692 self.folderList - sub-folder list of remote folder
692 693
693 694 self.fileList - file list of remote folder
694 695
695 696
696 697 """
697 698
698 699 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
699 700 server, username, password, remotefolder = self.parmsByDefault()
700 701
701 702 self.server = server
702 703 self.username = username
703 704 self.password = password
704 705 self.remotefolder = remotefolder
705 706 self.file = None
706 707 self.ftp = None
707 708 self.status = 0
708 709
709 710 try:
710 711 self.ftp = ftplib.FTP(self.server)
711 712 self.ftp.login(self.username,self.password)
712 713 self.ftp.cwd(self.remotefolder)
713 714 # print 'Connect to FTP Server: Successfully'
714 715
715 716 except ftplib.all_errors:
716 717 print 'Error FTP Service'
717 718 self.status = 1
718 719 return
719 720
720 721
721 722
722 723 self.dirList = []
723 724
724 725 try:
725 726 self.dirList = self.ftp.nlst()
726 727
727 728 except ftplib.error_perm, resp:
728 729 if str(resp) == "550 No files found":
729 730 print "no files in this directory"
730 731 self.status = 1
731 732 return
732 733
733 734 except ftplib.all_errors:
734 735 print 'Error Displaying Dir-Files'
735 736 self.status = 1
736 737 return
737 738
738 739 self.fileList = []
739 740 self.folderList = []
740 741 #only for test
741 742 for f in self.dirList:
742 743 name, ext = os.path.splitext(f)
743 744 if ext != '':
744 745 self.fileList.append(f)
745 746 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
746 747
747 748 def parmsByDefault(self):
748 749 server = 'jro-app.igp.gob.pe'
749 750 username = 'wmaster'
750 751 password = 'mst2010vhf'
751 752 remotefolder = '/home/wmaster/graficos'
752 753
753 754 return server, username, password, remotefolder
754 755
755 756
756 757 def mkd(self,dirname):
757 758 """
758 759 mkd is used to make directory in remote server
759 760
760 761 Input:
761 762 dirname - directory name
762 763
763 764 Return:
764 765 1 in error case else 0
765 766 """
766 767 try:
767 768 self.ftp.mkd(dirname)
768 769 except:
769 770 print 'Error creating remote folder:%s'%dirname
770 771 return 1
771 772
772 773 return 0
773 774
774 775
775 776 def delete(self,filename):
776 777 """
777 778 delete is used to delete file in current working directory of remote server
778 779
779 780 Input:
780 781 filename - filename to delete in remote folder
781 782
782 783 Return:
783 784 1 in error case else 0
784 785 """
785 786
786 787 try:
787 788 self.ftp.delete(filename)
788 789 except:
789 790 print 'Error deleting remote file:%s'%filename
790 791 return 1
791 792
792 793 return 0
793 794
794 795 def download(self,filename,localfolder):
795 796 """
796 797 download is used to downloading file from remote folder into local folder
797 798
798 799 Inputs:
799 800 filename - filename to donwload
800 801
801 802 localfolder - directory local to store filename
802 803
803 804 Returns:
804 805 self.status - 1 in error case else 0
805 806 """
806 807
807 808 self.status = 0
808 809
809 810
810 811 if not(filename in self.fileList):
811 812 print 'filename:%s not exists'%filename
812 813 self.status = 1
813 814 return self.status
814 815
815 816 newfilename = os.path.join(localfolder,filename)
816 817
817 818 self.file = open(newfilename, 'wb')
818 819
819 820 try:
820 821 print 'Download: ' + filename
821 822 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
822 823 print 'Download Complete'
823 824 except ftplib.all_errors:
824 825 print 'Error Downloading ' + filename
825 826 self.status = 1
826 827 return self.status
827 828
828 829 self.file.close()
829 830
830 831 return self.status
831 832
832 833
833 834 def __handleDownload(self,block):
834 835 """
835 836 __handleDownload is used to handle writing file
836 837 """
837 838 self.file.write(block)
838 839
839 840
840 841 def upload(self,filename,remotefolder=None):
841 842 """
842 843 upload is used to uploading local file to remote directory
843 844
844 845 Inputs:
845 846 filename - full path name of local file to store in remote directory
846 847
847 848 remotefolder - remote directory
848 849
849 850 Returns:
850 851 self.status - 1 in error case else 0
851 852 """
852 853
853 854 if remotefolder == None:
854 855 remotefolder = self.remotefolder
855 856
856 857 self.status = 0
857 858
858 859 try:
859 860 self.ftp.cwd(remotefolder)
860 861
861 862 self.file = open(filename, 'rb')
862 863
863 864 (head, tail) = os.path.split(filename)
864 865
865 866 command = "STOR " + tail
866 867
867 868 print 'Uploading: ' + tail
868 869 self.ftp.storbinary(command, self.file)
869 870 print 'Upload Completed'
870 871
871 872 except ftplib.all_errors:
872 873 print 'Error Uploading ' + tail
873 874 self.status = 1
874 875 return self.status
875 876
876 877 self.file.close()
877 878
878 879 #back to initial directory in __init__()
879 880 self.ftp.cwd(self.remotefolder)
880 881
881 882 return self.status
882 883
883 884
884 885 def dir(self,remotefolder):
885 886 """
886 887 dir is used to change working directory of remote server and get folder and file list
887 888
888 889 Input:
889 890 remotefolder - current working directory
890 891
891 892 Affects:
892 893 self.fileList - file list of working directory
893 894
894 895 Return:
895 896 infoList - list with filenames and size of file in bytes
896 897
897 898 self.folderList - folder list
898 899 """
899 900
900 901 self.remotefolder = remotefolder
901 902 print 'Change to ' + self.remotefolder
902 903 try:
903 904 self.ftp.cwd(remotefolder)
904 905 except ftplib.all_errors:
905 906 print 'Error Change to ' + self.remotefolder
906 907 infoList = None
907 908 self.folderList = None
908 909 return infoList,self.folderList
909 910
910 911 self.dirList = []
911 912
912 913 try:
913 914 self.dirList = self.ftp.nlst()
914 915
915 916 except ftplib.error_perm, resp:
916 917 if str(resp) == "550 No files found":
917 918 print "no files in this directory"
918 919 infoList = None
919 920 self.folderList = None
920 921 return infoList,self.folderList
921 922 except ftplib.all_errors:
922 923 print 'Error Displaying Dir-Files'
923 924 infoList = None
924 925 self.folderList = None
925 926 return infoList,self.folderList
926 927
927 928 infoList = []
928 929 self.fileList = []
929 930 self.folderList = []
930 931 for f in self.dirList:
931 932 name,ext = os.path.splitext(f)
932 933 if ext != '':
933 934 self.fileList.append(f)
934 935 value = (f,self.ftp.size(f))
935 936 infoList.append(value)
936 937
937 938 if ext == '':
938 939 self.folderList.append(f)
939 940
940 941 return infoList,self.folderList
941 942
942 943
943 944 def close(self):
944 945 """
945 946 close is used to close and end FTP connection
946 947
947 948 Inputs: None
948 949
949 950 Return: void
950 951
951 952 """
952 953 self.ftp.close()
953 954
954 955 class SendByFTP(Operation):
955 956
956 957 def __init__(self):
957 958
958 959 self.status = 1
959 960 self.counter = 0
960 961
961 962 def error_print(self, ValueError):
962 963
963 964 print ValueError, 'Error FTP'
964 965 print "don't worry the program is running..."
965 966
966 967 def worker_ftp(self, server, username, password, remotefolder, filenameList):
967 968
968 969 self.ftpClientObj = FTP(server, username, password, remotefolder)
969 970 for filename in filenameList:
970 971 self.ftpClientObj.upload(filename)
971 972 self.ftpClientObj.close()
972 973
973 974 def ftp_thread(self, server, username, password, remotefolder):
974 975 if not(self.status):
975 976 return
976 977
977 978 import multiprocessing
978 979
979 980 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
980 981 p.start()
981 982
982 983 p.join(3)
983 984
984 985 if p.is_alive():
985 986 p.terminate()
986 987 p.join()
987 988 print 'killing ftp process...'
988 989 self.status = 0
989 990 return
990 991
991 992 self.status = 1
992 993 return
993 994
994 995 def filterByExt(self, ext, localfolder):
995 996 fnameList = glob.glob1(localfolder,ext)
996 997 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
997 998
998 999 if len(self.filenameList) == 0:
999 1000 self.status = 0
1000 1001
1001 1002 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
1002 1003
1003 1004 self.counter += 1
1004 1005 if self.counter >= period:
1005 1006 self.filterByExt(ext, localfolder)
1006 1007
1007 1008 self.ftp_thread(server, username, password, remotefolder)
1008 1009
1009 1010 self.counter = 0
1010 1011
1011 1012 self.status = 1
1012 1013
Show file before | Show file after | Hide comments
@@ -1,97 +1,99
1 1
2 2 import os, sys
3 3
4 4 path = os.path.split(os.getcwd())[0]
5 5 path = os.path.split(path)[0]
6 6
7 7 sys.path.insert(0, path)
8 8
9 9 from schainpy.controller import Project
10 10
11 11 controllerObj = Project()
12 12 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
13 13
14 14 #-------------------------------------- Setup -----------------------------------------
15 15 #Verificar estas variables
16 16
17 17 #Path para los archivos
18 18 # path = '/mnt/jars/2016_08/NOCHE'
19 19 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
20 20 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
21 path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
21 path = '/home/nanosat/data/jasmet'
22 22
23 23 #Path para los graficos
24 24 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
25 25
26 26 #Path para los archivos HDF5 de meteoros
27 27 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
28 28
29 29 #Fechas para busqueda de archivos
30 startDate = '2016/08/29'
31 endDate = '2016/09/11'
30 startDate = '2010/08/29'
31 endDate = '2017/09/11'
32 32 #Horas para busqueda de archivos
33 33 startTime = '00:00:00'
34 34 endTime = '23:59:59'
35 35
36 36
37 37 #------------------------------ Voltage Reading Unit ----------------------------------
38 38
39 39 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
40 40 path=path,
41 41 startDate=startDate,
42 42 endDate=endDate,
43 43 startTime=startTime,
44 44 endTime=endTime,
45 45 online=0,
46 46 delay=30,
47 47 walk=1,
48 48 getblock=1,
49 49 blocktime=100)
50 50
51 51 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
52 52
53 53 #-------------------------- Voltage Processing Unit ------------------------------------
54 54
55 55 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
56 56
57 57 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
58 58 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
59 59
60 60 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
61 61 opObj01.addParameter(name='frequency', value='30.e6', format='float')
62 62
63 opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
64 opObj01.addParameter(name='topLim', value='73', format='int')
65 opObj01.addParameter(name='botLim', value='71', format='int')
63 # opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
64 # opObj01.addParameter(name='topLim', value='73', format='int')
65 # opObj01.addParameter(name='botLim', value='71', format='int')
66 66
67 67 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
68 68
69 69 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
70 70 opObj03.addParameter(name='n', value='2', format='int')
71 71
72 procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId())
73 opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
74 opObj11.addParameter(name='id', value='237', format='int')
75 opObj11.addParameter(name='xmin', value='9.0', format='float')
76 opObj11.addParameter(name='xmax', value='16.0', format='float')
77 opObj11.addParameter(name='zmin', value='15.0', format='float')
78 opObj11.addParameter(name='zmax', value='50.0', format='float')
79
72 80 #--------------------------- Parameters Processing Unit ------------------------------------
73 81
74 procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
75 #
76 opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
77 opObj10.addParameter(name='azimuth', value='45', format='float')
78 opObj10.addParameter(name='hmin', value='60', format='float')
79 opObj10.addParameter(name='hmax', value='120', format='float')
80
81 opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
82 opObj12.addParameter(name='path', value=pathfile)
83 opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
84 opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
85 opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
86 opObj12.addParameter(name='mode',value='2',format='int')
82 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
83 # #
84 # opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
85 # opObj10.addParameter(name='azimuth', value='45', format='float')
86 # opObj10.addParameter(name='hmin', value='60', format='float')
87 # opObj10.addParameter(name='hmax', value='120', format='float')
88
89 # opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
90 # opObj12.addParameter(name='path', value=pathfile)
91 # opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
92 # opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
93 # opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
94 # opObj12.addParameter(name='mode',value='2',format='int')
87 95
88 96 #--------------------------------------------------------------------------------------------------
89
90 print "Escribiendo el archivo XML"
91 controllerObj.writeXml("JASMET02.xml")
92 print "Leyendo el archivo XML"
93 controllerObj.readXml("JASMET02.xml")
94
95 97 controllerObj.createObjects()
96 98 controllerObj.connectObjects()
97 99 controllerObj.run() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now