##// END OF EJS Templates
schain
RSS
Cleaning IO modules (base, voltaje and spectra)
jespinoza -
r1251:6d7191c1eaa6
parent child
Show More
Side by Side Unified
Context file:
r1251:6d7191c1eaa6 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,9 +1,3
1 #from schainpy.model.data.jrodata import *
2 # from schainpy.model.io.jrodataIO import *
3 # from schainpy.model.proc.jroprocessing import *
4 # from schainpy.model.graphics.jroplot import *
5 # from schainpy.model.utils.jroutils import *
6 # from schainpy.serializer import *
7
1
8 from .graphics import *
2 from .graphics import *
9 from .data import *
3 from .data import *
Show file before | Show file after | Hide comments
@@ -181,7 +181,7 class Plot(Operation):
181 self.localtime = kwargs.pop('localtime', True)
181 self.localtime = kwargs.pop('localtime', True)
182 self.show = kwargs.get('show', True)
182 self.show = kwargs.get('show', True)
183 self.save = kwargs.get('save', False)
183 self.save = kwargs.get('save', False)
184 self.save_period = kwargs.get('save_period', 2)
184 self.save_period = kwargs.get('save_period', 1)
185 self.ftp = kwargs.get('ftp', False)
185 self.ftp = kwargs.get('ftp', False)
186 self.colormap = kwargs.get('colormap', self.colormap)
186 self.colormap = kwargs.get('colormap', self.colormap)
187 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
187 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (970 lines changed) Show them Hide them
@@ -1,8 +1,8
1 '''
1 """
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 """
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
@@ -12,7 +12,6 import fnmatch
12 import inspect
12 import inspect
13 import time
13 import time
14 import datetime
14 import datetime
15 import traceback
16 import zmq
15 import zmq
17
16
18 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
17 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
@@ -21,6 +20,17 from schainpy.utils import log
21 import schainpy.admin
20 import schainpy.admin
22
21
23 LOCALTIME = True
22 LOCALTIME = True
23 DT_DIRECTIVES = {
24 '%Y': 4,
25 '%y': 2,
26 '%m': 2,
27 '%d': 2,
28 '%j': 3,
29 '%H': 2,
30 '%M': 2,
31 '%S': 2,
32 '%f': 6
33 }
24
34
25
35
26 def isNumber(cad):
36 def isNumber(cad):
@@ -362,70 +372,6 def getlastFileFromPath(path, ext):
362 return None
372 return None
363
373
364
374
365 def checkForRealPath(path, foldercounter, year, doy, set, ext):
366 """
367 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
368 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
369 el path exacto de un determinado file.
370
371 Example :
372 nombre correcto del file es .../.../D2009307/P2009307367.ext
373
374 Entonces la funcion prueba con las siguientes combinaciones
375 .../.../y2009307367.ext
376 .../.../Y2009307367.ext
377 .../.../x2009307/y2009307367.ext
378 .../.../x2009307/Y2009307367.ext
379 .../.../X2009307/y2009307367.ext
380 .../.../X2009307/Y2009307367.ext
381 siendo para este caso, la ultima combinacion de letras, identica al file buscado
382
383 Return:
384 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
385 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
386 para el filename
387 """
388 fullfilename = None
389 find_flag = False
390 filename = None
391
392 prefixDirList = [None, 'd', 'D']
393 if ext.lower() == ".r": # voltage
394 prefixFileList = ['d', 'D']
395 elif ext.lower() == ".pdata": # spectra
396 prefixFileList = ['p', 'P']
397 else:
398 return None, filename
399
400 # barrido por las combinaciones posibles
401 for prefixDir in prefixDirList:
402 thispath = path
403 if prefixDir != None:
404 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
405 if foldercounter == 0:
406 thispath = os.path.join(path, "%s%04d%03d" %
407 (prefixDir, year, doy))
408 else:
409 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
410 prefixDir, year, doy, foldercounter))
411 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
412 # formo el nombre del file xYYYYDDDSSS.ext
413 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
414 fullfilename = os.path.join(
415 thispath, filename) # formo el path completo
416
417 if os.path.exists(fullfilename): # verifico que exista
418 find_flag = True
419 break
420 if find_flag:
421 break
422
423 if not(find_flag):
424 return None, filename
425
426 return fullfilename, filename
427
428
429 def isRadarFolder(folder):
375 def isRadarFolder(folder):
430 try:
376 try:
431 year = int(folder[1:5])
377 year = int(folder[1:5])
@@ -469,443 +415,366 def getDateFromRadarFolder(folder):
469 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
415 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
470 return thisDate
416 return thisDate
471
417
418 def parse_format(s, fmt):
472
419
473 class JRODataIO:
420 for i in range(fmt.count('%')):
421 x = fmt.index('%')
422 d = DT_DIRECTIVES[fmt[x:x+2]]
423 fmt = fmt.replace(fmt[x:x+2], s[x:x+d])
424 return fmt
474
425
475 c = 3E8
426 class Reader(object):
476
427
428 c = 3E8
477 isConfig = False
429 isConfig = False
478
479 basicHeaderObj = None
480
481 systemHeaderObj = None
482
483 radarControllerHeaderObj = None
484
485 processingHeaderObj = None
486
487 dtype = None
430 dtype = None
488
489 pathList = []
431 pathList = []
490
491 filenameList = []
432 filenameList = []
492
433 datetimeList = []
493 filename = None
434 filename = None
494
495 ext = None
435 ext = None
496
497 flagIsNewFile = 1
436 flagIsNewFile = 1
498
499 flagDiscontinuousBlock = 0
437 flagDiscontinuousBlock = 0
500
501 flagIsNewBlock = 0
438 flagIsNewBlock = 0
502
439 flagNoMoreFiles = 0
503 fp = None
440 fp = None
504
505 firstHeaderSize = 0
441 firstHeaderSize = 0
506
507 basicHeaderSize = 24
442 basicHeaderSize = 24
508
509 versionFile = 1103
443 versionFile = 1103
510
511 fileSize = None
444 fileSize = None
512
513 # ippSeconds = None
514
515 fileSizeByHeader = None
445 fileSizeByHeader = None
516
446 fileIndex = -1
517 fileIndex = None
518
519 profileIndex = None
447 profileIndex = None
520
448 blockIndex = 0
521 blockIndex = None
449 nTotalBlocks = 0
522
523 nTotalBlocks = None
524
525 maxTimeStep = 30
450 maxTimeStep = 30
526
527 lastUTTime = None
451 lastUTTime = None
528
529 datablock = None
452 datablock = None
530
531 dataOut = None
453 dataOut = None
532
533 blocksize = None
534
535 getByBlock = False
454 getByBlock = False
536
455 path = None
537 def __init__(self):
456 startDate = None
538
457 endDate = None
539 raise NotImplementedError
458 startTime = datetime.time(0, 0, 0)
459 endTime = datetime.time(23, 59, 59)
460 set = None
461 expLabel = ""
462 online = False
463 delay = 60
464 nTries = 3 # quantity tries
465 nFiles = 3 # number of files for searching
466 walk = True
467 getblock = False
468 nTxs = 1
469 realtime = False
470 blocksize = 0
471 blocktime = None
472 warnings = True
473 verbose = True
474 server = None
475 format = None
476 oneDDict = None
477 twoDDict = None
478 independentParam = None
479 filefmt = None
480 folderfmt = None
540
481
541 def run(self):
482 def run(self):
542
483
543 raise NotImplementedError
484 raise NotImplementedError
544
485
545 def getDtypeWidth(self):
546
547 dtype_index = get_dtype_index(self.dtype)
548 dtype_width = get_dtype_width(dtype_index)
549
550 return dtype_width
551
552 def getAllowedArgs(self):
486 def getAllowedArgs(self):
553 if hasattr(self, '__attrs__'):
487 if hasattr(self, '__attrs__'):
554 return self.__attrs__
488 return self.__attrs__
555 else:
489 else:
556 return inspect.getargspec(self.run).args
490 return inspect.getargspec(self.run).args
557
491
492 def set_kwargs(self, **kwargs):
558
493
559 class JRODataReader(JRODataIO):
494 for key, value in kwargs.items():
495 setattr(self, key, value)
560
496
561 online = 0
497 def find_folders(self, path, startDate, endDate, folderfmt, last=False):
562
498
563 realtime = 0
499 folders = [x for f in path.split(',')
500 for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
501 folders.sort()
564
502
565 nReadBlocks = 0
503 if last:
566
504 folders = [folders[-1]]
567 delay = 10 # number of seconds waiting a new file
568
569 nTries = 3 # quantity tries
570
571 nFiles = 3 # number of files for searching
572
505
573 path = None
506 for folder in folders:
574
507 try:
575 foldercounter = 0
508 dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
576
509 if dt >= startDate and dt <= endDate:
577 flagNoMoreFiles = 0
510 yield os.path.join(path, folder)
578
511 else:
579 datetimeList = []
512 log.log('Skiping folder {}'.format(folder), self.name)
580
513 except Exception as e:
581 __isFirstTimeOnline = 1
514 log.log('Skiping folder {}'.format(folder), self.name)
582
515 continue
583 __printInfo = True
516 return
584
585 profileIndex = None
586
587 nTxs = 1
588
589 txIndex = None
590
591 # Added--------------------
592
593 selBlocksize = None
594
595 selBlocktime = None
596
517
597 def __init__(self):
518 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
598 """
519 expLabel='', last=False):
599 This class is used to find data files
600
520
601 Example:
521 for path in folders:
602 reader = JRODataReader()
522 files = glob.glob1(path, '*{}'.format(ext))
603 fileList = reader.findDataFiles()
523 files.sort()
604
524 if last:
605 """
525 if files:
526 fo = files[-1]
527 try:
528 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
529 yield os.path.join(path, expLabel, fo)
530 except Exception as e:
606 pass
531 pass
607
532 return None
608 def createObjByDefault(self):
609 """
610
611 """
612 raise NotImplementedError
613
614 def getBlockDimension(self):
615
616 raise NotImplementedError
617
618 def searchFilesOffLine(self,
619 path,
620 startDate=None,
621 endDate=None,
622 startTime=datetime.time(0, 0, 0),
623 endTime=datetime.time(23, 59, 59),
624 set=None,
625 expLabel='',
626 ext='.r',
627 cursor=None,
628 skip=None,
629 walk=True):
630
631 self.filenameList = []
632 self.datetimeList = []
633
634 pathList = []
635
636 dateList, pathList = self.findDatafiles(
637 path, startDate, endDate, expLabel, ext, walk, include_path=True)
638
639 if dateList == []:
640 return [], []
641
642 if len(dateList) > 1:
643 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
644 else:
533 else:
645 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
534 return None
646
647 filenameList = []
648 datetimeList = []
649
650 for thisPath in pathList:
651
652 fileList = glob.glob1(thisPath, "*%s" % ext)
653 fileList.sort()
654
655 for file in fileList:
656
657 filename = os.path.join(thisPath, file)
658
659 if not isFileInDateRange(filename, startDate, endDate):
660 continue
661
662 thisDatetime = isFileInTimeRange(
663 filename, startDate, endDate, startTime, endTime)
664
535
665 if not(thisDatetime):
536 for fo in files:
537 try:
538 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
539 if dt >= startDate and dt <= endDate:
540 yield os.path.join(path, expLabel, fo)
541 else:
542 log.log('Skiping file {}'.format(fo), self.name)
543 except Exception as e:
544 log.log('Skiping file {}'.format(fo), self.name)
666 continue
545 continue
667
546
668 filenameList.append(filename)
547 def searchFilesOffLine(self, path, startDate, endDate,
669 datetimeList.append(thisDatetime)
548 expLabel, ext, walk,
670
549 filefmt, folderfmt):
671 if cursor is not None and skip is not None:
550 """Search files in offline mode for the given arguments
672 filenameList = filenameList[cursor * skip:cursor * skip + skip]
673 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
674
551
675 if not(filenameList):
552 Return:
676 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
553 Generator of files
677 return [], []
554 """
678
679 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
680
681 # for i in range(len(filenameList)):
682 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
683
555
684 self.filenameList = filenameList
556 if walk:
685 self.datetimeList = datetimeList
557 folders = self.find_folders(
558 path, startDate, endDate, folderfmt)
559 else:
560 folders = path.split(',')
686
561
687 return pathList, filenameList
562 return self.find_files(
563 folders, ext, filefmt, startDate, endDate, expLabel)
688
564
689 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
565 def searchFilesOnLine(self, path, startDate, endDate,
690 """
566 expLabel, ext, walk,
691 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
567 filefmt, folderfmt):
692 devuelve el archivo encontrado ademas de otros datos.
568 """Search for the last file of the last folder
693
569
694 Input:
570 Arguments:
695 path : carpeta donde estan contenidos los files que contiene data
571 path : carpeta donde estan contenidos los files que contiene data
696
697 expLabel : Nombre del subexperimento (subfolder)
572 expLabel : Nombre del subexperimento (subfolder)
698
699 ext : extension de los files
573 ext : extension de los files
700
701 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
574 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
702
575
703 Return:
576 Return:
704 directory : eL directorio donde esta el file encontrado
577 generator with the full path of last filename
705 filename : el ultimo file de una determinada carpeta
706 year : el anho
707 doy : el numero de dia del anho
708 set : el set del archivo
709
710
711 """
578 """
712 if not os.path.isdir(path):
713 return None, None, None, None, None, None
714
715 dirList = []
716
579
717 if not walk:
580 if walk:
718 fullpath = path
581 folders = self.find_folders(
719 foldercounter = 0
582 path, startDate, endDate, folderfmt, last=True)
720 else:
583 else:
721 # Filtra solo los directorios
584 folders = path.split(',')
722 for thisPath in os.listdir(path):
723 if not os.path.isdir(os.path.join(path, thisPath)):
724 continue
725 if not isRadarFolder(thisPath):
726 continue
727
585
728 dirList.append(thisPath)
586 return self.find_files(
729
587 folders, ext, filefmt, startDate, endDate, expLabel, last=True)
730 if not(dirList):
731 return None, None, None, None, None, None
732
733 dirList = sorted(dirList, key=str.lower)
734
588
735 doypath = dirList[-1]
589 def setNextFile(self):
736 foldercounter = int(doypath.split('_')[1]) if len(
590 """Set the next file to be readed open it and parse de file header"""
737 doypath.split('_')) > 1 else 0
738 fullpath = os.path.join(path, doypath, expLabel)
739
591
740 print("[Reading] %s folder was found: " % (fullpath))
592 if self.fp != None:
593 self.fp.close()
741
594
742 if set == None:
595 if self.online:
743 filename = getlastFileFromPath(fullpath, ext)
596 newFile = self.setNextFileOnline()
744 else:
597 else:
745 filename = getFileFromSet(fullpath, ext, set)
598 newFile = self.setNextFileOffline()
746
747 if not(filename):
748 return None, None, None, None, None, None
749
599
750 print("[Reading] %s file was found" % (filename))
600 if not(newFile):
601 if self.online:
602 raise schainpy.admin.SchainError('Time to wait for new files reach')
603 else:
604 if self.fileIndex == -1:
605 raise schainpy.admin.SchainWarning('No files found in the given path')
606 else:
607 raise schainpy.admin.SchainWarning('No more files to read')
751
608
752 if not(self.__verifyFile(os.path.join(fullpath, filename))):
609 if not(self.verifyFile(self.filename)):
753 return None, None, None, None, None, None
610 self.setNextFile()
754
611
755 year = int(filename[1:5])
612 log.log('Opening file: %s' % self.filename, self.name)
756 doy = int(filename[5:8])
757 set = int(filename[8:11])
758
613
759 return fullpath, foldercounter, filename, year, doy, set
614 self.readFirstHeader()
615 self.nReadBlocks = 0
760
616
761 def __setNextFileOffline(self):
617 def setNextFileOnline(self):
618 """Check for the next file to be readed in online mode.
762
619
763 idFile = self.fileIndex
620 Set:
621 self.filename
622 self.fp
623 self.filesize
764
624
765 while (True):
625 Return:
766 idFile += 1
626 boolean
767 if not(idFile < len(self.filenameList)):
768 self.flagNoMoreFiles = 1
769 return 0
770
627
771 filename = self.filenameList[idFile]
628 """
629 nextFile = True
630 nextDay = False
772
631
773 if not(self.__verifyFile(filename)):
632 for nFiles in range(self.nFiles+1):
633 for nTries in range(self.nTries):
634 fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
635 if fullfilename is not None:
636 break
637 log.warning(
638 "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1),
639 self.name)
640 time.sleep(self.delay)
641 nextFile = False
774 continue
642 continue
775
643
776 fileSize = os.path.getsize(filename)
644 if fullfilename:
777 fp = open(filename, 'rb')
778 break
645 break
779
646
647 self.nTries = 1
648 nextFile = True
649
650 if nFiles == (self.nFiles - 1):
651 log.log('Trying with next day...', self.name)
652 nextDay = True
653
654 if fullfilename:
655 self.fileSize = os.path.getsize(fullfilename)
656 self.filename = fullfilename
780 self.flagIsNewFile = 1
657 self.flagIsNewFile = 1
781 self.fileIndex = idFile
658 if self.fp != None:
782 self.filename = filename
659 self.fp.close()
783 self.fileSize = fileSize
660 self.fp = open(fullfilename, 'rb')
784 self.fp = fp
661 self.flagNoMoreFiles = 0
662 self.fileIndex += 1
663 return 1
664 else:
665 return 0
666
667 def setNextFileOffline(self):
668 """Open the next file to be readed in offline mode"""
669
670 try:
671 filename = next(self.filenameList)
672 self.fileIndex +=1
673 except StopIteration:
674 self.flagNoMoreFiles = 1
675 return 0
785
676
786 # print "[Reading] Setting the file: %s"%self.filename
677 self.filename = filename
678 self.fileSize = os.path.getsize(filename)
679 self.fp = open(filename, 'rb')
680 self.flagIsNewFile = 1
787
681
788 return 1
682 return 1
789
683
790 def __setNextFileOnline(self):
684 def verifyFile(self, filename):
791 """
685 """Check for a valid file
792 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
793 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
794 siguientes.
795
686
796 Affected:
687 Arguments:
797 self.flagIsNewFile
688 filename -- full path filename
798 self.filename
799 self.fileSize
800 self.fp
801 self.set
802 self.flagNoMoreFiles
803
689
804 Return:
690 Return:
805 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
691 boolean
806 1 : si el file fue abierto con exito y esta listo a ser leido
807
808 Excepciones:
809 Si un determinado file no puede ser abierto
810 """
692 """
811 nFiles = 0
812 fileOk_flag = False
813 firstTime_flag = True
814
693
815 self.set += 1
694 return True
816
695
817 if self.set > 999:
696 def checkForRealPath(self, nextFile, nextDay):
818 self.set = 0
697 """Check if the next file to be readed exists"""
819 self.foldercounter += 1
820
698
821 # busca el 1er file disponible
699 raise NotImplementedError
822 fullfilename, filename = checkForRealPath(
823 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
824 if fullfilename:
825 if self.__verifyFile(fullfilename, False):
826 fileOk_flag = True
827
700
828 # si no encuentra un file entonces espera y vuelve a buscar
701 def readFirstHeader(self):
829 if not(fileOk_flag):
702 """Parse the file header"""
830 # busco en los siguientes self.nFiles+1 files posibles
831 for nFiles in range(self.nFiles + 1):
832
703
833 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
704 pass
834 tries = self.nTries
835 else:
836 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
837
705
838 for nTries in range(tries):
706 class JRODataReader(Reader):
839 if firstTime_flag:
840 log.warning(
841 "Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1),
842 self.name)
843 time.sleep(self.delay)
844 else:
845 log.warning(
846 "Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext),
847 self.name)
848
707
849 fullfilename, filename = checkForRealPath(
708 utc = 0
850 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
709 nReadBlocks = 0
851 if fullfilename:
710 foldercounter = 0
852 if self.__verifyFile(fullfilename):
711 firstHeaderSize = 0
853 fileOk_flag = True
712 basicHeaderSize = 24
854 break
713 __isFirstTimeOnline = 1
714 __printInfo = True
715 filefmt = "*%Y%j***"
716 folderfmt = "*%Y%j"
855
717
856 if fileOk_flag:
718 def getDtypeWidth(self):
857 break
858
719
859 firstTime_flag = False
720 dtype_index = get_dtype_index(self.dtype)
721 dtype_width = get_dtype_width(dtype_index)
860
722
861 log.warning(
723 return dtype_width
862 'Skipping the file {} due to this file doesn\'t exist'.format(filename),
863 self.name)
864 self.set += 1
865
724
866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
725 def checkForRealPath(self, nextFile, nextDay):
867 if nFiles == (self.nFiles - 1):
726 """Check if the next file to be readed exists.
868 self.set = 0
869 self.doy += 1
870 self.foldercounter = 0
871
727
872 if fileOk_flag:
728 Example :
873 self.fileSize = os.path.getsize(fullfilename)
729 nombre correcto del file es .../.../D2009307/P2009307367.ext
874 self.filename = fullfilename
875 self.flagIsNewFile = 1
876 if self.fp != None:
877 self.fp.close()
878 self.fp = open(fullfilename, 'rb')
879 self.flagNoMoreFiles = 0
880 else:
881 raise schainpy.admin.SchainError('Time for waiting new files reach')
882 self.fileSize = 0
883 self.filename = None
884 self.flagIsNewFile = 0
885 self.fp = None
886 self.flagNoMoreFiles = 1
887
730
888 return fileOk_flag
731 Entonces la funcion prueba con las siguientes combinaciones
732 .../.../y2009307367.ext
733 .../.../Y2009307367.ext
734 .../.../x2009307/y2009307367.ext
735 .../.../x2009307/Y2009307367.ext
736 .../.../X2009307/y2009307367.ext
737 .../.../X2009307/Y2009307367.ext
738 siendo para este caso, la ultima combinacion de letras, identica al file buscado
889
739
890 def setNextFile(self):
740 Return:
891 if self.fp != None:
741 str -- fullpath of the file
892 self.fp.close()
742 """
893
743
894 if self.online:
895 newFile = self.__setNextFileOnline()
896 else:
897 newFile = self.__setNextFileOffline()
898
744
899 if not(newFile):
745 if nextFile:
900 raise schainpy.admin.SchainWarning('No more files to read')
746 self.set += 1
747 if nextDay:
748 self.set = 0
749 self.doy += 1
750 foldercounter = 0
751 prefixDirList = [None, 'd', 'D']
752 if self.ext.lower() == ".r": # voltage
753 prefixFileList = ['d', 'D']
754 elif self.ext.lower() == ".pdata": # spectra
755 prefixFileList = ['p', 'P']
901
756
757 # barrido por las combinaciones posibles
758 for prefixDir in prefixDirList:
759 thispath = self.path
760 if prefixDir != None:
761 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
762 if foldercounter == 0:
763 thispath = os.path.join(self.path, "%s%04d%03d" %
764 (prefixDir, self.year, self.doy))
765 else:
766 thispath = os.path.join(self.path, "%s%04d%03d_%02d" % (
767 prefixDir, self.year, self.doy, foldercounter))
768 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
769 # formo el nombre del file xYYYYDDDSSS.ext
770 filename = "%s%04d%03d%03d%s" % (prefixFile, self.year, self.doy, self.set, self.ext)
771 fullfilename = os.path.join(
772 thispath, filename)
902
773
903 if self.verbose:
774 if os.path.exists(fullfilename):
904 print('[Reading] Setting the file: %s' % self.filename)
775 return fullfilename, filename
905
776
906 self.__readFirstHeader()
777 return None, filename
907 self.nReadBlocks = 0
908 return 1
909
778
910 def __waitNewBlock(self):
779 def __waitNewBlock(self):
911 """
780 """
@@ -972,47 +841,11 class JRODataReader(JRODataIO):
972
841
973 return 0
842 return 0
974
843
975 def __jumpToLastBlock(self):
976
977 if not(self.__isFirstTimeOnline):
978 return
979
980 csize = self.fileSize - self.fp.tell()
981 blocksize = self.processingHeaderObj.blockSize
982
983 # salta el primer bloque de datos
984 if csize > self.processingHeaderObj.blockSize:
985 self.fp.seek(self.fp.tell() + blocksize)
986 else:
987 return
988
989 csize = self.fileSize - self.fp.tell()
990 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
991 while True:
992
993 if self.fp.tell() < self.fileSize:
994 self.fp.seek(self.fp.tell() + neededsize)
995 else:
996 self.fp.seek(self.fp.tell() - neededsize)
997 break
998
999 # csize = self.fileSize - self.fp.tell()
1000 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 # factor = int(csize/neededsize)
1002 # if factor > 0:
1003 # self.fp.seek(self.fp.tell() + factor*neededsize)
1004
1005 self.flagIsNewFile = 0
1006 self.__isFirstTimeOnline = 0
1007
1008 def __setNewBlock(self):
844 def __setNewBlock(self):
1009 # if self.server is None:
845
1010 if self.fp == None:
846 if self.fp == None:
1011 return 0
847 return 0
1012
848
1013 # if self.online:
1014 # self.__jumpToLastBlock()
1015
1016 if self.flagIsNewFile:
849 if self.flagIsNewFile:
1017 self.lastUTTime = self.basicHeaderObj.utc
850 self.lastUTTime = self.basicHeaderObj.utc
1018 return 1
851 return 1
@@ -1023,21 +856,19 class JRODataReader(JRODataIO):
1023 return 0
856 return 0
1024 else:
857 else:
1025 return 1
858 return 1
1026 # if self.server is None:
859
1027 currentSize = self.fileSize - self.fp.tell()
860 currentSize = self.fileSize - self.fp.tell()
1028 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
861 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
862
1029 if (currentSize >= neededSize):
863 if (currentSize >= neededSize):
1030 self.basicHeaderObj.read(self.fp)
864 self.basicHeaderObj.read(self.fp)
1031 self.lastUTTime = self.basicHeaderObj.utc
865 self.lastUTTime = self.basicHeaderObj.utc
1032 return 1
866 return 1
1033 # else:
867
1034 # self.basicHeaderObj.read(self.zHeader)
1035 # self.lastUTTime = self.basicHeaderObj.utc
1036 # return 1
1037 if self.__waitNewBlock():
868 if self.__waitNewBlock():
1038 self.lastUTTime = self.basicHeaderObj.utc
869 self.lastUTTime = self.basicHeaderObj.utc
1039 return 1
870 return 1
1040 # if self.server is None:
871
1041 if not(self.setNextFile()):
872 if not(self.setNextFile()):
1042 return 0
873 return 0
1043
874
@@ -1053,10 +884,8 class JRODataReader(JRODataIO):
1053
884
1054 def readNextBlock(self):
885 def readNextBlock(self):
1055
886
1056 # Skip block out of startTime and endTime
1057 while True:
887 while True:
1058 if not(self.__setNewBlock()):
888 self.__setNewBlock()
1059 raise schainpy.admin.SchainWarning('No more files to read')
1060
889
1061 if not(self.readBlock()):
890 if not(self.readBlock()):
1062 return 0
891 return 0
@@ -1076,13 +905,12 class JRODataReader(JRODataIO):
1076 self.dataOut.datatime.ctime()))
905 self.dataOut.datatime.ctime()))
1077 return 1
906 return 1
1078
907
1079 def __readFirstHeader(self):
908 def readFirstHeader(self):
1080
909
1081 self.basicHeaderObj.read(self.fp)
910 self.basicHeaderObj.read(self.fp)
1082 self.systemHeaderObj.read(self.fp)
911 self.systemHeaderObj.read(self.fp)
1083 self.radarControllerHeaderObj.read(self.fp)
912 self.radarControllerHeaderObj.read(self.fp)
1084 self.processingHeaderObj.read(self.fp)
913 self.processingHeaderObj.read(self.fp)
1085
1086 self.firstHeaderSize = self.basicHeaderObj.size
914 self.firstHeaderSize = self.basicHeaderObj.size
1087
915
1088 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
916 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
@@ -1111,7 +939,7 class JRODataReader(JRODataIO):
1111 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
939 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1112 self.getBlockDimension()
940 self.getBlockDimension()
1113
941
1114 def __verifyFile(self, filename, msgFlag=True):
942 def verifyFile(self, filename, msgFlag=True):
1115
943
1116 msg = None
944 msg = None
1117
945
@@ -1268,48 +1096,17 class JRODataReader(JRODataIO):
1268
1096
1269 return dateList
1097 return dateList
1270
1098
1271 def setup(self,
1099 def setup(self, **kwargs):
1272 path=None,
1100
1273 startDate=None,
1101 self.set_kwargs(**kwargs)
1274 endDate=None,
1102 if not self.ext.startswith('.'):
1275 startTime=datetime.time(0, 0, 0),
1103 self.ext = '.{}'.format(self.ext)
1276 endTime=datetime.time(23, 59, 59),
1104
1277 set=None,
1105 if self.server is not None:
1278 expLabel="",
1106 if 'tcp://' in self.server:
1279 ext=None,
1280 online=False,
1281 delay=60,
1282 walk=True,
1283 getblock=False,
1284 nTxs=1,
1285 realtime=False,
1286 blocksize=None,
1287 blocktime=None,
1288 skip=None,
1289 cursor=None,
1290 warnings=True,
1291 verbose=True,
1292 server=None,
1293 format=None,
1294 oneDDict=None,
1295 twoDDict=None,
1296 independentParam=None):
1297
1298 self.online = online
1299 self.realtime = realtime
1300 self.delay = delay
1301 self.getByBlock = getblock
1302 self.nTxs = nTxs
1303 self.startTime = startTime
1304 self.endTime = endTime
1305 self.endDate = endDate
1306 self.startDate = startDate
1307
1308 if server is not None:
1309 if 'tcp://' in server:
1310 address = server
1107 address = server
1311 else:
1108 else:
1312 address = 'ipc:///tmp/%s' % server
1109 address = 'ipc:///tmp/%s' % self.server
1313 self.server = address
1110 self.server = address
1314 self.context = zmq.Context()
1111 self.context = zmq.Context()
1315 self.receiver = self.context.socket(zmq.PULL)
1112 self.receiver = self.context.socket(zmq.PULL)
@@ -1318,84 +1115,46 class JRODataReader(JRODataIO):
1318 print('[Starting] ReceiverData from {}'.format(self.server))
1115 print('[Starting] ReceiverData from {}'.format(self.server))
1319 else:
1116 else:
1320 self.server = None
1117 self.server = None
1321 if path == None:
1118 if self.path == None:
1322 raise ValueError("[Reading] The path is not valid")
1119 raise ValueError("[Reading] The path is not valid")
1323
1120
1324 if ext == None:
1121 if self.online:
1325 ext = self.ext
1122 log.log("[Reading] Searching files in online mode...", self.name)
1326
1327 if online:
1328 print("[Reading] Searching files in online mode...")
1329
1123
1330 for nTries in range(self.nTries):
1124 for nTries in range(self.nTries):
1331 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1125 fullpath = self.searchFilesOnLine(self.path, self.startDate,
1332 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1126 self.endDate, self.expLabel, self.ext, self.walk,
1127 self.filefmt, self.folderfmt)
1128
1129 try:
1130 fullpath = next(fullpath)
1131 except:
1132 fullpath = None
1333
1133
1334 if fullpath:
1134 if fullpath:
1335 break
1135 break
1336
1136
1337 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1137 log.warning(
1138 'Waiting {} sec for a valid file in {}: try {} ...'.format(
1139 self.delay, self.path, nTries + 1),
1140 self.name)
1338 time.sleep(self.delay)
1141 time.sleep(self.delay)
1339
1142
1340 if not(fullpath):
1143 if not(fullpath):
1341 raise schainpy.admin.SchainError('There isn\'t any valid file in {}'.format(path))
1144 raise schainpy.admin.SchainError(
1342 return
1145 'There isn\'t any valid file in {}'.format(self.path))
1343
1146
1344 self.year = year
1147 pathname, filename = os.path.split(fullpath)
1345 self.doy = doy
1148 self.year = int(filename[1:5])
1346 self.set = set - 1
1149 self.doy = int(filename[5:8])
1347 self.path = path
1150 self.set = int(filename[8:11]) - 1
1348 self.foldercounter = foldercounter
1349 last_set = None
1350 else:
1351 print("[Reading] Searching files in offline mode ...")
1352 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1353 startTime=startTime, endTime=endTime,
1354 set=set, expLabel=expLabel, ext=ext,
1355 walk=walk, cursor=cursor,
1356 skip=skip)
1357
1358 if not(pathList):
1359 self.fileIndex = -1
1360 self.pathList = []
1361 self.filenameList = []
1362 return
1363
1364 self.fileIndex = -1
1365 self.pathList = pathList
1366 self.filenameList = filenameList
1367 file_name = os.path.basename(filenameList[-1])
1368 basename, ext = os.path.splitext(file_name)
1369 last_set = int(basename[-3:])
1370
1371
1372 ext = ext.lower()
1373 self.ext = ext
1374
1375 # Added-----------------
1376 self.selBlocksize = blocksize
1377 self.selBlocktime = blocktime
1378
1379 # Verbose-----------
1380 self.verbose = verbose
1381 self.warnings = warnings
1382
1383 if not(self.setNextFile()):
1384 if (startDate != None) and (endDate != None):
1385 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1386 elif startDate != None:
1387 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1388 else:
1151 else:
1389 print("[Reading] No files")
1152 log.log("Searching files in {}".format(self.path), self.name)
1153 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
1154 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
1390
1155
1391 self.fileIndex = -1
1156 self.setNextFile()
1392 self.pathList = []
1393 self.filenameList = []
1394 return
1395
1157
1396 if last_set != None:
1397 self.dataOut.last_block = last_set * \
1398 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1399 return
1158 return
1400
1159
1401 def getBasicHeader(self):
1160 def getBasicHeader(self):
@@ -1465,85 +1224,57 class JRODataReader(JRODataIO):
1465
1224
1466 self.__printInfo = False
1225 self.__printInfo = False
1467
1226
1468 def run(self,
1227 def run(self, **kwargs):
1469 path=None,
1228 """
1470 startDate=None,
1229
1471 endDate=None,
1230 Arguments:
1472 startTime=datetime.time(0, 0, 0),
1231 path :
1473 endTime=datetime.time(23, 59, 59),
1232 startDate :
1474 set=None,
1233 endDate :
1475 expLabel="",
1234 startTime :
1476 ext=None,
1235 endTime :
1477 online=False,
1236 set :
1478 delay=60,
1237 expLabel :
1479 walk=True,
1238 ext :
1480 getblock=False,
1239 online :
1481 nTxs=1,
1240 delay :
1482 realtime=False,
1241 walk :
1483 blocksize=None,
1242 getblock :
1484 blocktime=None,
1243 nTxs :
1485 skip=None,
1244 realtime :
1486 cursor=None,
1245 blocksize :
1487 warnings=True,
1246 blocktime :
1488 server=None,
1247 skip :
1489 verbose=True,
1248 cursor :
1490 format=None,
1249 warnings :
1491 oneDDict=None,
1250 server :
1492 twoDDict=None,
1251 verbose :
1493 independentParam=None, **kwargs):
1252 format :
1253 oneDDict :
1254 twoDDict :
1255 independentParam :
1256 """
1494
1257
1495 if not(self.isConfig):
1258 if not(self.isConfig):
1496 self.setup(path=path,
1259 self.setup(**kwargs)
1497 startDate=startDate,
1498 endDate=endDate,
1499 startTime=startTime,
1500 endTime=endTime,
1501 set=set,
1502 expLabel=expLabel,
1503 ext=ext,
1504 online=online,
1505 delay=delay,
1506 walk=walk,
1507 getblock=getblock,
1508 nTxs=nTxs,
1509 realtime=realtime,
1510 blocksize=blocksize,
1511 blocktime=blocktime,
1512 skip=skip,
1513 cursor=cursor,
1514 warnings=warnings,
1515 server=server,
1516 verbose=verbose,
1517 format=format,
1518 oneDDict=oneDDict,
1519 twoDDict=twoDDict,
1520 independentParam=independentParam)
1521 self.isConfig = True
1260 self.isConfig = True
1522 if server is None:
1261 if self.server is None:
1523 self.getData()
1262 self.getData()
1524 else:
1263 else:
1525 self.getFromServer()
1264 self.getFromServer()
1526
1265
1527
1266
1528 class JRODataWriter(JRODataIO):
1267 class JRODataWriter(Reader):
1529
1268
1530 """
1269 """
1531 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1270 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1532 de los datos siempre se realiza por bloques.
1271 de los datos siempre se realiza por bloques.
1533 """
1272 """
1534
1273
1535 blockIndex = 0
1536
1537 path = None
1538
1539 setFile = None
1274 setFile = None
1540
1541 profilesPerBlock = None
1275 profilesPerBlock = None
1542
1543 blocksPerFile = None
1276 blocksPerFile = None
1544
1545 nWriteBlocks = 0
1277 nWriteBlocks = 0
1546
1547 fileDate = None
1278 fileDate = None
1548
1279
1549 def __init__(self, dataOut=None):
1280 def __init__(self, dataOut=None):
@@ -1561,6 +1292,13 class JRODataWriter(JRODataIO):
1561 def putData(self):
1292 def putData(self):
1562 raise NotImplementedError
1293 raise NotImplementedError
1563
1294
1295 def getDtypeWidth(self):
1296
1297 dtype_index = get_dtype_index(self.dtype)
1298 dtype_width = get_dtype_width(dtype_index)
1299
1300 return dtype_width
1301
1564 def getProcessFlags(self):
1302 def getProcessFlags(self):
1565
1303
1566 processFlags = 0
1304 processFlags = 0
@@ -1688,8 +1426,7 class JRODataWriter(JRODataIO):
1688 return 1
1426 return 1
1689
1427
1690 def setNextFile(self):
1428 def setNextFile(self):
1691 """
1429 """Determina el siguiente file que sera escrito
1692 Determina el siguiente file que sera escrito
1693
1430
1694 Affected:
1431 Affected:
1695 self.filename
1432 self.filename
@@ -1708,6 +1445,9 class JRODataWriter(JRODataIO):
1708 if self.fp != None:
1445 if self.fp != None:
1709 self.fp.close()
1446 self.fp.close()
1710
1447
1448 if not os.path.exists(path):
1449 os.mkdir(path)
1450
1711 timeTuple = time.localtime(self.dataOut.utctime)
1451 timeTuple = time.localtime(self.dataOut.utctime)
1712 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1452 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1713
1453
@@ -1748,15 +1488,12 class JRODataWriter(JRODataIO):
1748 fp = open(filename, 'wb')
1488 fp = open(filename, 'wb')
1749
1489
1750 self.blockIndex = 0
1490 self.blockIndex = 0
1751
1752 # guardando atributos
1753 self.filename = filename
1491 self.filename = filename
1754 self.subfolder = subfolder
1492 self.subfolder = subfolder
1755 self.fp = fp
1493 self.fp = fp
1756 self.setFile = setFile
1494 self.setFile = setFile
1757 self.flagIsNewFile = 1
1495 self.flagIsNewFile = 1
1758 self.fileDate = self.dataOut.datatime.date()
1496 self.fileDate = self.dataOut.datatime.date()
1759
1760 self.setFirstHeader()
1497 self.setFirstHeader()
1761
1498
1762 print('[Writing] Opening file: %s' % self.filename)
1499 print('[Writing] Opening file: %s' % self.filename)
@@ -1799,12 +1536,9 class JRODataWriter(JRODataIO):
1799 self.setFile = set - 1
1536 self.setFile = set - 1
1800
1537
1801 self.blocksPerFile = blocksPerFile
1538 self.blocksPerFile = blocksPerFile
1802
1803 self.profilesPerBlock = profilesPerBlock
1539 self.profilesPerBlock = profilesPerBlock
1804
1805 self.dataOut = dataOut
1540 self.dataOut = dataOut
1806 self.fileDate = self.dataOut.datatime.date()
1541 self.fileDate = self.dataOut.datatime.date()
1807 # By default
1808 self.dtype = self.dataOut.dtype
1542 self.dtype = self.dataOut.dtype
1809
1543
1810 if datatype is not None:
1544 if datatype is not None:
Show file before | Show file after | Hide comments
@@ -53,24 +53,6 class SpectraReader(JRODataReader, ProcessingUnit):
53
53
54 """
54 """
55
55
56 pts2read_SelfSpectra = 0
57
58 pts2read_CrossSpectra = 0
59
60 pts2read_DCchannels = 0
61
62 ext = ".pdata"
63
64 optchar = "P"
65
66 dataOut = None
67
68 nRdChannels = None
69
70 nRdPairs = None
71
72 rdPairList = []
73
74 def __init__(self):#, **kwargs):
56 def __init__(self):#, **kwargs):
75 """
57 """
76 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
58 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
@@ -89,89 +71,24 class SpectraReader(JRODataReader, ProcessingUnit):
89 Return : None
71 Return : None
90 """
72 """
91
73
92 #Eliminar de la base la herencia
74 ProcessingUnit.__init__(self)
93 ProcessingUnit.__init__(self)#, **kwargs)
94
95
75
96 self.pts2read_SelfSpectra = 0
76 self.pts2read_SelfSpectra = 0
97
98 self.pts2read_CrossSpectra = 0
77 self.pts2read_CrossSpectra = 0
99
100 self.pts2read_DCchannels = 0
78 self.pts2read_DCchannels = 0
101
102 self.datablock = None
103
104 self.utc = None
105
106 self.ext = ".pdata"
79 self.ext = ".pdata"
107
108 self.optchar = "P"
80 self.optchar = "P"
109
110 self.basicHeaderObj = BasicHeader(LOCALTIME)
81 self.basicHeaderObj = BasicHeader(LOCALTIME)
111
112 self.systemHeaderObj = SystemHeader()
82 self.systemHeaderObj = SystemHeader()
113
114 self.radarControllerHeaderObj = RadarControllerHeader()
83 self.radarControllerHeaderObj = RadarControllerHeader()
115
116 self.processingHeaderObj = ProcessingHeader()
84 self.processingHeaderObj = ProcessingHeader()
117
118 self.online = 0
119
120 self.fp = None
121
122 self.idFile = None
123
124 self.dtype = None
125
126 self.fileSizeByHeader = None
127
128 self.filenameList = []
129
130 self.filename = None
131
132 self.fileSize = None
133
134 self.firstHeaderSize = 0
135
136 self.basicHeaderSize = 24
137
138 self.pathList = []
139
140 self.lastUTTime = 0
85 self.lastUTTime = 0
141
142 self.maxTimeStep = 30
86 self.maxTimeStep = 30
143
87 self.dataOut = Spectra()
144 self.flagNoMoreFiles = 0
88 self.profileIndex = 1
145
89 self.nRdChannels = None
146 self.set = 0
90 self.nRdPairs = None
147
91 self.rdPairList = []
148 self.path = None
149
150 self.delay = 60 #seconds
151
152 self.nTries = 3 #quantity tries
153
154 self.nFiles = 3 #number of files for searching
155
156 self.nReadBlocks = 0
157
158 self.flagIsNewFile = 1
159
160 self.__isFirstTimeOnline = 1
161
162
163 self.flagDiscontinuousBlock = 0
164
165 self.flagIsNewBlock = 0
166
167 self.nTotalBlocks = 0
168
169 self.blocksize = 0
170
171 self.dataOut = self.createObjByDefault()
172
173 self.profileIndex = 1 #Always
174
175
92
176 def createObjByDefault(self):
93 def createObjByDefault(self):
177
94
@@ -224,9 +141,6 class SpectraReader(JRODataReader, ProcessingUnit):
224 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
141 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
225 self.blocksize += self.pts2read_DCchannels
142 self.blocksize += self.pts2read_DCchannels
226
143
227 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
228
229
230 def readBlock(self):
144 def readBlock(self):
231 """
145 """
232 Lee el bloque de datos desde la posicion actual del puntero del archivo
146 Lee el bloque de datos desde la posicion actual del puntero del archivo
@@ -248,7 +162,7 class SpectraReader(JRODataReader, ProcessingUnit):
248 Exceptions:
162 Exceptions:
249 Si un bloque leido no es un bloque valido
163 Si un bloque leido no es un bloque valido
250 """
164 """
251 blockOk_flag = False
165
252 fpointer = self.fp.tell()
166 fpointer = self.fp.tell()
253
167
254 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
168 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
@@ -262,7 +176,6 class SpectraReader(JRODataReader, ProcessingUnit):
262 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
176 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
263 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
177 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
264
178
265
266 if not self.processingHeaderObj.shif_fft:
179 if not self.processingHeaderObj.shif_fft:
267 #desplaza a la derecha en el eje 2 determinadas posiciones
180 #desplaza a la derecha en el eje 2 determinadas posiciones
268 shift = int(self.processingHeaderObj.profilesPerBlock/2)
181 shift = int(self.processingHeaderObj.profilesPerBlock/2)
@@ -298,39 +211,19 class SpectraReader(JRODataReader, ProcessingUnit):
298 def getFirstHeader(self):
211 def getFirstHeader(self):
299
212
300 self.getBasicHeader()
213 self.getBasicHeader()
301
302 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
214 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
303
304 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
215 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
305
306 # self.dataOut.ippSeconds = self.ippSeconds
307
308 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
309
310 self.dataOut.dtype = self.dtype
216 self.dataOut.dtype = self.dtype
311
312 # self.dataOut.nPairs = self.nPairs
313
314 self.dataOut.pairsList = self.rdPairList
217 self.dataOut.pairsList = self.rdPairList
315
316 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
218 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
317
318 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
219 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
319
320 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
220 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
321
322 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
221 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
323
324 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
222 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
325
326 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
223 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
327
328 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
224 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
329
330 self.dataOut.flagShiftFFT = True #Data is always shifted
225 self.dataOut.flagShiftFFT = True #Data is always shifted
331
332 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
226 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
333
334 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
227 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
335
228
336 def getData(self):
229 def getData(self):
@@ -347,7 +240,6 class SpectraReader(JRODataReader, ProcessingUnit):
347
240
348 Affected:
241 Affected:
349 self.dataOut
242 self.dataOut
350
351 self.flagDiscontinuousBlock
243 self.flagDiscontinuousBlock
352 self.flagIsNewBlock
244 self.flagIsNewBlock
353 """
245 """
@@ -372,20 +264,16 class SpectraReader(JRODataReader, ProcessingUnit):
372 return 0
264 return 0
373
265
374 self.getBasicHeader()
266 self.getBasicHeader()
375
376 self.getFirstHeader()
267 self.getFirstHeader()
377
378 self.dataOut.data_spc = self.data_spc
268 self.dataOut.data_spc = self.data_spc
379
380 self.dataOut.data_cspc = self.data_cspc
269 self.dataOut.data_cspc = self.data_cspc
381
382 self.dataOut.data_dc = self.data_dc
270 self.dataOut.data_dc = self.data_dc
383
384 self.dataOut.flagNoData = False
271 self.dataOut.flagNoData = False
385
386 self.dataOut.realtime = self.online
272 self.dataOut.realtime = self.online
387
273
388 return self.dataOut.data_spc
274 return self.dataOut.data_spc
275
276
389 @MPDecorator
277 @MPDecorator
390 class SpectraWriter(JRODataWriter, Operation):
278 class SpectraWriter(JRODataWriter, Operation):
391
279
@@ -394,22 +282,6 class SpectraWriter(JRODataWriter, Operation):
394 de los datos siempre se realiza por bloques.
282 de los datos siempre se realiza por bloques.
395 """
283 """
396
284
397 ext = ".pdata"
398
399 optchar = "P"
400
401 shape_spc_Buffer = None
402
403 shape_cspc_Buffer = None
404
405 shape_dc_Buffer = None
406
407 data_spc = None
408
409 data_cspc = None
410
411 data_dc = None
412
413 def __init__(self):
285 def __init__(self):
414 """
286 """
415 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
287 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
@@ -426,41 +298,21 class SpectraWriter(JRODataWriter, Operation):
426
298
427 Operation.__init__(self)
299 Operation.__init__(self)
428
300
429 self.nTotalBlocks = 0
301 self.ext = ".pdata"
430
302 self.optchar = "P"
303 self.shape_spc_Buffer = None
304 self.shape_cspc_Buffer = None
305 self.shape_dc_Buffer = None
431 self.data_spc = None
306 self.data_spc = None
432
433 self.data_cspc = None
307 self.data_cspc = None
434
435 self.data_dc = None
308 self.data_dc = None
436
437 self.fp = None
438
439 self.flagIsNewFile = 1
440
441 self.nTotalBlocks = 0
442
443 self.flagIsNewBlock = 0
444
445 self.setFile = None
309 self.setFile = None
446
447 self.dtype = None
448
449 self.path = None
450
451 self.noMoreFiles = 0
310 self.noMoreFiles = 0
452
453 self.filename = None
454
455 self.basicHeaderObj = BasicHeader(LOCALTIME)
311 self.basicHeaderObj = BasicHeader(LOCALTIME)
456
457 self.systemHeaderObj = SystemHeader()
312 self.systemHeaderObj = SystemHeader()
458
459 self.radarControllerHeaderObj = RadarControllerHeader()
313 self.radarControllerHeaderObj = RadarControllerHeader()
460
461 self.processingHeaderObj = ProcessingHeader()
314 self.processingHeaderObj = ProcessingHeader()
462
315
463
464 def hasAllDataInBuffer(self):
316 def hasAllDataInBuffer(self):
465 return 1
317 return 1
466
318
Show file before | Show file after | Hide comments
@@ -57,12 +57,7 class VoltageReader(JRODataReader, ProcessingUnit):
57
57
58 """
58 """
59
59
60 ext = ".r"
60 def __init__(self):
61
62 optchar = "D"
63 dataOut = None
64
65 def __init__(self):#, **kwargs):
66 """
61 """
67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
62 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
68
63
@@ -81,89 +76,19 class VoltageReader(JRODataReader, ProcessingUnit):
81 None
76 None
82 """
77 """
83
78
84 ProcessingUnit.__init__(self)#, **kwargs)
79 ProcessingUnit.__init__(self)
85
86 self.isConfig = False
87
88 self.datablock = None
89
90 self.utc = 0
91
80
92 self.ext = ".r"
81 self.ext = ".r"
93
94 self.optchar = "D"
82 self.optchar = "D"
95
96 self.basicHeaderObj = BasicHeader(LOCALTIME)
83 self.basicHeaderObj = BasicHeader(LOCALTIME)
97
98 self.systemHeaderObj = SystemHeader()
84 self.systemHeaderObj = SystemHeader()
99
100 self.radarControllerHeaderObj = RadarControllerHeader()
85 self.radarControllerHeaderObj = RadarControllerHeader()
101
102 self.processingHeaderObj = ProcessingHeader()
86 self.processingHeaderObj = ProcessingHeader()
103
104 self.online = 0
105
106 self.fp = None
107
108 self.idFile = None
109
110 self.dtype = None
111
112 self.fileSizeByHeader = None
113
114 self.filenameList = []
115
116 self.filename = None
117
118 self.fileSize = None
119
120 self.firstHeaderSize = 0
121
122 self.basicHeaderSize = 24
123
124 self.pathList = []
125
126 self.filenameList = []
127
128 self.lastUTTime = 0
87 self.lastUTTime = 0
129
130 self.maxTimeStep = 30
131
132 self.flagNoMoreFiles = 0
133
134 self.set = 0
135
136 self.path = None
137
138 self.profileIndex = 2**32 - 1
88 self.profileIndex = 2**32 - 1
139
89 self.dataOut = Voltage()
140 self.delay = 3 # seconds
90 self.selBlocksize = None
141
91 self.selBlocktime = None
142 self.nTries = 3 # quantity tries
143
144 self.nFiles = 3 # number of files for searching
145
146 self.nReadBlocks = 0
147
148 self.flagIsNewFile = 1
149
150 self.__isFirstTimeOnline = 1
151
152 # self.ippSeconds = 0
153
154 self.flagDiscontinuousBlock = 0
155
156 self.flagIsNewBlock = 0
157
158 self.nTotalBlocks = 0
159
160 self.blocksize = 0
161
162 self.dataOut = self.createObjByDefault()
163
164 self.nTxs = 1
165
166 self.txIndex = 0
167
92
168 def createObjByDefault(self):
93 def createObjByDefault(self):
169
94
@@ -545,6 +470,7 class VoltageReader(JRODataReader, ProcessingUnit):
545
470
546 return self.dataOut.data
471 return self.dataOut.data
547
472
473
548 @MPDecorator
474 @MPDecorator
549 class VoltageWriter(JRODataWriter, Operation):
475 class VoltageWriter(JRODataWriter, Operation):
550 """
476 """
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now