schain-jc Commit - r1215:785b27922a3d

Add reading online mode for BLTR parameters

Juan C. Espinoza -

r1215:785b27922a3d

parent child

Context file:

r1215:785b27922a3d

Collapse all files

schainpy/model/io/bltrIO_param.py +54 -13

                     self.endTime = endTime
                     self.status_value = status_value
                     self.datatime = datetime.datetime(1900,1,1)
+                    self.delay = kwargs.get('delay', 10)
+                    self.online = kwargs.get('online', False)
+                    self.nTries = kwargs.get('nTries', 3)
                     if self.path is None:
                         raise ValueError("The path is not valid")
                     if ext is None:
                         ext = self.ext
-                    self.search_files(self.path, startDate, endDate, ext)
+                    self.fileList = self.search_files(self.path, startDate, endDate, ext)
                     self.timezone = timezone
                     self.fileIndex = 0
                     self.setNextFile()
+                def search_last_file(self):
+                    '''
+                    Get last file and add it to the list
+                    '''
+                    for n in range(self.nTries):
+                        if n > 0:
+                            log.warning(
+                                "Waiting %0.2f sec for the next file, try %03d ..." % (self.delay, self.nTries + 1),
+                                self.name
+                            )
+                            time.sleep(self.delay)
+                        file_list = os.listdir(self.path)
+                        file_list.sort()
+                        if file_list:
+                            if self.filename:
+                                if file_list[-1] not in self.filename:
+                                    return file_list[-1]
+                                else:
+                                    continue
+                            return file_list[-1]
+                    return 0
                 def search_files(self, path, startDate, endDate, ext):
                     '''
                      Searching for BLTR rawdata file in path
                     fileList0 = glob.glob1(path, "*%s" % ext)
                     fileList0.sort()
-                    self.fileList = []
+                    #self.fileList = []
-                    self.dateFileList = []
+                    #self.dateFileList = []
                     for thisFile in fileList0:
                         year = thisFile[-14:-10]
                         if (startDate > dateFile) or (endDate < dateFile):
                             continue
-                        self.fileList.append(thisFile)
+                        yield thisFile
-                        self.dateFileList.append(dateFile)
+                        # self.dateFileList.append(dateFile)
                     return
                 def setNextFile(self):
-                    file_id = self.fileIndex
+                    if self.online:
+                        filename = self.search_last_file()
-                    if file_id == len(self.fileList):
+                        if not filename:
+                            self.flagNoMoreFiles = 1
+                            return 0
+                    else:
+                        try:
+                            filename = next(self.fileList)
+                        except StopIteration:
+                            print('Noooo files')
                             self.flagNoMoreFiles = 1
                             return 0
-                    log.success('Opening {}'.format(self.fileList[file_id]), 'BLTRParamReader')
+                    log.success('Opening {}'.format(filename), 'BLTRParamReader')
-                    filename = os.path.join(self.path, self.fileList[file_id])
                     dirname, name = os.path.split(filename)
                     # 'peru2' ---> Piura  -   'peru1' ---> Huancayo or Porcuya
-                    self.siteFile = name.split('.')[0]
+                    self.siteFile = filename.split('.')[0]
                     if self.filename is not None:
                         self.fp.close()
-                    self.filename = filename
+                    self.filename = os.path.join(self.path, filename)
                     self.fp = open(self.filename, 'rb')
                     self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
                     self.nrecords = self.header_file['nrec'][0]
                             if not self.setNextFile():
                                 return 0
+                        try:
+                            pointer = self.fp.tell()
                             self.readBlock()
+                        except:
+                            if self.waitDataBlock(pointer, 38512) == 1:
+                                continue
+                            else:
+                                if not self.setNextFile():
+                                    return 0
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         self.dataOut.error = 'No More files to read'
+                        return
                     if not self.readNextBlock():
                         self.dataOut.flagNoData = True
-                        return 0
+                        self.dataOut.error = 'Time for wait new file reach!!!'
                     self.set_output()

schainpy/model/io/jroIO_base.py +8 -3

                     return 0
-                def waitDataBlock(self, pointer_location):
+                def waitDataBlock(self, pointer_location, blocksize=None):
                     currentPointer = pointer_location
+                    if blocksize is None:
                         neededSize = self.processingHeaderObj.blockSize  # + self.basicHeaderSize
+                    else:
+                        neededSize = blocksize
                     for nTries in range(self.nTries):
                         self.fp.close()
                         if (currentSize >= neededSize):
                             return 1
-                        print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
+                        log.warning(
+                            "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
+                            self.name
+                            )
                         sleep(self.delay)
                     return 0

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages