##// END OF EJS Templates
schain-jc
RSS
Add reading online mode for BLTR parameters
Juan C. Espinoza -
r1215:785b27922a3d
parent child
Show More
Side by Side Unified
Context file:
r1215:785b27922a3d Loading diff...:
Show file before | Show file after | Hide comments
@@ -118,6 +118,9 class BLTRParamReader(JRODataReader, ProcessingUnit):
118 118 self.endTime = endTime
119 119 self.status_value = status_value
120 120 self.datatime = datetime.datetime(1900,1,1)
121 self.delay = kwargs.get('delay', 10)
122 self.online = kwargs.get('online', False)
123 self.nTries = kwargs.get('nTries', 3)
121 124
122 125 if self.path is None:
123 126 raise ValueError("The path is not valid")
@@ -125,7 +128,7 class BLTRParamReader(JRODataReader, ProcessingUnit):
125 128 if ext is None:
126 129 ext = self.ext
127 130
128 self.search_files(self.path, startDate, endDate, ext)
131 self.fileList = self.search_files(self.path, startDate, endDate, ext)
129 132 self.timezone = timezone
130 133 self.fileIndex = 0
131 134
@@ -135,6 +138,29 class BLTRParamReader(JRODataReader, ProcessingUnit):
135 138
136 139 self.setNextFile()
137 140
141 def search_last_file(self):
142 '''
143 Get last file and add it to the list
144 '''
145
146 for n in range(self.nTries):
147 if n > 0:
148 log.warning(
149 "Waiting %0.2f sec for the next file, try %03d ..." % (self.delay, self.nTries + 1),
150 self.name
151 )
152 time.sleep(self.delay)
153 file_list = os.listdir(self.path)
154 file_list.sort()
155 if file_list:
156 if self.filename:
157 if file_list[-1] not in self.filename:
158 return file_list[-1]
159 else:
160 continue
161 return file_list[-1]
162 return 0
163
138 164 def search_files(self, path, startDate, endDate, ext):
139 165 '''
140 166 Searching for BLTR rawdata file in path
@@ -152,8 +178,8 class BLTRParamReader(JRODataReader, ProcessingUnit):
152 178 fileList0 = glob.glob1(path, "*%s" % ext)
153 179 fileList0.sort()
154 180
155 self.fileList = []
156 self.dateFileList = []
181 #self.fileList = []
182 #self.dateFileList = []
157 183
158 184 for thisFile in fileList0:
159 185 year = thisFile[-14:-10]
@@ -174,28 +200,34 class BLTRParamReader(JRODataReader, ProcessingUnit):
174 200 if (startDate > dateFile) or (endDate < dateFile):
175 201 continue
176 202
177 self.fileList.append(thisFile)
178 self.dateFileList.append(dateFile)
203 yield thisFile
204 # self.dateFileList.append(dateFile)
179 205
180 206 return
181 207
182 208 def setNextFile(self):
183 209
184 file_id = self.fileIndex
185
186 if file_id == len(self.fileList):
210 if self.online:
211 filename = self.search_last_file()
212 if not filename:
213 self.flagNoMoreFiles = 1
214 return 0
215 else:
216 try:
217 filename = next(self.fileList)
218 except StopIteration:
219 print('Noooo files')
187 220 self.flagNoMoreFiles = 1
188 221 return 0
189 222
190 log.success('Opening {}'.format(self.fileList[file_id]), 'BLTRParamReader')
191 filename = os.path.join(self.path, self.fileList[file_id])
223 log.success('Opening {}'.format(filename), 'BLTRParamReader')
192 224
193 225 dirname, name = os.path.split(filename)
194 226 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
195 self.siteFile = name.split('.')[0]
227 self.siteFile = filename.split('.')[0]
196 228 if self.filename is not None:
197 229 self.fp.close()
198 self.filename = filename
230 self.filename = os.path.join(self.path, filename)
199 231 self.fp = open(self.filename, 'rb')
200 232 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
201 233 self.nrecords = self.header_file['nrec'][0]
@@ -214,7 +246,15 class BLTRParamReader(JRODataReader, ProcessingUnit):
214 246 if not self.setNextFile():
215 247 return 0
216 248
249 try:
250 pointer = self.fp.tell()
217 251 self.readBlock()
252 except:
253 if self.waitDataBlock(pointer, 38512) == 1:
254 continue
255 else:
256 if not self.setNextFile():
257 return 0
218 258
219 259 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
220 260 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
@@ -357,10 +397,11 class BLTRParamReader(JRODataReader, ProcessingUnit):
357 397 if self.flagNoMoreFiles:
358 398 self.dataOut.flagNoData = True
359 399 self.dataOut.error = 'No More files to read'
400 return
360 401
361 402 if not self.readNextBlock():
362 403 self.dataOut.flagNoData = True
363 return 0
404 self.dataOut.error = 'Time for wait new file reach!!!'
364 405
365 406 self.set_output()
366 407
Show file before | Show file after | Hide comments
@@ -946,11 +946,13 class JRODataReader(JRODataIO):
946 946
947 947 return 0
948 948
949 def waitDataBlock(self, pointer_location):
949 def waitDataBlock(self, pointer_location, blocksize=None):
950 950
951 951 currentPointer = pointer_location
952
952 if blocksize is None:
953 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 else:
955 neededSize = blocksize
954 956
955 957 for nTries in range(self.nTries):
956 958 self.fp.close()
@@ -963,7 +965,10 class JRODataReader(JRODataIO):
963 965 if (currentSize >= neededSize):
964 966 return 1
965 967
966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
968 log.warning(
969 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
970 self.name
971 )
967 972 sleep(self.delay)
968 973
969 974 return 0
General Comments 0
You need to be logged in to leave comments. Login now