##// END OF EJS Templates
schain
RSS
Se añadio dos atributos a la clase de lectura de voltajes: nReadBlocks y flagIsNewBlock.
Miguel Valdez -
r13:9243316017ec
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,34 +1,35
1 1 '''
2 2 Created on 23/01/2012
3 3
4 4 @author: danielangelsuarezmunoz
5 5 '''
6 6
7 7 import Voltage
8 8 import datetime
9 import time
9 import time, datetime
10 10
11 11 objReader = Voltage.VoltageReader()
12 12
13 13
14 14 path = '/home/roj-idl71/Data/RAWDATA/DP_Faraday/'
15 15 #path = '/remote/puma/2011_03/DP_Faraday'
16 16
17 17
18 startDateTime = datetime.datetime(2011,3,11,16,0,0)
19 endDateTime = datetime.datetime(2011,3,12,20,1,0)
18 startDateTime = datetime.datetime(2011,3,11,16,40,0)
19 endDateTime = datetime.datetime(2011,3,11,16,59,0)
20 20
21 21 t0 = time.time()
22 22 objReader.setup(path, startDateTime, endDateTime)
23 23 print time.time() - t0
24 24
25 25
26 26 while(not(objReader.noMoreFiles)):
27 27
28 28 objReader.getData()
29 # if objReader.flagIsNewFile:
29 if objReader.flagIsNewBlock:
30 print "Block No %04d, Time: %s" %(objReader.nReadBlocks, datetime.datetime.fromtimestamp(objReader.m_BasicHeader.utc))
30 31 # print objReader.m_BasicHeader.dataBlock
31 32 #print objReader.objStructShortHeader.dataBlock
32 #print time.localtime(objReader.m_BasicHeader.utc)
33
33 34
34 35 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,445 +1,454
1 1 '''
2 2 Created on 23/01/2012
3 3
4 4 @author: danielangelsuarezmunoz
5 5 '''
6 6
7 7
8 8 import os, sys
9 9 import numpy
10 10 import glob
11 11 import fnmatch
12 12 import time
13 13 import datetime
14 14
15 15 from Header import *
16 16 from Data import DataReader
17 17 from Data import DataWriter
18 18
19 19 path = os.path.split(os.getcwd())[0]
20 20 sys.path.append(path)
21 21
22 22 from Model.Voltage import Voltage
23 23
24 24 class VoltageReader(DataReader):
25 25
26 26 __idFile = None
27 27
28 28 __fp = None
29 29
30 30 __startDateTime = None
31 31
32 32 __endDateTime = None
33 33
34 34 __dataType = None
35 35
36 36 __fileSizeByHeader = 0
37 37
38 38 __pathList = []
39 39
40 40 filenameList = []
41 41
42 42 __lastUTTime = 0
43 43
44 44 __maxTimeStep = 5
45 45
46 __flagIsNewFile = 0
47
46 48 flagResetProcessing = 0
47 49
48 __flagIsNewFile = 0
50 flagIsNewBlock = 0
49 51
50 52 noMoreFiles = 0
51 53
54 nReadBlocks = 0
55
52 56 online = 0
53 57
54 58 filename = None
55 59
56 60 fileSize = None
57 61
58 62 firstHeaderSize = 0
59 63
60 64 basicHeaderSize = 24
61 65
62 66 m_BasicHeader = BasicHeader()
63 67
64 68 m_SystemHeader = SystemHeader()
65 69
66 70 m_RadarControllerHeader = RadarControllerHeader()
67 71
68 72 m_ProcessingHeader = ProcessingHeader()
69 73
70 74 m_Voltage = None
71 75
72 76 __buffer = 0
73 77
74 78 __buffer_id = 9999
75 79
76 80 def __init__(self, m_Voltage = None):
77 81
78 82 if m_Voltage == None:
79 83 m_Voltage = Voltage()
80 84
81 85 self.m_Voltage = m_Voltage
82 86
83 87 def __rdSystemHeader(self,fp=None):
84 88 if fp == None:
85 89 fp = self.__fp
86 90
87 91 self.m_SystemHeader.read(fp)
88 92
89 93 def __rdRadarControllerHeader(self,fp=None):
90 94 if fp == None:
91 95 fp = self.__fp
92 96
93 97 self.m_RadarControllerHeader.read(fp)
94 98
95 99 def __rdProcessingHeader(self,fp=None):
96 100 if fp == None:
97 101 fp = self.__fp
98 102
99 103 self.m_ProcessingHeader.read(fp)
100 104
101 105 def __searchFiles(self,path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r"):
102 106
103 107 print "Searching files ..."
104 108
105 109 startUtSeconds = time.mktime(startDateTime.timetuple())
106 110 endUtSeconds = time.mktime(endDateTime.timetuple())
107 111
108 112 # startYear = startDateTime.timetuple().tm_year
109 113 # endYear = endDateTime.timetuple().tm_year
110 114 #
111 115 # startDoy = startDateTime.timetuple().tm_yday
112 116 # endDoy = endDateTime.timetuple().tm_yday
113 117 #
114 118 # yearRange = range(startYear,endYear+1)
115 119 #
116 120 # doyDoubleList = []
117 121 # if startYear == endYear:
118 122 # doyList = range(startDoy,endDoy+1)
119 123 # else:
120 124 # for year in yearRange:
121 125 # if (year == startYear):
122 126 # doyDoubleList.append(range(startDoy,365+1))
123 127 # elif (year == endYear):
124 128 # doyDoubleList.append(range(1,endDoy+1))
125 129 # else:
126 130 # doyDoubleList.append(range(1,365+1))
127 131 # doyList = []
128 132 # for list in doyDoubleList:
129 133 # doyList = doyList + list
130 134 #
131 135 # dirList = []
132 136 # for thisPath in os.listdir(path):
133 137 # if os.path.isdir(os.path.join(path,thisPath)):
134 138 # #dirList.append(os.path.join(path,thisPath))
135 139 # dirList.append(thisPath)
136 140 #
137 141 # pathList = []
138 142 # pathDict = {}
139 143 # for year in yearRange:
140 144 # for doy in doyList:
141 145 # match = fnmatch.filter(dirList, 'D' + '%4.4d%3.3d' % (year,doy))
142 146 # if len(match) == 0:
143 147 # match = fnmatch.filter(dirList, 'd' + '%4.4d%3.3d' % (year,doy))
144 148 # if len(match) == 0: continue
145 149 # if expLabel == '':
146 150 # pathList.append(os.path.join(path,match[0]))
147 151 # pathDict.setdefault(os.path.join(path,match[0]))
148 152 # pathDict[os.path.join(path,match[0])] = []
149 153 # else:
150 154 # pathList.append(os.path.join(path,os.path.join(match[0],expLabel)))
151 155 # pathDict.setdefault(os.path.join(path,os.path.join(match[0],expLabel)))
152 156 # pathDict[os.path.join(path,os.path.join(match[0],expLabel))] = []
153 157
154 158
155 159 dirList = []
156 160 for thisPath in os.listdir(path):
157 161 if os.path.isdir(os.path.join(path,thisPath)):
158 162 dirList.append(thisPath)
159 163
160 164 pathList = []
161 165
162 166 thisDateTime = startDateTime
163 167
164 168 while(thisDateTime <= endDateTime):
165 169 year = thisDateTime.timetuple().tm_year
166 170 doy = thisDateTime.timetuple().tm_yday
167 171
168 172 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
169 173 if len(match) == 0:
170 174 thisDateTime += datetime.timedelta(1)
171 175 continue
172 176
173 177 pathList.append(os.path.join(path,match[0],expLabel))
174 178 thisDateTime += datetime.timedelta(1)
175 179
176 180 filenameList = []
177 181 for thisPath in pathList:
178 182 fileList = glob.glob1(thisPath, "*%s" %ext)
179 183 #pathDict[thisPath].append(fileList)
180 184 fileList.sort()
181 185 for file in fileList:
182 186 filename = os.path.join(thisPath,file)
183 187 if self.isThisFileinRange(filename, startUtSeconds, endUtSeconds):
184 188 filenameList.append(filename)
185 189
186 190 self.filenameList = filenameList
187 191
188 192 return pathList, filenameList
189 193
190 194 def isThisFileinRange(self, filename, startUTSeconds=None, endUTSeconds=None):
191 195
192 196 try:
193 197 fp = open(filename,'rb')
194 198 except:
195 199 raise IOError, "The file %s can't be opened" %(filename)
196 200
197 201 if startUTSeconds==None:
198 202 startUTSeconds = self.startUTCSeconds
199 203
200 204 if endUTSeconds==None:
201 205 endUTSeconds = self.endUTCSeconds
202 206
203 207 m_BasicHeader = BasicHeader()
204 208
205 209 if not(m_BasicHeader.read(fp)):
206 210 return 0
207 211
208 212 fp.close()
209 213
210 214 if not ((startUTSeconds <= m_BasicHeader.utc) and (endUTSeconds >= m_BasicHeader.utc)):
211 215 return 0
212 216
213 217 return 1
214 218
215 219 def __readBasicHeader(self, fp=None):
216 220
217 221 if fp == None:
218 222 fp = self.__fp
219 223
220 224 self.m_BasicHeader.read(fp)
221 225
222 226 def __readFirstHeader(self):
223 227
224 228 self.__readBasicHeader()
225 229 self.__rdSystemHeader()
226 230 self.__rdRadarControllerHeader()
227 231 self.__rdProcessingHeader()
228 232 self.firstHeaderSize = self.m_BasicHeader.size
229 233
230 234 data_type=int(numpy.log2((self.m_ProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
231 235 if data_type == 0:
232 236 tmp=numpy.dtype([('real','<i1'),('imag','<i1')])
233 237 elif data_type == 1:
234 238 tmp=numpy.dtype([('real','<i2'),('imag','<i2')])
235 239 elif data_type == 2:
236 240 tmp=numpy.dtype([('real','<i4'),('imag','<i4')])
237 241 elif data_type == 3:
238 242 tmp=numpy.dtype([('real','<i8'),('imag','<i8')])
239 243 elif data_type == 4:
240 244 tmp=numpy.dtype([('real','<f4'),('imag','<f4')])
241 245 elif data_type == 5:
242 246 tmp=numpy.dtype([('real','<f8'),('imag','<f8')])
243 247 else:
244 print 'no define data type'
245 tmp = 0
248 raise ValueError, 'Data type was not defined'
246 249
247 250 self.__dataType = tmp
248 251 self.__fileSizeByHeader = self.m_ProcessingHeader.dataBlocksPerFile * self.m_ProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.m_ProcessingHeader.dataBlocksPerFile - 1)
249 252
250 253 def __setNextFileOnline(self):
251 254 return 0
252 255
253 256 def __setNextFileOffline(self):
254 257
255 258 idFile = self.__idFile
256 259 while(True):
257 260
258 261 idFile += 1
259 262
260 263 if not(idFile < len(self.filenameList)):
261 264 self.noMoreFiles = 1
262 265 return 0
263 266
264 267 filename = self.filenameList[idFile]
265 268 fileSize = os.path.getsize(filename)
266 269 fp = open(filename,'rb')
267 270
268 271 currentSize = fileSize - fp.tell()
269 272 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
270 273
271 274 if (currentSize < neededSize):
272 275 continue
273 276
274 277 break
275 278
276 279 self.__flagIsNewFile = 1
277 280 self.__idFile = idFile
278 281 self.filename = filename
279 282 self.fileSize = fileSize
280 283 self.__fp = fp
281 284
282 285 print 'Setting the file: %s'%self.filename
283 286
284 287 return 1
285 288
286 289 def __setNextFile(self):
287 290
288 291 if self.online:
289 292 return self.__setNextFileOnline()
290 293 else:
291 294 return self.__setNextFileOffline()
292 295
293 296 def __setNewBlock(self):
294 297
295 298 if self.__fp == None:
296 299 return 0
297 300
298 301 if self.__flagIsNewFile:
299 302 return 1
300 303
301 304 currentSize = self.fileSize - self.__fp.tell()
302 305 neededSize = self.m_ProcessingHeader.blockSize + self.basicHeaderSize
303 306
304 307 # Bloque Completo
305 308 if (currentSize >= neededSize):
306 309 self.__readBasicHeader()
307 310 return 1
308 311
309 312 if not(self.__setNextFile()):
310 313 return 0
311 314
312 315 self.__readFirstHeader()
313 316
314 317 deltaTime = self.m_BasicHeader.utc - self.__lastUTTime # check this
315 318
316 319 self.flagResetProcessing = 0
317 320 if deltaTime > self.__maxTimeStep:
318 321 self.flagResetProcessing = 1
322 self.nReadBlocks = 0
319 323
320 324 return 1
321 325
322 326 def __readBlock(self):
323 327 """Lee el bloque de datos desde la posicion actual del puntero del archivo y
324 328 actualiza todos los parametros relacionados al bloque de datos (data, time,
325 329 etc). La data leida es almacenada en el buffer y el contador de datos leidos es
326 330 seteado a 0
327 331 """
328 332
329 333 pts2read = self.m_ProcessingHeader.profilesPerBlock*self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels
330 334
331 335 data = numpy.fromfile(self.__fp,self.__dataType,pts2read)
332 336
333 337 data = data.reshape((self.m_ProcessingHeader.profilesPerBlock, self.m_ProcessingHeader.numHeights, self.m_SystemHeader.numChannels))
334 338
335 339 self.__flagIsNewFile = 0
336 340
341 self.flagIsNewBlock = 1
342
343 self.nReadBlocks += 1
344
337 345 self.__buffer = data
338 346
339 347 self.__buffer_id = 0
340 348
341 349 def readNextBlock(self):
342 350
343 351 if not(self.__setNewBlock()):
344 352 return 0
345 353
346 354 self.__readBlock()
347 355
348 356 self.__lastUTTime = self.m_BasicHeader.utc
349 357
350 358 return 1
351 359
352 360 def __hasNotDataInBuffer(self):
353 361 if self.__buffer_id >= self.m_ProcessingHeader.profilesPerBlock:
354 362 return 1
355 363
356 364 return 0
357 365
358 366 def getData(self):
359 367 """Obtiene un unidad de datos del buffer de lectura y es copiada a la clase "Voltage"
360 368 con todos los parametros asociados a este. cuando no hay datos en el buffer de
361 369 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
362 370 """
363 371 self.flagResetProcessing = 0
372 self.flagIsNewBlock = 0
364 373
365 374 if self.__hasNotDataInBuffer():
366 375 self.readNextBlock()
367 376
368 377 if self.noMoreFiles == 1:
369 378 print 'Process finished'
370 379 return None
371 380
372 381 data = self.__buffer[self.__buffer_id,:,:]
373 382 time = 111
374 383
375 384 # self.m_Voltage.data = data
376 385 # self.m_Voltage.timeProfile = time
377 386 # self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy()
378 387 # self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy()
379 388 # self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
380 389 # self.m_Voltage.m_SystemHeader = self.m_systemHeader.copy()
381 390
382 391 self.__buffer_id += 1
383 392
384 393 #call setData - to Data Object
385 394
386 395 return data
387 396
388 397
389 398 def setup(self, path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r", online = 0):
390 399
391 400 if online == 0:
392 401 pathList, filenameList = self.__searchFiles(path, startDateTime, endDateTime, set, expLabel, ext)
393 402
394 403 if len(filenameList) == 0:
395 404 self.__fp = None
396 405 self.noMoreFiles = 1
397 406 print 'Do not exist files in range: %s - %s'%(startDateTime.ctime(), endDateTime.ctime())
398 407 return 0
399 408
400 409 # for thisFile in filenameList:
401 410 # print thisFile
402 411
403 412 self.__idFile = -1
404 413
405 414 if not(self.__setNextFile()):
406 415 print "No more files"
407 416 return 0
408 417
409 418 self.__readFirstHeader()
410 419
411 420 self.startUTCSeconds = time.mktime(startDateTime.timetuple())
412 421 self.endUTCSeconds = time.mktime(endDateTime.timetuple())
413 422
414 423 self.startYear = startDateTime.timetuple().tm_year
415 424 self.endYear = endDateTime.timetuple().tm_year
416 425
417 426 self.startDoy = startDateTime.timetuple().tm_yday
418 427 self.endDoy = endDateTime.timetuple().tm_yday
419 428 #call fillHeaderValues() - to Data Object
420 429
421 430 self.__pathList = pathList
422 431 self.filenameList = filenameList
423 432 self.online = online
424 433
425 434 class VoltageWriter(DataWriter):
426 435
427 436 m_BasicHeader= BasicHeader()
428 437
429 438
430 439 m_SystemHeader = SystemHeader()
431 440
432 441
433 442 m_RadarControllerHeader = RadarControllerHeader()
434 443
435 444
436 445 m_ProcessingHeader = ProcessingHeader()
437 446
438 447 m_Voltage = None
439 448
440 449 def __init__(self, m_Voltage):
441 450
442 451 self.m_Voltage = m_Voltage
443 452
444 453
445 454 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now