##// END OF EJS Templates
schain-jc
RSS
First Version of Voltage Reader Module
Daniel Valdez -
r2:e0918654d2e9
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,12
1 '''
2 Created on 23/01/2012
3
4 @author: danielangelsuarezmunoz
5 '''
6
7
8 class DataReader:
9 __buffer = 0
10 __buffer_count = 0
11 def __init__(self):
12 pass No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,31
1 '''
2 Created on 23/01/2012
3
4 @author: danielangelsuarezmunoz
5 '''
6
7 import VoltageReader
8 import datetime
9 import time
10
11 objReader = VoltageReader.VoltageReader()
12
13 path = '/Users/danielangelsuarezmunoz/Documents/Projects'
14 startDateTime = datetime.datetime(2007,1,1,16,0,0)
15 endDateTime = datetime.datetime(2007,12,1,17,1,0)
16 set = None
17 expLabel = ''
18 ext = '*.r'
19
20 t0 = time.time()
21 objReader.setup(path, startDateTime, endDateTime, set, expLabel, ext)
22 print time.time() - t0
23
24
25 while(not(objReader.noMoreFiles)):
26
27 objReader.getData()
28 #print objReader.objStructShortHeader.dataBlock
29 #print time.localtime(objReader.objStructShortHeader.universalTime)
30
31 No newline at end of file
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (592 lines changed) Show them Hide them
@@ -0,0 +1,592
1 '''
2 Created on 23/01/2012
3
4 @author: danielangelsuarezmunoz
5 '''
6
7 from DataReader import DataReader
8
9 import numpy
10 import os.path
11 import glob
12 import fnmatch
13 import time
14 import datetime
15
16 class PROCFLAG:
17 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
18 DECODE_DATA = numpy.uint32(0x00000002)
19 SPECTRA_CALC = numpy.uint32(0x00000004)
20 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
21 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
22 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
23
24 DATATYPE_CHAR = numpy.uint32(0x00000040)
25 DATATYPE_SHORT = numpy.uint32(0x00000080)
26 DATATYPE_LONG = numpy.uint32(0x00000100)
27 DATATYPE_INT64 = numpy.uint32(0x00000200)
28 DATATYPE_FLOAT = numpy.uint32(0x00000400)
29 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
30
31 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
32 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
33 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
34
35 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
36 DEFLIP_DATA = numpy.uint32(0x00010000)
37 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
38
39 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
40 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
41 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
42 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
43 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
44
45 EXP_NAME_ESP = numpy.uint32(0x00200000)
46 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
47
48 OPERATION_MASK = numpy.uint32(0x0000003F)
49 DATATYPE_MASK = numpy.uint32(0x00000FC0)
50 DATAARRANGE_MASK = numpy.uint32(0x00007000)
51 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
52
53 class StructShortHeader():
54
55 size = 0
56 version = 0
57 dataBlock = 0
58 utc = 0
59 miliSecond = 0
60 timeZone = 0
61 dstFlag = 0
62 errorCount = 0
63 struct = numpy.dtype([
64 ('nSize','<u4'),
65 ('nVersion','<u2'),
66 ('nDataBlockId','<u4'),
67 ('nUtime','<u4'),
68 ('nMilsec','<u2'),
69 ('nTimezone','<i2'),
70 ('nDstflag','<i2'),
71 ('nErrorCount','<u4')
72 ])
73
74 def __init__(self):
75 pass
76
77 def read(self, fp):
78
79 header = numpy.fromfile(fp, self.struct,1)
80 self.size = header['nSize'][0]
81 self.version = header['nVersion'][0]
82 self.dataBlock = header['nDataBlockId'][0]
83 self.utc = header['nUtime'][0]
84 self.miliSecond = header['nMilsec'][0]
85 self.timeZone = header['nTimezone'][0]
86 self.dstFlag = header['nDstflag'][0]
87 self.errorCount = header['nErrorCount'][0]
88
89 return 1
90
91 class StructSystemHeader():
92 size = 0
93 numSamples = 0
94 numProfiles = 0
95 numChannels = 0
96 adcResolution = 0
97 pciDioBusWidth = 0
98 struct = numpy.dtype([
99 ('nSize','<u4'),
100 ('nNumSamples','<u4'),
101 ('nNumProfiles','<u4'),
102 ('nNumChannels','<u4'),
103 ('nADCResolution','<u4'),
104 ('nPCDIOBusWidth','<u4'),
105 ])
106
107 def __init__(self):
108 pass
109
110 def read(self, fp):
111
112
113 return 1
114
115 class StructRadarController():
116 size = 0
117 expType = 0
118 nTx = 0
119 ipp = 0
120 txA = 0
121 txB = 0
122 numWindows = 0
123 numTaus = 0
124 codeType = 0
125 line6Function = 0
126 line5Fuction = 0
127 fClock = 0
128 prePulseBefore = 0
129 prePulserAfter = 0
130 rangeIpp = 0
131 rangeTxA = 0
132 rangeTxB = 0
133 struct = numpy.dtype([
134 ('nSize','<u4'),
135 ('nExpType','<u4'),
136 ('nNTx','<u4'),
137 ('fIpp','<f4'),
138 ('fTxA','<f4'),
139 ('fTxB','<f4'),
140 ('nNumWindows','<u4'),
141 ('nNumTaus','<u4'),
142 ('nCodeType','<u4'),
143 ('nLine6Function','<u4'),
144 ('nLine5Function','<u4'),
145 ('fClock','<f4'),
146 ('nPrePulseBefore','<u4'),
147 ('nPrePulseAfter','<u4'),
148 ('sRangeIPP','<a20'),
149 ('sRangeTxA','<a20'),
150 ('sRangeTxB','<a20'),
151 ])
152
153
154 def __init__(self):
155 pass
156
157 def read(self, fp):
158
159 return 1
160
161 class StructProcessing():
162 size = 0
163 dataType = 0
164 blockSize = 0
165 profilesPerBlock = 0
166 dataBlocksPerFile = 0
167 numWindows = 0
168 processFlags = 0
169 coherentInt = 0
170 incoherentInt = 0
171 totalSpectra = 0
172 struct = numpy.dtype([
173 ('nSize','<u4'),
174 ('nDataType','<u4'),
175 ('nSizeOfDataBlock','<u4'),
176 ('nProfilesperBlock','<u4'),
177 ('nDataBlocksperFile','<u4'),
178 ('nNumWindows','<u4'),
179 ('nProcessFlags','<u4'),
180 ('nCoherentIntegrations','<u4'),
181 ('nIncoherentIntegrations','<u4'),
182 ('nTotalSpectra','<u4')
183 ])
184 samplingWindow = 0
185 structSamplingWindow = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
186 numHeights = 0
187 firstHeight = 0
188 deltaHeight = 0
189 samplesWin = 0
190 spectraComb = 0
191 numCode = 0
192 codes = 0
193 numBaud = 0
194
195 def __init__(self):
196 pass
197
198 def read(self, fp):
199
200 return 1
201
202 class VoltageReader(DataReader):
203 # Este flag indica que la data leida no es continua
204 __jumpDataFlag = False
205
206 __idFile = 0
207
208 __fp = 0
209
210 __startDateTime = 0
211
212 __endDateTime = 0
213
214 __dataType = 0
215
216 __sizeOfFileByHeader = 0
217
218 __listOfPath = []
219
220 filenameList = []
221
222 __flagReadShortHeader = 0
223
224 __lastUTTime = 0
225
226 __maxTimeStep = 5
227
228 __flagResetProcessing = 0
229
230 __flagIsNewFile = 0
231
232 noMoreFiles = 0
233
234 online = 0
235
236 filename = ''
237
238 fileSize = 0
239
240 firstHeaderSize = 0
241
242 basicHeaderSize = 24
243
244 objStructShortHeader = StructShortHeader()
245
246 objStructSystemHeader = StructSystemHeader()
247
248 objStructRadarController = StructRadarController()
249
250 objStructProcessing = StructProcessing()
251
252 __buffer = 0
253
254 __buffer_id = 9999
255
256 def __init__(self):
257 pass
258
259 def __rdSystemHeader(self):
260 header = numpy.fromfile(self.__fp,self.objStructSystemHeader.struct,1)
261 self.objStructSystemHeader.size = header['nSize'][0]
262 self.objStructSystemHeader.numSamples = header['nNumSamples'][0]
263 self.objStructSystemHeader.numProfiles = header['nNumProfiles'][0]
264 self.objStructSystemHeader.numChannels = header['nNumChannels'][0]
265 self.objStructSystemHeader.adcResolution = header['nADCResolution'][0]
266 self.objStructSystemHeader.pciDioBusWidth = header['nPCDIOBusWidth'][0]
267
268 def __rdRadarControllerHeader(self):
269 header = numpy.fromfile(self.__fp,self.objStructRadarController.struct,1)
270 self.objStructRadarController.size = header['nSize'][0]
271 self.objStructRadarController.expType = header['nExpType'][0]
272 self.objStructRadarController.nTx = header['nNTx'][0]
273 self.objStructRadarController.ipp = header['fIpp'][0]
274 self.objStructRadarController.txA = header['fTxA'][0]
275 self.objStructRadarController.txB = header['fTxB'][0]
276 self.objStructRadarController.numWindows = header['nNumWindows'][0]
277 self.objStructRadarController.numTaus = header['nNumTaus'][0]
278 self.objStructRadarController.codeType = header['nCodeType'][0]
279 self.objStructRadarController.line6Function = header['nLine6Function'][0]
280 self.objStructRadarController.line5Fuction = header['nLine5Function'][0]
281 self.objStructRadarController.fClock = header['fClock'][0]
282 self.objStructRadarController.prePulseBefore = header['nPrePulseBefore'][0]
283 self.objStructRadarController.prePulserAfter = header['nPrePulseAfter'][0]
284 self.objStructRadarController.rangeIpp = header['sRangeIPP'][0]
285 self.objStructRadarController.rangeTxA = header['sRangeTxA'][0]
286 self.objStructRadarController.rangeTxB = header['sRangeTxB'][0]
287 # jump Dynamic Radar Controller Header
288 jumpHeader = self.objStructRadarController.size - 116
289 self.__fp.seek(self.__fp.tell() + jumpHeader)
290
291 def __rdProcessingHeader(self):
292 header = numpy.fromfile(self.__fp,self.objStructProcessing.struct,1)
293 self.objStructProcessing.size = header['nSize'][0]
294 self.objStructProcessing.dataType = header['nDataType'][0]
295 self.objStructProcessing.blockSize = header['nSizeOfDataBlock'][0]
296 self.objStructProcessing.profilesPerBlock = header['nProfilesperBlock'][0]
297 self.objStructProcessing.dataBlocksPerFile = header['nDataBlocksperFile'][0]
298 self.objStructProcessing.numWindows = header['nNumWindows'][0]
299 self.objStructProcessing.processFlags = header['nProcessFlags']
300 self.objStructProcessing.coherentInt = header['nCoherentIntegrations'][0]
301 self.objStructProcessing.incoherentInt = header['nIncoherentIntegrations'][0]
302 self.objStructProcessing.totalSpectra = header['nTotalSpectra'][0]
303 self.objStructProcessing.samplingWindow = numpy.fromfile(self.__fp,self.objStructProcessing.structSamplingWindow,self.objStructProcessing.numWindows)
304 self.objStructProcessing.numHeights = numpy.sum(self.objStructProcessing.samplingWindow['nsa'])
305 self.objStructProcessing.firstHeight = self.objStructProcessing.samplingWindow['h0']
306 self.objStructProcessing.deltaHeight = self.objStructProcessing.samplingWindow['dh']
307 self.objStructProcessing.samplesWin = self.objStructProcessing.samplingWindow['nsa']
308 self.objStructProcessing.spectraComb = numpy.fromfile(self.__fp,'u1',2*self.objStructProcessing.totalSpectra)
309 if self.objStructProcessing.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
310 self.objStructProcessing.numCode = numpy.fromfile(self.__fp,'<u4',1)
311 self.objStructProcessing.numBaud = numpy.fromfile(self.__fp,'<u4',1)
312 self.objStructProcessing.codes = numpy.fromfile(self.__fp,'<f4',self.objStructProcessing.numCode*self.objStructProcessing.numBaud).reshape(self.objStructProcessing.numBaud,self.objStructProcessing.numCode)
313
314 def __searchFiles(self,path, startDateTime, endDateTime, set=None, expLabel = "", ext = "*.r"):
315
316 startUtSeconds = time.mktime(startDateTime.timetuple())
317 endUtSeconds = time.mktime(endDateTime.timetuple())
318
319 startYear = startDateTime.timetuple().tm_year
320 endYear = endDateTime.timetuple().tm_year
321
322 startDoy = startDateTime.timetuple().tm_yday
323 endDoy = endDateTime.timetuple().tm_yday
324
325 rangeOfYears = range(startYear,endYear+1)
326
327 listOfListDoys = []
328 if startYear == endYear:
329 doyList = range(startDoy,endDoy+1)
330 else:
331 for year in rangeOfYears:
332 if (year == startYear):
333 listOfListDoys.append(range(startDoy,365+1))
334 elif (year == endYear):
335 listOfListDoys.append(range(1,endDoy+1))
336 else:
337 listOfListDoys.append(range(1,365+1))
338 doyList = []
339 for list in listOfListDoys:
340 doyList = doyList + list
341
342 folders = []
343 for thisPath in os.listdir(path):
344 if os.path.isdir(os.path.join(path,thisPath)):
345 #folders.append(os.path.join(path,thisPath))
346 folders.append(thisPath)
347
348 listOfPath = []
349 dicOfPath = {}
350 for year in rangeOfYears:
351 for doy in doyList:
352 tmp = fnmatch.filter(folders, 'D' + '%4.4d%3.3d' % (year,doy))
353 if len(tmp) == 0:
354 continue
355 if expLabel == '':
356 listOfPath.append(os.path.join(path,tmp[0]))
357 dicOfPath.setdefault(os.path.join(path,tmp[0]))
358 dicOfPath[os.path.join(path,tmp[0])] = []
359 else:
360 listOfPath.append(os.path.join(path,os.path.join(tmp[0],expLabel)))
361 dicOfPath.setdefault(os.path.join(path,os.path.join(tmp[0],expLabel)))
362 dicOfPath[os.path.join(path,os.path.join(tmp[0],expLabel))] = []
363
364
365 filenameList = []
366 for thisPath in listOfPath:
367 fileList = glob.glob1(thisPath, ext)
368 #dicOfPath[thisPath].append(fileList)
369 fileList.sort()
370 for file in fileList:
371 filename = os.path.join(thisPath,file)
372 if self.isThisFileinRange(filename, startUtSeconds, endUtSeconds):
373 filenameList.append(filename)
374
375 self.filenameList = filenameList
376
377 return listOfPath, filenameList
378
379
380 def isThisFileinRange(self, filename, startUTSeconds=None, endUTSeconds=None):
381
382 try:
383 fp = open(filename,'rb')
384 except:
385 raise IOError, "The file %s can't be opened" %(filename)
386
387 if startUTSeconds==None:
388 startUTSeconds = self.startUTCSeconds
389
390 if endUTSeconds==None:
391 endUTSeconds = self.endUTCSeconds
392
393 objShortHeader = StructShortHeader()
394
395 if not(objShortHeader.read(fp)):
396 return 0
397
398 if not ((startUTSeconds <= objShortHeader.utc) and (endUTSeconds >= objShortHeader.utc)):
399 return 0
400
401 return 1
402
403 def __readBasicHeader(self, fp=None):
404
405 if fp == None:
406 fp = self.__fp
407
408 self.objStructShortHeader.read(fp)
409
410
411 def __readFirstHeader(self):
412
413 self.__readBasicHeader()
414 self.__rdSystemHeader()
415 self.__rdRadarControllerHeader()
416 self.__rdProcessingHeader()
417 self.firstHeaderSize = self.objStructShortHeader.size
418
419 data_type=int(numpy.log2((self.objStructProcessing.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
420 if data_type == 0:
421 tmp=numpy.dtype([('real','<i1'),('imag','<i1')])
422 elif data_type == 1:
423 tmp=numpy.dtype([('real','<i2'),('imag','<i2')])
424 elif data_type == 2:
425 tmp=numpy.dtype([('real','<i4'),('imag','<i4')])
426 elif data_type == 3:
427 tmp=numpy.dtype([('real','<i8'),('imag','<i8')])
428 elif data_type == 4:
429 tmp=numpy.dtype([('real','<f4'),('imag','<f4')])
430 elif data_type == 5:
431 tmp=numpy.dtype([('real','<f8'),('imag','<f8')])
432 else:
433 print 'no define data type'
434 tmp = 0
435
436 self.__flagIsNewFile = 0
437 self.__dataType = tmp
438 self.__sizeOfFileByHeader = self.objStructProcessing.dataBlocksPerFile * self.objStructProcessing.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.objStructProcessing.dataBlocksPerFile - 1)
439
440
441
442 def __setNextFileOnline(self):
443 return 0
444
445 def __setNextFileOffline(self):
446
447 idFile = self.__idFile
448 while(True):
449
450 idFile += 1
451
452 if not(idFile < len(self.filenameList)):
453 self.noMoreFiles = 1
454 return 0
455
456 filename = self.filenameList[idFile]
457 fileSize = os.path.getsize(filename)
458 fp = open(filename,'rb')
459
460 currentSize = fileSize - fp.tell()
461 neededSize = self.objStructProcessing.blockSize + self.firstHeaderSize
462
463 if (currentSize < neededSize):
464 continue
465
466 break
467
468 self.__flagIsNewFile = 1
469 self.__idFile = idFile
470 self.filename = filename
471 self.fileSize = fileSize
472 self.__fp = fp
473
474 print 'Setting the file: %s'%self.filename
475
476 return 1
477
478 def __setNextFile(self):
479
480 if self.online:
481 return self.__setNextFileOnline()
482 else:
483 return self.__setNextFileOffline()
484
485 def __setNewBlock(self):
486
487 currentSize = self.fileSize - self.__fp.tell()
488 neededSize = self.objStructProcessing.blockSize + self.basicHeaderSize
489
490 # Bloque Completo
491 if (currentSize >= neededSize):
492 self.__readBasicHeader()
493 return 1
494
495 self.__setNextFile()
496 self.__readFirstHeader()
497
498 deltaTime = self.objStructShortHeader.utc - self.__lastUTTime # check this
499 if deltaTime > self.__maxTimeStep:
500 self.__flagResetProcessing = 1
501
502 return 1
503
504 def __readBlock(self):
505 """Lee el bloque de datos desde la posicion actual del puntero del archivo y
506 actualiza todos los parametros relacionados al bloque de datos (data, time,
507 etc). La data leida es almacenada en el buffer y el contador de datos leidos es
508 seteado a 0
509 """
510
511 pts2read = self.objStructProcessing.profilesPerBlock*self.objStructProcessing.numHeights*self.objStructSystemHeader.numChannels
512
513 data = numpy.fromfile(self.__fp,self.__dataType,pts2read)
514
515 data = data.reshape((self.objStructProcessing.profilesPerBlock, self.objStructProcessing.numHeights, self.objStructSystemHeader.numChannels))
516
517 self.__buffer = data
518
519 self.__buffer_id = 0
520
521 def readNextBlock(self):
522
523 self.__setNewBlock()
524
525 self.__readBlock()
526
527 self.__lastUTTime = self.objStructShortHeader.utc
528
529 def __hasNotDataInBuffer(self):
530 if self.__buffer_id >= self.objStructProcessing.profilesPerBlock:
531 return 1
532
533 return 0
534
535 def getData(self):
536 """Obtiene un unidad de datos del buffer de lectura y es copiada a la clase "Data"
537 con todos los parametros asociados a este. cuando no hay datos en el buffer de
538 lectura es necesario hacer una nueva lectura de los bloques de datos
539 "__readBlock"
540 """
541
542 if self.__hasNotDataInBuffer():
543 self.readNextBlock()
544
545 if self.noMoreFiles == 1:
546 print 'read finished'
547 return None
548
549 data = self.__buffer[self.__buffer_id,:,:]
550
551 #print self.__buffer_id
552
553 self.__buffer_id += 1
554
555 #call setData - to Data Object
556
557 return data
558
559
560 def setup(self, path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r", online = 0):
561
562 if online == 0:
563 pathList, filenameList = self.__searchFiles(path, startDateTime, endDateTime, set, expLabel, ext)
564
565 if len(filenameList) == 0:
566 print 'Do not exist files in range: %s - %s'%(startDateTime.ctime(), endDateTime.ctime())
567 return 0
568
569 # for thisFile in filenameList:
570 # print thisFile
571
572 self.__idFile = -1
573
574 if not(self.__setNextFile()):
575 print "No more files"
576 return 0
577
578 self.__readFirstHeader()
579
580 self.startUTCSeconds = time.mktime(startDateTime.timetuple())
581 self.endUTCSeconds = time.mktime(endDateTime.timetuple())
582
583 self.startYear = startDateTime.timetuple().tm_year
584 self.endYear = endDateTime.timetuple().tm_year
585
586 self.startDoy = startDateTime.timetuple().tm_yday
587 self.endDoy = endDateTime.timetuple().tm_yday
588 #call fillHeaderValues() - to Data Object
589
590 self.__listOfPath = pathList
591 self.filenameList = filenameList
592 self.online = online
General Comments 0
You need to be logged in to leave comments. Login now