##// END OF EJS Templates
schain-jc
RSS
Modificaciones realizadas para mejoras en el Modelo, Funciones de Lectura y Escritura de Voltage
Miguel Valdez -
r120:777f5fd367a9
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,48 +1,57
1
1 import os, sys
2 2 import copy
3 3 import numpy
4 from JROHeader import BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
5 4
6 class Data:
7 def __init__(self):
8 pass
5 path = os.path.split(os.getcwd())[0]
6 sys.path.append(path)
9 7
10 def copy(self, inputObj=None):
11 if inputObj == None:
12 return copy.deepcopy(self)
8 from IO.JROHeader import SystemHeader, RadarControllerHeader
13 9
14 for key in inputObj.__dict__.keys():
15 self.__dict__[key] = inputObj.__dict__[key]
10 class JROData():
16 11
17 def deepcopy(self):
18 return copy.deepcopy(self)
19
20 class JROData(Data):
21 data = None
22 12 # m_BasicHeader = BasicHeader()
23 # m_SystemHeader = SystemHeader()
24 # m_RadarControllerHeader = RadarControllerHeader()
25 13 # m_ProcessingHeader = ProcessingHeader()
14
15 systemHeaderObj = SystemHeader()
16
17 radarControllerHeaderObj = RadarControllerHeader()
18
19 data = None
20
26 21 type = None
27 dataType = None
28 heightList = None
29 channelIndexList = None
30 channelList = None
22
23 dtype = None
24
25 nChannels = None
26
31 27 nHeights = None
28
32 29 nProfiles = None
33 nBlocksPerFile = None
34 nChannels = None
30
31 heightList = None
32
33 channelList = None
34
35 channelIndexList = None
36
35 37 flagNoData = False
36 flagResetProcessing = False
38
39 flagTimeBlock = False
40
41 dataUtcTime = None
37 42
38 43 def __init__(self):
44
39 45 raise ValueError, "This class has not been implemented"
40 46
41 # def updateHeaderFromObj(self):
42 # pass
43 #
44 #
45 # def updateObjFromHeader(self):
46 # pass
47 def copy(self, inputObj=None):
47 48
49 if inputObj == None:
50 return copy.deepcopy(self)
48 51
52 for key in inputObj.__dict__.keys():
53 self.__dict__[key] = inputObj.__dict__[key]
54
55 def deepcopy(self):
56
57 return copy.deepcopy(self) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,27 +1,52
1 import os, sys
1 2 import numpy
3
4 path = os.path.split(os.getcwd())[0]
5 sys.path.append(path)
6
2 7 from JROData import JROData
3 # No deberia importar los Headers
8 from IO.JROHeader import SystemHeader, RadarControllerHeader
9
4 10 class Voltage(JROData):
5 data = None
11
6 12 nCohInt = None
7 13
8 14 def __init__(self):
15 '''
16 Constructor
17 '''
18
19 self.m_RadarControllerHeader = RadarControllerHeader()
20
21 self.m_SystemHeader = SystemHeader()
22
9 23 self.type = "Voltage"
10 24
11 def updateObjFromHeader(self):
12 xi = self.m_ProcessingHeader.firstHeight
13 step = self.m_ProcessingHeader.deltaHeight
14 xf = xi + self.m_ProcessingHeader.numHeights*step
25 #data es un numpy array de 2 dmensiones ( canales, alturas)
26 self.data = None
27
28 self.dtype = None
29
30 self.nChannels = 0
31
32 self.nHeights = 0
33
34 self.nProfiles = None
35
36 self.heightList = None
37
38 self.channelList = None
39
40 self.channelIndexList = None
41
42 self.flagNoData = True
43
44 self.flagTimeBlock = False
45
46 self.dataUtcTime = None
47
48 self.nCohInt = None
49
15 50
16 self.heightList = numpy.arange(xi, xf, step)
17 self.channelIndexList = numpy.arange(self.m_SystemHeader.numChannels)
18 self.channelList = numpy.arange(self.m_SystemHeader.numChannels)
19 51
20 self.nHeights = len(self.heightList)
21 self.nChannels = len(self.channelList)
22 self.nProfiles = self.m_ProcessingHeader.profilesPerBlock
23 self.nBlocksPerFile = self.m_ProcessingHeader.dataBlocksPerFile
24 self.nCohInt = self.m_ProcessingHeader.coherentInt
25 52
No newline at end of file
26 def updateHeaderFromObj(self):
27 pass No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,378 +1,667
1 1 import os, sys
2 2 import glob
3 3 import time
4 4 import numpy
5 5 import fnmatch
6 6 import time, datetime
7 7
8 8 path = os.path.split(os.getcwd())[0]
9 9 sys.path.append(path)
10 10
11 from Model.JROHeader import *
12 from Model.JROData import JROData
11 from JROHeader import *
12 from Data.JROData import JROData
13 13
14 14 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
15 15 """
16 16 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
17 17
18 18 Inputs:
19 19 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
20 20
21 21 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
22 22 segundos contados desde 01/01/1970.
23 23 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
24 24 segundos contados desde 01/01/1970.
25 25
26 26 Return:
27 27 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
28 28 fecha especificado, de lo contrario retorna False.
29 29
30 30 Excepciones:
31 31 Si el archivo no existe o no puede ser abierto
32 32 Si la cabecera no puede ser leida.
33 33
34 34 """
35 m_BasicHeader = BasicHeader()
35 basicHeaderObj = BasicHeader()
36 36
37 37 try:
38 38 fp = open(filename,'rb')
39 39 except:
40 40 raise IOError, "The file %s can't be opened" %(filename)
41 41
42 sts = m_BasicHeader.read(fp)
42 sts = basicHeaderObj.read(fp)
43 43 fp.close()
44 44
45 45 if not(sts):
46 46 print "Skipping the file %s because it has not a valid header" %(filename)
47 47 return 0
48 48
49 if not ((startUTSeconds <= m_BasicHeader.utc) and (endUTSeconds > m_BasicHeader.utc)):
49 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
50 50 return 0
51 51
52 52 return 1
53 53
54 54
55 55
56 56
57 57 class JRODataIO:
58
58 59 c = 3E8
59 m_BasicHeader = BasicHeader()
60 m_SystemHeader = SystemHeader()
61 m_RadarControllerHeader = RadarControllerHeader()
62 m_ProcessingHeader = ProcessingHeader()
60
61 basicHeaderObj = BasicHeader()
62
63 systemHeaderObj = SystemHeader()
64
65 radarControllerHeaderObj = RadarControllerHeader()
66
67 processingHeaderObj = ProcessingHeader()
68
63 69 online = 0
64 dataType = None
70
71 dtype = None
72
65 73 pathList = []
74
66 75 filenameList = []
76
67 77 filename = None
78
68 79 ext = None
69 fileIndex = None
80
70 81 flagNoMoreFiles = 0
82
71 83 flagIsNewFile = 1
72 flagResetProcessing = 0
84
85 flagTimeBlock = 0
86
73 87 flagIsNewBlock = 0
88
74 89 fp = None
90
75 91 firstHeaderSize = 0
92
76 93 basicHeaderSize = 24
94
95 versionFile = 1103
96
77 97 fileSize = None
98
78 99 ippSeconds = None
100
79 101 fileSizeByHeader = None
102
103 fileIndex = None
104
105 profileIndex = None
106
107 blockIndex = None
108
109 nTotalBlocks = None
110
80 111 def __init__(self):
81 112 pass
82 113
83 114 class JRODataReader(JRODataIO):
84 115 def __init__(self):
85 116 pass
86 117
87 118 def createObjByDefault(self):
88 119 """
89 120
90 121 """
91 122 raise ValueError, "This method has not been implemented"
92 123
93 124 def getBlockDimension(self):
94 125
95 126 raise ValueError, "No implemented"
96 127
97 128 def __searchFilesOffLine(self,
98 129 path,
99 130 startDate,
100 131 endDate,
101 132 startTime=datetime.time(0,0,0),
102 133 endTime=datetime.time(23,59,59),
103 134 set=None,
104 135 expLabel="",
105 136 ext=".r"):
106 137 dirList = []
107 138 for thisPath in os.listdir(path):
108 139 if os.path.isdir(os.path.join(path,thisPath)):
109 140 dirList.append(thisPath)
110 141
111 142 if not(dirList):
112 143 return None, None
113 144
114 145 pathList = []
115 146 dateList = []
116 147
117 148 thisDate = startDate
118 149
119 150 while(thisDate <= endDate):
120 151 year = thisDate.timetuple().tm_year
121 152 doy = thisDate.timetuple().tm_yday
122 153
123 154 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
124 155 if len(match) == 0:
125 156 thisDate += datetime.timedelta(1)
126 157 continue
127 158
128 159 pathList.append(os.path.join(path,match[0],expLabel))
129 160 dateList.append(thisDate)
130 161 thisDate += datetime.timedelta(1)
131 162
132 163 filenameList = []
133 164 for index in range(len(pathList)):
134 165
135 166 thisPath = pathList[index]
136 167 fileList = glob.glob1(thisPath, "*%s" %ext)
137 168 fileList.sort()
138 169
139 170 #Busqueda de datos en el rango de horas indicados
140 171 thisDate = dateList[index]
141 172 startDT = datetime.datetime.combine(thisDate, startTime)
142 173 endDT = datetime.datetime.combine(thisDate, endTime)
143 174
144 175 startUtSeconds = time.mktime(startDT.timetuple())
145 176 endUtSeconds = time.mktime(endDT.timetuple())
146 177
147 178 for file in fileList:
148 179
149 180 filename = os.path.join(thisPath,file)
150 181
151 182 if isThisFileinRange(filename, startUtSeconds, endUtSeconds):
152 183 filenameList.append(filename)
153 184
154 185 if not(filenameList):
155 186 return None, None
156 187
157 188 self.filenameList = filenameList
158 189
159 190 return pathList, filenameList
160 191
161 192 def setup(self,dataOutObj=None,
162 193 path=None,startDate=None,
163 194 endDate=None,
164 195 startTime=datetime.time(0,0,0),
165 196 endTime=datetime.time(23,59,59),
166 197 set=0,
167 198 expLabel = "",
168 199 ext = None,
169 200 online = 0):
170 201
171 202 if path == None:
172 203 raise ValueError, "The path is not valid"
173 204
174 205 if ext == None:
175 206 ext = self.ext
176 207
177 208 if dataOutObj == None:
178 209 dataOutObj = self.createObjByDefault()
179 210
180 211 self.dataOutObj = dataOutObj
181 212
182 213 if online:
183 214 pass
184 215
185 216 else:
186 217 print "Searching file in offline mode"
187 218 pathList, filenameList = self.__searchFilesOffLine(path, startDate, endDate, startTime, endTime, set, expLabel, ext)
188 219 if not(pathList):
189 220 print "No files in range: %s - %s"%(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
190 221 return None
191 222 self.fileIndex = -1
192 223 self.pathList = pathList
193 224 self.filenameList = filenameList
194 225
195 226 self.online = online
196 227 ext = ext.lower()
197 228 self.ext = ext
198 229
199 230 if not(self.setNextFile()):
200 231 if (startDate!=None) and (endDate!=None):
201 232 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
202 233 elif startDate != None:
203 234 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
204 235 else:
205 236 print "No files"
206 237
207 238 return None
208 239
209 240 self.updateDataHeader()
210 241
211 242 return self.dataOutObj
212 243
213 244 def __setNextFileOffline(self):
214 245 idFile = self.fileIndex
215 246
216 247 while (True):
217 248 idFile += 1
218 249 if not(idFile < len(self.filenameList)):
219 250 self.flagNoMoreFiles = 1
220 251 print "No more Files"
221 252 return 0
222 253
223 254 filename = self.filenameList[idFile]
224 255
225 256 if not(self.__verifyFile(filename)):
226 257 continue
227 258
228 259 fileSize = os.path.getsize(filename)
229 260 fp = open(filename,'rb')
230 261 break
231 262
232 263 self.flagIsNewFile = 1
233 264 self.fileIndex = idFile
234 265 self.filename = filename
235 266 self.fileSize = fileSize
236 267 self.fp = fp
237 268
238 269 print "Setting the file: %s"%self.filename
239 270
240 271 return 1
241 272
242 273
243 274
244 275 def setNextFile(self):
245 276 if self.fp != None:
246 277 self.fp.close()
247 278
248 279 if self.online:
249 280 newFile = self.__setNextFileOnline()
250 281 else:
251 282 newFile = self.__setNextFileOffline()
252 283
253 284 if not(newFile):
254 285 return 0
255 286
256 287 self.__readFirstHeader()
257 288 self.nReadBlocks = 0
258 289 return 1
259 290
260 291 def __rdProcessingHeader(self, fp=None):
261 292 if fp == None:
262 293 fp = self.fp
263 294
264 self.m_ProcessingHeader.read(fp)
295 self.processingHeaderObj.read(fp)
265 296
266 297 def __rdRadarControllerHeader(self, fp=None):
267 298 if fp == None:
268 299 fp = self.fp
269 300
270 self.m_RadarControllerHeader.read(fp)
301 self.radarControllerHeaderObj.read(fp)
271 302
272 303 def __rdSystemHeader(self, fp=None):
273 304 if fp == None:
274 305 fp = self.fp
275 306
276 self.m_SystemHeader.read(fp)
307 self.systemHeaderObj.read(fp)
277 308
278 309 def __rdBasicHeader(self, fp=None):
279 310 if fp == None:
280 311 fp = self.fp
281 312
282 self.m_BasicHeader.read(fp)
313 self.basicHeaderObj.read(fp)
283 314
284 315
285 316 def __readFirstHeader(self):
286 317 self.__rdBasicHeader()
287 318 self.__rdSystemHeader()
288 319 self.__rdRadarControllerHeader()
289 320 self.__rdProcessingHeader()
290 321
291 self.firstHeaderSize = self.m_BasicHeader.size
322 self.firstHeaderSize = self.basicHeaderObj.size
292 323
293 datatype = int(numpy.log2((self.m_ProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
324 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
294 325 if datatype == 0:
295 326 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
296 327 elif datatype == 1:
297 328 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
298 329 elif datatype == 2:
299 330 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
300 331 elif datatype == 3:
301 332 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
302 333 elif datatype == 4:
303 334 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
304 335 elif datatype == 5:
305 336 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
306 337 else:
307 338 raise ValueError, 'Data type was not defined'
308 339
309 self.dataType = datatype_str
310 self.ippSeconds = 2 * 1000 * self.m_RadarControllerHeader.ipp / self.c
311 self.fileSizeByHeader = self.m_ProcessingHeader.dataBlocksPerFile * self.m_ProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.m_ProcessingHeader.dataBlocksPerFile - 1)
312 # self.dataOutObj.channelList = numpy.arange(self.m_SystemHeader.numChannels)
313 # self.dataOutObj.channelIndexList = numpy.arange(self.m_SystemHeader.numChannels)
340 self.dtype = datatype_str
341 self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
342 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
343 # self.dataOutObj.channelList = numpy.arange(self.systemHeaderObj.numChannels)
344 # self.dataOutObj.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
314 345 self.getBlockDimension()
315 346
316 347
317 348 def __verifyFile(self, filename, msgFlag=True):
318 349 msg = None
319 350 try:
320 351 fp = open(filename, 'rb')
321 352 currentPosition = fp.tell()
322 353 except:
323 354 if msgFlag:
324 355 print "The file %s can't be opened" % (filename)
325 356 return False
326 357
327 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
358 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
328 359
329 360 if neededSize == 0:
330 m_BasicHeader = BasicHeader()
331 m_SystemHeader = SystemHeader()
332 m_RadarControllerHeader = RadarControllerHeader()
333 m_ProcessingHeader = ProcessingHeader()
361 basicHeaderObj = BasicHeader()
362 systemHeaderObj = SystemHeader()
363 radarControllerHeaderObj = RadarControllerHeader()
364 processingHeaderObj = ProcessingHeader()
334 365
335 366 try:
336 if not( m_BasicHeader.read(fp) ): raise ValueError
337 if not( m_SystemHeader.read(fp) ): raise ValueError
338 if not( m_RadarControllerHeader.read(fp) ): raise ValueError
339 if not( m_ProcessingHeader.read(fp) ): raise ValueError
340 data_type = int(numpy.log2((m_ProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
367 if not( basicHeaderObj.read(fp) ): raise ValueError
368 if not( systemHeaderObj.read(fp) ): raise ValueError
369 if not( radarControllerHeaderObj.read(fp) ): raise ValueError
370 if not( processingHeaderObj.read(fp) ): raise ValueError
371 data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
341 372
342 neededSize = m_ProcessingHeader.blockSize + m_BasicHeader.size
373 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
343 374
344 375 except:
345 376 if msgFlag:
346 377 print "\tThe file %s is empty or it hasn't enough data" % filename
347 378
348 379 fp.close()
349 380 return False
350 381 else:
351 382 msg = "\tSkipping the file %s due to it hasn't enough data" %filename
352 383
353 384 fp.close()
354 385 fileSize = os.path.getsize(filename)
355 386 currentSize = fileSize - currentPosition
356 387 if currentSize < neededSize:
357 388 if msgFlag and (msg != None):
358 389 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
359 390 return False
360 391
361 392 return True
362 393
394 class JRODataWriter(JRODataIO):
395
396 """
397 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
398 de los datos siempre se realiza por bloques.
399 """
400
401 blockIndex = 0
363 402
403 setFile = None
364 404
365 def updateDataHeader(self):
366 self.dataOutObj.m_BasicHeader = self.m_BasicHeader.copy()
367 self.dataOutObj.m_ProcessingHeader = self.m_ProcessingHeader.copy()
368 self.dataOutObj.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
369 self.dataOutObj.m_SystemHeader = self.m_SystemHeader.copy()
370 self.dataOutObj.dataType = self.dataType
371 self.dataOutObj.updateObjFromHeader() # actualiza los atributos del objeto de salida de la clase JROData
372 405
406 def __init__(self, dataOutObj=None):
407 raise ValueError, "Not implemented"
408
409
410 def hasAllDataInBuffer(self):
411 raise ValueError, "Not implemented"
412
413
414 def setBlockDimension(self):
415 raise ValueError, "Not implemented"
416
417
418 def writeBlock(self):
419 raise ValueError, "No implemented"
420
421
422 def putData(self):
423 raise ValueError, "No implemented"
424
425
426 def __writeFirstHeader(self):
427 """
428 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
429
430 Affected:
431 __dataType
432
433 Return:
434 None
435 """
436
437 # CALCULAR PARAMETROS
438
439 sizeLongHeader = 0#XXXX
440 self.basicHeaderObj.size = 24 + sizeLongHeader
441
442 self.__writeBasicHeader()
443 self.__wrSystemHeader()
444 self.__wrRadarControllerHeader()
445 self.__wrProcessingHeader()
446 self.dtype = self.dataOutObj.dtype
447
448
449 def __writeBasicHeader(self, fp=None):
450 """
451 Escribe solo el Basic header en el file creado
452
453 Return:
454 None
455 """
456 if fp == None:
457 fp = self.fp
458
459 self.dataOutObj.basicHeaderObj.write(fp)
460
461
462 def __wrSystemHeader(self, fp=None):
463 """
464 Escribe solo el System header en el file creado
465
466 Return:
467 None
468 """
469 if fp == None:
470 fp = self.fp
471
472 self.dataOutObj.systemHeaderObj.write(fp)
473
474
475 def __wrRadarControllerHeader(self, fp=None):
476 """
477 Escribe solo el RadarController header en el file creado
478
479 Return:
480 None
481 """
482 if fp == None:
483 fp = self.fp
484
485 self.dataOutObj.radarControllerHeaderObj.write(fp)
486
487
488 def __wrProcessingHeader(self, fp=None):
489 """
490 Escribe solo el Processing header en el file creado
491
492 Return:
493 None
494 """
495 if fp == None:
496 fp = self.fp
497
498 self.dataOutObj.processingHeaderObj.write(fp)
499
500
501 def setNextFile(self):
502 """
503 Determina el siguiente file que sera escrito
504
505 Affected:
506 self.filename
507 self.subfolder
508 self.fp
509 self.setFile
510 self.flagIsNewFile
511
512 Return:
513 0 : Si el archivo no puede ser escrito
514 1 : Si el archivo esta listo para ser escrito
515 """
516 ext = self.ext
517 path = self.path
518
519 if self.fp != None:
520 self.fp.close()
521
522 timeTuple = time.localtime( self.dataOutObj.dataUtcTime)
523 subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
524
525 doypath = os.path.join( path, subfolder )
526 if not( os.path.exists(doypath) ):
527 os.mkdir(doypath)
528 self.setFile = -1 #inicializo mi contador de seteo
529 else:
530 filesList = os.listdir( doypath )
531 if len( filesList ) > 0:
532 filesList = sorted( filesList, key=str.lower )
533 filen = filesList[-1]
534 # el filename debera tener el siguiente formato
535 # 0 1234 567 89A BCDE (hex)
536 # x YYYY DDD SSS .ext
537 if isNumber( filen[8:11] ):
538 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
539 else:
540 self.setFile = -1
541 else:
542 self.setFile = -1 #inicializo mi contador de seteo
543
544 setFile = self.setFile
545 setFile += 1
546
547 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
548 timeTuple.tm_year,
549 timeTuple.tm_yday,
550 setFile,
551 ext )
552
553 filename = os.path.join( path, subfolder, file )
554
555 fp = open( filename,'wb' )
556
557 self.blockIndex = 0
558
559 #guardando atributos
560 self.filename = filename
561 self.subfolder = subfolder
562 self.fp = fp
563 self.setFile = setFile
564 self.flagIsNewFile = 1
565
566 print 'Writing the file: %s'%self.filename
567
568 self.__writeFirstHeader()
569
570 return 1
571
572
573 def __setNewBlock(self):
574 """
575 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
576
577 Return:
578 0 : si no pudo escribir nada
579 1 : Si escribio el Basic el First Header
580 """
581 if self.fp == None:
582 self.setNextFile()
583
584 if self.flagIsNewFile:
585 return 1
586
587 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
588 self.__writeBasicHeader()
589 return 1
590
591 if not( self.setNextFile() ):
592 return 0
593
594 return 1
595
596
597 def writeNextBlock(self):
598 """
599 Selecciona el bloque siguiente de datos y los escribe en un file
600
601 Return:
602 0 : Si no hizo pudo escribir el bloque de datos
603 1 : Si no pudo escribir el bloque de datos
604 """
605 if not( self.__setNewBlock() ):
606 return 0
607
608 self.writeBlock()
609
610 return 1
611
612
613 def getDataHeader(self):
614 """
615 Obtiene una copia del First Header
616
617 Affected:
618 self.basicHeaderObj
619 self.systemHeaderObj
620 self.radarControllerHeaderObj
621 self.processingHeaderObj
622 self.dtype
623
624 Return:
625 None
626 """
627
628 raise ValueError, "No implemented"
629
630 def setup(self, path, set=0, ext=None):
631 """
632 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
633
634 Inputs:
635 path : el path destino en el cual se escribiran los files a crear
636 format : formato en el cual sera salvado un file
637 set : el setebo del file
638
639 Return:
640 0 : Si no realizo un buen seteo
641 1 : Si realizo un buen seteo
642 """
643
644 if ext == None:
645 ext = self.ext
646
647 ext = ext.lower()
648
649 self.path = path
650 self.setFile = set - 1
651 self.ext = ext
652 #self.format = format
653 self.getDataHeader()
654
655 self.setBlockDimension()
656
657 if not( self.setNextFile() ):
658 print "There isn't a next file"
659 return 0
660
661 return 1
373 662
374 663
375 664
376 665
377 666
378 667
Show file before | Show file after | Hide comments
@@ -1,505 +1,506
1 1 '''
2 2 Created on 23/01/2012
3 3
4 4 @author $Author: vsarmiento $
5 5 @version $Id: HeaderIO.py 37 2012-03-26 22:55:13Z vsarmiento $
6 6 '''
7 7
8 8 import numpy
9 9 import copy
10 10
11 11 class Header:
12 12
13 13 def __init__(self):
14 14 raise
15 15
16 16 def copy(self):
17 17 return copy.deepcopy(self)
18 18
19 19 def read():
20 20 pass
21 21
22 22 def write():
23 23 pass
24 24
25 25 class BasicHeader(Header):
26 26
27 27 size = None
28 28 version = None
29 29 dataBlock = None
30 30 utc = None
31 31 miliSecond = None
32 32 timeZone = None
33 33 dstFlag = None
34 34 errorCount = None
35 35 struct = None
36 36
37 37 def __init__(self):
38
38 39 self.size = 0
39 40 self.version = 0
40 41 self.dataBlock = 0
41 42 self.utc = 0
42 43 self.miliSecond = 0
43 44 self.timeZone = 0
44 45 self.dstFlag = 0
45 46 self.errorCount = 0
46 47 self.struct = numpy.dtype([
47 48 ('nSize','<u4'),
48 49 ('nVersion','<u2'),
49 50 ('nDataBlockId','<u4'),
50 51 ('nUtime','<u4'),
51 52 ('nMilsec','<u2'),
52 53 ('nTimezone','<i2'),
53 54 ('nDstflag','<i2'),
54 55 ('nErrorCount','<u4')
55 56 ])
56 57
57 58
58 59 def read(self, fp):
59 60 try:
60 61 header = numpy.fromfile(fp, self.struct,1)
61 62 self.size = header['nSize'][0]
62 63 self.version = header['nVersion'][0]
63 64 self.dataBlock = header['nDataBlockId'][0]
64 65 self.utc = header['nUtime'][0]
65 66 self.miliSecond = header['nMilsec'][0]
66 67 self.timeZone = header['nTimezone'][0]
67 68 self.dstFlag = header['nDstflag'][0]
68 69 self.errorCount = header['nErrorCount'][0]
69 70 except:
70 71 return 0
71 72
72 73 return 1
73 74
74 75 def write(self, fp):
75 76 headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
76 77 header = numpy.array(headerTuple,self.struct)
77 78 header.tofile(fp)
78 79
79 80 return 1
80 81
81 82 class SystemHeader(Header):
82 83
83 84 size = None
84 numSamples = None
85 numProfiles = None
86 numChannels = None
85 nSamples = None
86 nProfiles = None
87 nChannels = None
87 88 adcResolution = None
88 89 pciDioBusWidth = None
89 90 struct = None
90 91
91 92 def __init__(self):
92 93 self.size = 0
93 self.numSamples = 0
94 self.numProfiles = 0
95 self.numChannels = 0
94 self.nSamples = 0
95 self.nProfiles = 0
96 self.nChannels = 0
96 97 self.adcResolution = 0
97 98 self.pciDioBusWidth = 0
98 99 self.struct = numpy.dtype([
99 100 ('nSize','<u4'),
100 101 ('nNumSamples','<u4'),
101 102 ('nNumProfiles','<u4'),
102 103 ('nNumChannels','<u4'),
103 104 ('nADCResolution','<u4'),
104 105 ('nPCDIOBusWidth','<u4'),
105 106 ])
106 107
107 108
108 109 def read(self, fp):
109 110 try:
110 111 header = numpy.fromfile(fp,self.struct,1)
111 112 self.size = header['nSize'][0]
112 self.numSamples = header['nNumSamples'][0]
113 self.numProfiles = header['nNumProfiles'][0]
114 self.numChannels = header['nNumChannels'][0]
113 self.nSamples = header['nNumSamples'][0]
114 self.nProfiles = header['nNumProfiles'][0]
115 self.nChannels = header['nNumChannels'][0]
115 116 self.adcResolution = header['nADCResolution'][0]
116 117 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
117 118 except:
118 119 return 0
119 120
120 121 return 1
121 122
122 123 def write(self, fp):
123 headerTuple = (self.size,self.numSamples,self.numProfiles,self.numChannels,self.adcResolution,self.pciDioBusWidth)
124 headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
124 125 header = numpy.array(headerTuple,self.struct)
125 126 header.tofile(fp)
126 127
127 128 return 1
128 129
129 130 class RadarControllerHeader(Header):
130 131
131 132 size = None
132 133 expType = None
133 134 nTx = None
134 135 ipp = None
135 136 txA = None
136 137 txB = None
137 numWindows = None
138 nWindows = None
138 139 numTaus = None
139 140 codeType = None
140 141 line6Function = None
141 142 line5Function = None
142 143 fClock = None
143 144 prePulseBefore = None
144 145 prePulserAfter = None
145 146 rangeIpp = None
146 147 rangeTxA = None
147 148 rangeTxB = None
148 149 struct = None
149 150
150 151 def __init__(self):
151 152 self.size = 0
152 153 self.expType = 0
153 154 self.nTx = 0
154 155 self.ipp = 0
155 156 self.txA = 0
156 157 self.txB = 0
157 self.numWindows = 0
158 self.nWindows = 0
158 159 self.numTaus = 0
159 160 self.codeType = 0
160 161 self.line6Function = 0
161 162 self.line5Function = 0
162 163 self.fClock = 0
163 164 self.prePulseBefore = 0
164 165 self.prePulserAfter = 0
165 166 self.rangeIpp = 0
166 167 self.rangeTxA = 0
167 168 self.rangeTxB = 0
168 169 self.struct = numpy.dtype([
169 170 ('nSize','<u4'),
170 171 ('nExpType','<u4'),
171 172 ('nNTx','<u4'),
172 173 ('fIpp','<f4'),
173 174 ('fTxA','<f4'),
174 175 ('fTxB','<f4'),
175 176 ('nNumWindows','<u4'),
176 177 ('nNumTaus','<u4'),
177 178 ('nCodeType','<u4'),
178 179 ('nLine6Function','<u4'),
179 180 ('nLine5Function','<u4'),
180 181 ('fClock','<f4'),
181 182 ('nPrePulseBefore','<u4'),
182 183 ('nPrePulseAfter','<u4'),
183 184 ('sRangeIPP','<a20'),
184 185 ('sRangeTxA','<a20'),
185 186 ('sRangeTxB','<a20'),
186 187 ])
187 188
188 189 self.samplingWindowStruct = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
189 190
190 191 self.samplingWindow = None
191 192 self.numHeights = None
192 193 self.firstHeight = None
193 194 self.deltaHeight = None
194 195 self.samplesWin = None
195 196
196 197 self.numCode = None
197 198 self.numBaud = None
198 199 self.code = None
199 200 self.flip1 = None
200 201 self.flip2 = None
201 202
202 203 self.dynamic = numpy.array([],numpy.dtype('byte'))
203 204
204 205
205 206 def read(self, fp):
206 207 try:
207 208 startFp = fp.tell()
208 209 header = numpy.fromfile(fp,self.struct,1)
209 210 self.size = header['nSize'][0]
210 211 self.expType = header['nExpType'][0]
211 212 self.nTx = header['nNTx'][0]
212 213 self.ipp = header['fIpp'][0]
213 214 self.txA = header['fTxA'][0]
214 215 self.txB = header['fTxB'][0]
215 self.numWindows = header['nNumWindows'][0]
216 self.nWindows = header['nNumWindows'][0]
216 217 self.numTaus = header['nNumTaus'][0]
217 218 self.codeType = header['nCodeType'][0]
218 219 self.line6Function = header['nLine6Function'][0]
219 220 self.line5Function = header['nLine5Function'][0]
220 221 self.fClock = header['fClock'][0]
221 222 self.prePulseBefore = header['nPrePulseBefore'][0]
222 223 self.prePulserAfter = header['nPrePulseAfter'][0]
223 224 self.rangeIpp = header['sRangeIPP'][0]
224 225 self.rangeTxA = header['sRangeTxA'][0]
225 226 self.rangeTxB = header['sRangeTxB'][0]
226 227 # jump Dynamic Radar Controller Header
227 228 jumpFp = self.size - 116
228 229 self.dynamic = numpy.fromfile(fp,numpy.dtype('byte'),jumpFp)
229 230 #pointer backward to dynamic header and read
230 231 backFp = fp.tell() - jumpFp
231 232 fp.seek(backFp)
232 233
233 self.samplingWindow = numpy.fromfile(fp,self.samplingWindowStruct,self.numWindows)
234 self.samplingWindow = numpy.fromfile(fp,self.samplingWindowStruct,self.nWindows)
234 235 self.numHeights = numpy.sum(self.samplingWindow['nsa'])
235 236 self.firstHeight = self.samplingWindow['h0']
236 237 self.deltaHeight = self.samplingWindow['dh']
237 238 self.samplesWin = self.samplingWindow['nsa']
238 239
239 240 self.Taus = numpy.fromfile(fp,'<f4',self.numTaus)
240 241
241 242 if self.codeType != 0:
242 243 self.numCode = numpy.fromfile(fp,'<u4',1)
243 244 self.numBaud = numpy.fromfile(fp,'<u4',1)
244 245 self.code = numpy.empty([self.numCode,self.numBaud],dtype='u1')
245 246 tempList = []
246 247 for ic in range(self.numCode):
247 248 temp = numpy.fromfile(fp,'u1',4*numpy.ceil(self.numBaud/32.))
248 249 tempList.append(temp)
249 250 self.code[ic] = numpy.unpackbits(temp[::-1])[-1*self.numBaud:]
250 251 self.code = 2.0*self.code - 1.0
251 252
252 253 if self.line5Function == RCfunction.FLIP:
253 254 self.flip1 = numpy.fromfile(fp,'<u4',1)
254 255
255 256 if self.line6Function == RCfunction.FLIP:
256 257 self.flip2 = numpy.fromfile(fp,'<u4',1)
257 258
258 259 endFp = self.size + startFp
259 260 jumpFp = endFp - fp.tell()
260 261 if jumpFp > 0:
261 262 fp.seek(jumpFp)
262 263
263 264 except:
264 265 return 0
265 266
266 267 return 1
267 268
268 269 def write(self, fp):
269 270 headerTuple = (self.size,
270 271 self.expType,
271 272 self.nTx,
272 273 self.ipp,
273 274 self.txA,
274 275 self.txB,
275 self.numWindows,
276 self.nWindows,
276 277 self.numTaus,
277 278 self.codeType,
278 279 self.line6Function,
279 280 self.line5Function,
280 281 self.fClock,
281 282 self.prePulseBefore,
282 283 self.prePulserAfter,
283 284 self.rangeIpp,
284 285 self.rangeTxA,
285 286 self.rangeTxB)
286 287
287 288 header = numpy.array(headerTuple,self.struct)
288 289 header.tofile(fp)
289 290
290 291 dynamic = self.dynamic
291 292 dynamic.tofile(fp)
292 293
293 294 return 1
294 295
295 296
296 297
297 298 class ProcessingHeader(Header):
298 299
299 300 size = None
300 dataType = None
301 dtype = None
301 302 blockSize = None
302 303 profilesPerBlock = None
303 304 dataBlocksPerFile = None
304 numWindows = None
305 nWindows = None
305 306 processFlags = None
306 coherentInt = None
307 incoherentInt = None
307 nCohInt = None
308 nIncohInt = None
308 309 totalSpectra = None
309 310 struct = None
310 311 flag_dc = None
311 312 flag_cspc = None
312 313
313 314 def __init__(self):
314 315 self.size = 0
315 316 self.dataType = 0
316 317 self.blockSize = 0
317 318 self.profilesPerBlock = 0
318 319 self.dataBlocksPerFile = 0
319 self.numWindows = 0
320 self.nWindows = 0
320 321 self.processFlags = 0
321 self.coherentInt = 0
322 self.incoherentInt = 0
322 self.nCohInt = 0
323 self.nIncohInt = 0
323 324 self.totalSpectra = 0
324 325 self.struct = numpy.dtype([
325 326 ('nSize','<u4'),
326 327 ('nDataType','<u4'),
327 328 ('nSizeOfDataBlock','<u4'),
328 329 ('nProfilesperBlock','<u4'),
329 330 ('nDataBlocksperFile','<u4'),
330 331 ('nNumWindows','<u4'),
331 332 ('nProcessFlags','<u4'),
332 333 ('nCoherentIntegrations','<u4'),
333 334 ('nIncoherentIntegrations','<u4'),
334 335 ('nTotalSpectra','<u4')
335 336 ])
336 337 self.samplingWindow = 0
337 338 self.structSamplingWindow = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
338 339 self.numHeights = 0
339 340 self.firstHeight = 0
340 341 self.deltaHeight = 0
341 342 self.samplesWin = 0
342 343 self.spectraComb = 0
343 344 self.numCode = 0
344 345 self.code = 0
345 346 self.numBaud = 0
346 347 self.shif_fft = False
347 348 self.flag_dc = False
348 349 self.flag_cspc = False
349 350
350 351 def read(self, fp):
351 352 try:
352 353 header = numpy.fromfile(fp,self.struct,1)
353 354 self.size = header['nSize'][0]
354 355 self.dataType = header['nDataType'][0]
355 356 self.blockSize = header['nSizeOfDataBlock'][0]
356 357 self.profilesPerBlock = header['nProfilesperBlock'][0]
357 358 self.dataBlocksPerFile = header['nDataBlocksperFile'][0]
358 self.numWindows = header['nNumWindows'][0]
359 self.nWindows = header['nNumWindows'][0]
359 360 self.processFlags = header['nProcessFlags']
360 self.coherentInt = header['nCoherentIntegrations'][0]
361 self.incoherentInt = header['nIncoherentIntegrations'][0]
361 self.nCohInt = header['nCoherentIntegrations'][0]
362 self.nIncohInt = header['nIncoherentIntegrations'][0]
362 363 self.totalSpectra = header['nTotalSpectra'][0]
363 self.samplingWindow = numpy.fromfile(fp,self.structSamplingWindow,self.numWindows)
364 self.samplingWindow = numpy.fromfile(fp,self.structSamplingWindow,self.nWindows)
364 365 self.numHeights = numpy.sum(self.samplingWindow['nsa'])
365 366 self.firstHeight = self.samplingWindow['h0']
366 367 self.deltaHeight = self.samplingWindow['dh']
367 368 self.samplesWin = self.samplingWindow['nsa']
368 369 self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
369 370
370 371 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
371 372 self.numCode = numpy.fromfile(fp,'<u4',1)
372 373 self.numBaud = numpy.fromfile(fp,'<u4',1)
373 374 self.code = numpy.fromfile(fp,'<f4',self.numCode*self.numBaud).reshape(self.numBaud,self.numCode)
374 375
375 376 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
376 377 self.shif_fft = True
377 378 else:
378 379 self.shif_fft = False
379 380
380 381 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
381 382 self.flag_dc = True
382 383
383 384 nChannels = 0
384 385 nPairs = 0
385 386 pairList = []
386 387
387 388 for i in range( 0, self.totalSpectra*2, 2 ):
388 389 if self.spectraComb[i] == self.spectraComb[i+1]:
389 390 nChannels = nChannels + 1 #par de canales iguales
390 391 else:
391 392 nPairs = nPairs + 1 #par de canales diferentes
392 393 pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
393 394
394 395 self.flag_cspc = False
395 396 if nPairs > 0:
396 397 self.flag_cspc = True
397 398
398 399 except:
399 400 return 0
400 401
401 402 return 1
402 403
403 404 def write(self, fp):
404 405 headerTuple = (self.size,
405 406 self.dataType,
406 407 self.blockSize,
407 408 self.profilesPerBlock,
408 409 self.dataBlocksPerFile,
409 self.numWindows,
410 self.nWindows,
410 411 self.processFlags,
411 self.coherentInt,
412 self.incoherentInt,
412 self.nCohInt,
413 self.nIncohInt,
413 414 self.totalSpectra)
414 415
415 416 header = numpy.array(headerTuple,self.struct)
416 417 header.tofile(fp)
417 418
418 if self.numWindows != 0:
419 if self.nWindows != 0:
419 420 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
420 421 samplingWindow = numpy.array(sampleWindowTuple,self.structSamplingWindow)
421 422 samplingWindow.tofile(fp)
422 423
423 424
424 425 if self.totalSpectra != 0:
425 426 spectraComb = numpy.array([],numpy.dtype('u1'))
426 427 spectraComb = self.spectraComb
427 428 spectraComb.tofile(fp)
428 429
429 430
430 431 if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
431 432 numCode = self.numCode
432 433 numCode.tofile(fp)
433 434
434 435 numBaud = self.numBaud
435 436 numBaud.tofile(fp)
436 437
437 438 code = self.code.reshape(numCode*numBaud)
438 439 code.tofile(fp)
439 440
440 441 return 1
441 442
442 443 class RCfunction:
443 444 NONE=0
444 445 FLIP=1
445 446 CODE=2
446 447 SAMPLING=3
447 448 LIN6DIV256=4
448 449 SYNCHRO=5
449 450
450 451 class nCodeType:
451 452 NONE=0
452 453 USERDEFINE=1
453 454 BARKER2=2
454 455 BARKER3=3
455 456 BARKER4=4
456 457 BARKER5=5
457 458 BARKER7=6
458 459 BARKER11=7
459 460 BARKER13=8
460 461 AC128=9
461 462 COMPLEMENTARYCODE2=10
462 463 COMPLEMENTARYCODE4=11
463 464 COMPLEMENTARYCODE8=12
464 465 COMPLEMENTARYCODE16=13
465 466 COMPLEMENTARYCODE32=14
466 467 COMPLEMENTARYCODE64=15
467 468 COMPLEMENTARYCODE128=16
468 469 CODE_BINARY28=17
469 470
470 471 class PROCFLAG:
471 472 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
472 473 DECODE_DATA = numpy.uint32(0x00000002)
473 474 SPECTRA_CALC = numpy.uint32(0x00000004)
474 475 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
475 476 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
476 477 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
477 478
478 479 DATATYPE_CHAR = numpy.uint32(0x00000040)
479 480 DATATYPE_SHORT = numpy.uint32(0x00000080)
480 481 DATATYPE_LONG = numpy.uint32(0x00000100)
481 482 DATATYPE_INT64 = numpy.uint32(0x00000200)
482 483 DATATYPE_FLOAT = numpy.uint32(0x00000400)
483 484 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
484 485
485 486 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
486 487 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
487 488 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
488 489
489 490 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
490 491 DEFLIP_DATA = numpy.uint32(0x00010000)
491 492 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
492 493
493 494 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
494 495 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
495 496 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
496 497 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
497 498 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
498 499
499 500 EXP_NAME_ESP = numpy.uint32(0x00200000)
500 501 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
501 502
502 503 OPERATION_MASK = numpy.uint32(0x0000003F)
503 504 DATATYPE_MASK = numpy.uint32(0x00000FC0)
504 505 DATAARRANGE_MASK = numpy.uint32(0x00007000)
505 506 ACQ_SYS_MASK = numpy.uint32(0x001C0000) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,41 +1,494
1 '''
2 Created on 23/01/2012
3
4 @author $Author: dsuarez $
5 @version $Id: VoltageIO.py 110 2012-07-19 15:18:18Z dsuarez $
6 '''
1 7
2 8 import os, sys
3 9 import numpy
4 10 import glob
5 11 import fnmatch
6 12 import time, datetime
7 13
8 14 path = os.path.split(os.getcwd())[0]
9 15 sys.path.append(path)
10 16
11 from Model.JROHeader import *
12 from Model.Voltage import Voltage
13
14 from IO.JRODataIO import JRODataReader
17 from JROHeader import *
18 from JRODataIO import JRODataReader
19 from JRODataIO import JRODataWriter
15 20
21 from Data.Voltage import Voltage
16 22
17 23 class VoltageReader(JRODataReader):
24 """
25 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
26 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
27 perfiles*alturas*canales) son almacenados en la variable "buffer".
28
29 perfiles * alturas * canales
30
31 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
32 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
33 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
34 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
35
36 Example:
37
38 dpath = "/home/myuser/data"
39
40 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
41
42 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
43
44 readerObj = VoltageReader()
45
46 readerObj.setup(dpath, startTime, endTime)
47
48 while(True):
49
50 #to get one profile
51 profile = readerObj.getData()
52
53 #print the profile
54 print profile
55
56 #If you want to see all datablock
57 print readerObj.datablock
58
59 if readerObj.flagNoMoreFiles:
60 break
61
62 """
18 63 dataOutObj = None
64
19 65 datablock = None
66
20 67 ext = ".r"
68
21 69 optchar = "D"
22 70
71
23 72 def __init__(self, dataOutObj=None):
73 """
74 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
75
76 Input:
77 dataOutObj : Objeto de la clase Voltage. Este objeto sera utilizado para
78 almacenar un perfil de datos cada vez que se haga un requerimiento
79 (getData). El perfil sera obtenido a partir del buffer de datos,
80 si el buffer esta vacio se hara un nuevo proceso de lectura de un
81 bloque de datos.
82 Si este parametro no es pasado se creara uno internamente.
83
84 Variables afectadas:
85 self.dataOutObj
86
87 Return:
88 None
89 """
90
24 91 self.datablock = None
25 #Por herencia no necesito instanciar nuevamente estos objetos
26 #self.m_BasicHeader = BasicHeader()
27 #self.m_SystemHeader = SystemHeader()
28 #self.m_RadarControllerHeader = RadarControllerHeader()
29 #self.m_ProcessingHeader = ProcessingHeader()
92
93 self.utc = 0
94
95 self.ext = ".r"
96
97 self.optchar = "D"
98
99 self.basicHeaderObj = BasicHeader()
100
101 self.systemHeaderObj = SystemHeader()
102
103 self.radarControllerHeaderObj = RadarControllerHeader()
104
105 self.processingHeaderObj = ProcessingHeader()
106
30 107 self.online = 0
31 108
109 self.fp = None
110
111 self.idFile = None
112
113 self.startDateTime = None
114
115 self.endDateTime = None
116
117 self.dtype = None
118
119 self.fileSizeByHeader = None
120
121 self.filenameList = []
122
123 self.filename = None
124
125 self.fileSize = None
126
127 self.firstHeaderSize = 0
128
129 self.basicHeaderSize = 24
130
131 self.pathList = []
132
133 self.filenameList = []
134
135 self.lastUTTime = 0
136
137 self.maxTimeStep = 30
138
139 self.flagNoMoreFiles = 0
140
141 self.set = 0
142
143 self.path = None
144
145 self.profileIndex = 9999
146
147 self.delay = 3 #seconds
148
149 self.nTries = 3 #quantity tries
150
151 self.nFiles = 3 #number of files for searching
152
153 self.nReadBlocks = 0
154
155 self.flagIsNewFile = 1
156
157 self.ippSeconds = 0
158
159 self.flagTimeBlock = 0
160
161 self.flagIsNewBlock = 0
162
163 self.nTotalBlocks = 0
164
165 self.blocksize = 0
166
32 167 def createObjByDefault(self):
168
33 169 dataObj = Voltage()
34 170
35 171 return dataObj
36 172
173 def __hasNotDataInBuffer(self):
174 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
175 return 1
176 return 0
177
178
37 179 def getBlockDimension(self):
38 pts2read = self.m_ProcessingHeader.profilesPerBlock * self.m_ProcessingHeader.numHeights * self.m_SystemHeader.numChannels
180 """
181 Obtiene la cantidad de puntos a leer por cada bloque de datos
182
183 Affected:
184 self.blocksize
185
186 Return:
187 None
188 """
189 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.numHeights * self.systemHeaderObj.numChannels
39 190 self.blocksize = pts2read
40 191
41 192
193 def readBlock(self):
194 """
195 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
196 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
197 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
198 es seteado a 0
199
200 Inputs:
201 None
202
203 Return:
204 None
205
206 Affected:
207 self.profileIndex
208 self.datablock
209 self.flagIsNewFile
210 self.flagIsNewBlock
211 self.nTotalBlocks
212
213 Exceptions:
214 Si un bloque leido no es un bloque valido
215 """
216
217 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
218
219 try:
220 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.numHeights, self.systemHeaderObj.numChannels) )
221 except:
222 print "The read block (%3d) has not enough data" %self.nReadBlocks
223 return 0
224
225 junk = numpy.transpose(junk, (2,0,1))
226 self.datablock = junk['real'] + junk['imag']*1j
227
228 self.profileIndex = 0
229
230 self.flagIsNewFile = 0
231 self.flagIsNewBlock = 1
232
233 self.nTotalBlocks += 1
234 self.nReadBlocks += 1
235
236 return 1
237
238
239 def getData(self):
240 """
241 getData obtiene una unidad de datos del buffer de lectura y la copia a la clase "Voltage"
242 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
243 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
244
245 Ademas incrementa el contador del buffer en 1.
246
247 Return:
248 data : retorna un perfil de voltages (alturas * canales) copiados desde el
249 buffer. Si no hay mas archivos a leer retorna None.
250
251 Variables afectadas:
252 self.dataOutObj
253 self.profileIndex
254
255 Affected:
256 self.dataOutObj
257 self.profileIndex
258 self.flagTimeBlock
259 self.flagIsNewBlock
260 """
261 if self.flagNoMoreFiles: return 0
262
263 self.flagTimeBlock = 0
264 self.flagIsNewBlock = 0
265
266 if self.__hasNotDataInBuffer():
267
268 if not( self.readNextBlock() ):
269 return 0
270
271 self.updateDataHeader()
272
273 if self.flagNoMoreFiles == 1:
274 print 'Process finished'
275 return 0
276
277 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
278
279 if self.datablock == None:
280 self.dataOutObj.flagNoData = True
281 return 0
282
283 self.dataOutObj.data = self.datablock[:,self.profileIndex,:]
284
285 self.dataOutObj.dtype = self.dtype
286
287 self.dataOutObj.nChannels = self.systemHeaderObj.numChannels
288
289 self.dataOutObj.nHeights = self.processingHeaderObj.numHeights
290
291 self.dataOutObj.nProfiles = self.processingHeaderObj.profilesPerBlock
292
293 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.numHeights*self.processingHeaderObj.deltaHeight
294
295 self.dataOutObj.heightList = range(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
296
297 self.dataOutObj.channelList = range(self.systemHeaderObj.numChannels)
298
299 self.dataOutObj.channelIndexList = range(self.systemHeaderObj.numChannels)
300
301 self.dataOutObj.flagNoData = True
302
303 self.dataOutObj.flagTimeBlock = self.flagTimeBlock
304
305 self.dataOutObj.dataUtcTime = self.basicHeaderObj.utc + self.profileIndex * self.ippSeconds
306
307 self.dataOutObj.nCohInt = self.processingHeaderObj.nCohInt
308
309 self.profileIndex += 1
310
311 self.dataOutObj.systemHeaderObj = self.systemHeaderObj.copy()
312
313 self.dataOutObj.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
314
315 return 1
316
317
318 class VoltageWriter(JRODataWriter):
319 """
320 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
321 de los datos siempre se realiza por bloques.
322 """
323 __configHeaderFile = 'wrSetHeadet.txt'
324
325 dataOutObj = None
326
327 ext = ".r"
328
329 optchar = "D"
330
331 datablock = None
332
333 profileIndex = 0
334
335 shapeBuffer = None
336
337
338 def __init__(self, dataOutObj=None):
339 """
340 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
341
342 Affected:
343 self.dataOutObj
344
345 Return: None
346 """
347 if dataOutObj == None:
348 dataOutObj = Voltage()
349
350 if not( isinstance(dataOutObj, Voltage) ):
351 raise ValueError, "in VoltageReader, dataOutObj must be an Spectra class object"
352
353 self.dataOutObj = dataOutObj
354
355
356 def hasAllDataInBuffer(self):
357 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
358 return 1
359 return 0
360
361
362 def setBlockDimension(self):
363 """
364 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
365
366 Affected:
367 self.shape_spc_Buffer
368 self.shape_cspc_Buffer
369 self.shape_dc_Buffer
370
371 Return: None
372 """
373 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
374 self.processingHeaderObj.numHeights,
375 self.systemHeaderObj.numChannels )
376
377 self.datablock = numpy.zeros((self.systemHeaderObj.numChannels,
378 self.processingHeaderObj.profilesPerBlock,
379 self.processingHeaderObj.numHeights),
380 dtype=numpy.dtype('complex'))
381
382
383 def writeBlock(self):
384 """
385 Escribe el buffer en el file designado
386
387 Affected:
388 self.profileIndex
389 self.flagIsNewFile
390 self.flagIsNewBlock
391 self.nTotalBlocks
392 self.blockIndex
393
394 Return: None
395 """
396 data = numpy.zeros( self.shapeBuffer, self.dtype )
397
398 junk = numpy.transpose(self.datablock, (1,2,0))
399
400 data['real'] = junk.real
401 data['imag'] = junk.imag
402
403 data = data.reshape( (-1) )
404
405 data.tofile( self.fp )
406
407 self.datablock.fill(0)
408
409 self.profileIndex = 0
410 self.flagIsNewFile = 0
411 self.flagIsNewBlock = 1
412
413 self.blockIndex += 1
414 self.nTotalBlocks += 1
415
416 def putData(self):
417 """
418 Setea un bloque de datos y luego los escribe en un file
419
420 Affected:
421 self.flagIsNewBlock
422 self.profileIndex
423
424 Return:
425 0 : Si no hay data o no hay mas files que puedan escribirse
426 1 : Si se escribio la data de un bloque en un file
427 """
428 self.flagIsNewBlock = 0
429
430 if self.dataOutObj.flagNoData:
431 return 0
432
433 if self.dataOutObj.flagTimeBlock:
434
435 self.datablock.fill(0)
436 self.profileIndex = 0
437 self.setNextFile()
438
439 self.datablock[:,self.profileIndex,:] = self.dataOutObj.data
440
441 self.profileIndex += 1
442
443 if self.hasAllDataInBuffer():
444 #if self.flagIsNewFile:
445 self.getDataHeader()
446 self.writeNextBlock()
447
448 if self.flagNoMoreFiles:
449 #print 'Process finished'
450 return 0
451
452 return 1
453
454 def getDataHeader(self):
455
456 """
457 Obtiene una copia del First Header
458
459 Affected:
460 self.systemHeaderObj
461 self.radarControllerHeaderObj
462 self.dtype
463
464 Return:
465 None
466 """
467
468 # CALCULAR PARAMETROS
469
470 self.systemHeaderObj = self.dataOutObj.systemHeaderObj.copy()
471 self.radarControllerHeaderObj = self.dataOutObj.radarControllerHeaderObj.copy()
472
473 self.basicHeaderObj.size = self.basicHeaderSize
474 self.basicHeaderObj.version = self.versionFile
475 self.basicHeaderObj.dataBlock = self.nTotalBlocks
476 self.basicHeaderObj.utc = self.dataOutObj.dataUtcTime
477 self.basicHeaderObj.miliSecond = 0
478 self.basicHeaderObj.timeZone = 0
479 self.basicHeaderObj.dstFlag = 0
480 self.basicHeaderObj.errorCount = 0
481
482 self.processingHeaderObj.size = 0
483 self.processingHeaderObj.dtype = self.dataOutObj.dtype
484 self.processingHeaderObj.blockSize = 0
485 self.processingHeaderObj.profilesPerBlock = 0
486 self.processingHeaderObj.dataBlocksPerFile = 0
487 self.processingHeaderObj.numWindows = 0
488 self.processingHeaderObj.processFlags = 0
489 self.processingHeaderObj.coherentInt = 0
490 self.processingHeaderObj.incoherentInt = 0
491 self.processingHeaderObj.totalSpectra = 0
492
493 self.dtype = self.dataOutObj.dtype
494 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,133 +1,133
1 1 import os
2 2 import sys
3 3 import numpy
4 4
5 5 path = os.path.split(os.getcwd())[0]
6 6 sys.path.append(path)
7 7
8 from Model.Voltage import Voltage
8 from Data.Voltage import Voltage
9 9
10 10
11 11 class VoltageProcessor:
12 12 dataInObj = None
13 13 dataOutObj = None
14 14 integratorObjIndex = None
15 15 writerObjIndex = None
16 16 integratorObjList = None
17 17 writerObjList = None
18 18
19 19 def __init__(self):
20 20 self.integratorObjIndex = None
21 21 self.writerObjIndex = None
22 22 self.integratorObjList = []
23 23 self.writerObjList = []
24 24
25 25 def setup(self,dataInObj=None,dataOutObj=None):
26 26 self.dataInObj = dataInObj
27 27
28 28 if self.dataOutObj == None:
29 29 dataOutObj = Voltage()
30 30
31 31 self.dataOutObj = dataOutObj
32 32
33 33 return self.dataOutObj
34 34
35 35 def init(self):
36 36 self.integratorObjIndex = 0
37 37 self.writerObjIndex = 0
38 38 # No necesita copiar en cada init() los atributos de dataInObj
39 39 # la copia deberia hacerse por cada nuevo bloque de datos
40 40
41 41 def addIntegrator(self,N,timeInterval):
42 42 objCohInt = CoherentIntegrator(N,timeInterval)
43 43 self.integratorObjList.append(objCohInt)
44 44
45 45 def addWriter(self):
46 46 pass
47 47
48 48 def integrator(self, N=None, timeInterval=None):
49 49 if self.dataOutObj.flagNoData:
50 50 return 0
51 51 if len(self.integratorObjList) <= self.integratorObjIndex:
52 52 self.addIntegrator(N,timeInterval)
53 53
54 54 myCohIntObj = self.integratorObjList[self.integratorObjIndex]
55 55 myCohIntObj.exe(data=self.dataOutObj.data,timeOfData=None)
56 56
57 57 pass
58 58
59 59 def writeData(self):
60 60 pass
61 61
62 62 class CoherentIntegrator:
63 63
64 64 integ_counter = None
65 65 data = None
66 66 navg = None
67 67 buffer = None
68 68 nCohInt = None
69 69
70 70 def __init__(self, N=None,timeInterval=None):
71 71
72 72 self.data = None
73 73 self.navg = None
74 74 self.buffer = None
75 75 self.timeOut = None
76 76 self.exitCondition = False
77 77 self.isReady = False
78 78 self.nCohInt = N
79 79 self.integ_counter = 0
80 80 if timeInterval!=None:
81 81 self.timeIntervalInSeconds = timeInterval * 60. #if (type(timeInterval)!=integer) -> change this line
82 82
83 83 if ((timeInterval==None) and (N==None)):
84 84 raise ValueError, "N = None ; timeInterval = None"
85 85
86 86 if timeInterval == None:
87 87 self.timeFlag = False
88 88 else:
89 89 self.timeFlag = True
90 90
91 91 def exe(self, data, timeOfData):
92 92
93 93 if self.timeFlag:
94 94 if self.timeOut == None:
95 95 self.timeOut = timeOfData + self.timeIntervalInSeconds
96 96
97 97 if timeOfData < self.timeOut:
98 98 if self.buffer == None:
99 99 self.buffer = data
100 100 else:
101 101 self.buffer = self.buffer + data
102 102 self.integ_counter += 1
103 103 else:
104 104 self.exitCondition = True
105 105
106 106 else:
107 107 if self.integ_counter < self.nCohInt:
108 108 if self.buffer == None:
109 109 self.buffer = data
110 110 else:
111 111 self.buffer = self.buffer + data
112 112
113 113 self.integ_counter += 1
114 114
115 115 if self.integ_counter == self.nCohInt:
116 116 self.exitCondition = True
117 117
118 118 if self.exitCondition:
119 119 self.data = self.buffer
120 120 self.navg = self.integ_counter
121 121 self.isReady = True
122 122 self.buffer = None
123 123 self.timeOut = None
124 124 self.integ_counter = 0
125 125 self.exitCondition = False
126 126
127 127 if self.timeFlag:
128 128 self.buffer = data
129 129 self.timeOut = timeOfData + self.timeIntervalInSeconds
130 130 else:
131 131 self.isReady = False
132 132
133 133
Show file before | Show file after | Hide comments
@@ -1,64 +1,65
1 1
2 2 import os, sys
3 3 import time, datetime
4 4
5 5 path = os.path.split(os.getcwd())[0]
6 6 sys.path.append(path)
7 7
8 8 from Data.Voltage import Voltage
9 9 from IO.VoltageIO import *
10 10
11 11 from Processing.VoltageProcessor import *
12 12
13 13
14 14
15 15 class TestSChain():
16 16
17 17 def __init__(self):
18 18 self.setValues()
19 19 self.createObjects()
20 20 self.testSChain()
21 21
22 22 def setValues(self):
23 23 self.path = "/Users/jro/Documents/RadarData/MST_ISR/MST"
24 self.path = "/home/roj-idl71/Data/RAWDATA/IMAGING"
24 25
25 26 self.wrpath = "/Users/jro/Documents/RadarData/wr_data"
26 27
27 28 self.startDate = datetime.date(2009,1,17)
28 29 self.endDate = datetime.date(2009,1,17)
29 30
30 31 self.startTime = datetime.time(0,0,0)
31 32 self.endTime = datetime.time(14,1,1)
32 33
33 34 def createObjects(self):
35
34 36 self.readerObj = VoltageReader()
35 self.voltProcObj = VoltageProcessor()
36 37
37 38 self.voltObj1 = self.readerObj.setup(
38 39 path = self.path,
39 40 startDate = self.startDate,
40 41 endDate = self.endDate,
41 42 startTime = self.startTime,
42 43 endTime = self.endTime,
43 44 expLabel = '',
44 45 online = 0)
45 46
46 47
47 48
48 49 def testSChain(self):
49 50
50 51 ini = time.time()
51 52
52 53 while(True):
53 54 self.readerObj.getData()
54 55
55 56 if self.readerObj.flagNoMoreFiles:
56 57 break
57 58
58 59 if self.readerObj.flagIsNewBlock:
59 60 print 'Block No %04d, Time: %s' %(self.readerObj.nTotalBlocks,
60 61 datetime.datetime.fromtimestamp(self.readerObj.m_BasicHeader.utc),)
61 62
62 63
63 64 if __name__ == '__main__':
64 65 TestSChain() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now