##// END OF EJS Templates
schain-jc
RSS
El modulo de búsqueda de archivos fue corregido
Miguel Valdez -
r12:d0fb1dc19c25
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,34 +1,34
1 1 '''
2 2 Created on 23/01/2012
3 3
4 4 @author: danielangelsuarezmunoz
5 5 '''
6 6
7 7 import Voltage
8 8 import datetime
9 9 import time
10 10
11 11 objReader = Voltage.VoltageReader()
12 12
13 13
14 14 path = '/home/roj-idl71/Data/RAWDATA/DP_Faraday/'
15 #path = '/remote/puma/2011_03/DP_Faraday'
16
17
15 18 startDateTime = datetime.datetime(2011,3,11,16,0,0)
16 endDateTime = datetime.datetime(2011,3,11,20,1,0)
17 set = None
18 expLabel = ''
19 ext = '*.r'
19 endDateTime = datetime.datetime(2011,3,12,20,1,0)
20 20
21 21 t0 = time.time()
22 22 objReader.setup(path, startDateTime, endDateTime)
23 #print time.time() - t0
23 print time.time() - t0
24 24
25 25
26 26 while(not(objReader.noMoreFiles)):
27 27
28 28 objReader.getData()
29 29 # if objReader.flagIsNewFile:
30 30 # print objReader.m_BasicHeader.dataBlock
31 31 #print objReader.objStructShortHeader.dataBlock
32 32 #print time.localtime(objReader.m_BasicHeader.utc)
33 33
34 34 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,425 +1,445
1 1 '''
2 2 Created on 23/01/2012
3 3
4 4 @author: danielangelsuarezmunoz
5 5 '''
6 6
7 7
8 8 import os, sys
9 9 import numpy
10 10 import glob
11 11 import fnmatch
12 12 import time
13 13 import datetime
14 14
15 15 from Header import *
16 16 from Data import DataReader
17 17 from Data import DataWriter
18 18
19 19 path = os.path.split(os.getcwd())[0]
20 20 sys.path.append(path)
21 21
22 22 from Model.Voltage import Voltage
23 23
24 24 class VoltageReader(DataReader):
25 25
26 26 __idFile = None
27 27
28 28 __fp = None
29 29
30 30 __startDateTime = None
31 31
32 32 __endDateTime = None
33 33
34 34 __dataType = None
35 35
36 36 __fileSizeByHeader = 0
37 37
38 38 __pathList = []
39 39
40 40 filenameList = []
41 41
42 42 __lastUTTime = 0
43 43
44 44 __maxTimeStep = 5
45 45
46 46 flagResetProcessing = 0
47 47
48 48 __flagIsNewFile = 0
49 49
50 50 noMoreFiles = 0
51 51
52 52 online = 0
53 53
54 54 filename = None
55 55
56 56 fileSize = None
57 57
58 58 firstHeaderSize = 0
59 59
60 60 basicHeaderSize = 24
61 61
62 62 m_BasicHeader = BasicHeader()
63 63
64 64 m_SystemHeader = SystemHeader()
65 65
66 66 m_RadarControllerHeader = RadarControllerHeader()
67 67
68 68 m_ProcessingHeader = ProcessingHeader()
69 69
70 70 m_Voltage = None
71 71
72 72 __buffer = 0
73 73
74 74 __buffer_id = 9999
75 75
76 76 def __init__(self, m_Voltage = None):
77 77
78 78 if m_Voltage == None:
79 79 m_Voltage = Voltage()
80 80
81 81 self.m_Voltage = m_Voltage
82 82
83 83 def __rdSystemHeader(self,fp=None):
84 84 if fp == None:
85 85 fp = self.__fp
86 86
87 87 self.m_SystemHeader.read(fp)
88 88
89 89 def __rdRadarControllerHeader(self,fp=None):
90 90 if fp == None:
91 91 fp = self.__fp
92 92
93 93 self.m_RadarControllerHeader.read(fp)
94 94
95 95 def __rdProcessingHeader(self,fp=None):
96 96 if fp == None:
97 97 fp = self.__fp
98 98
99 99 self.m_ProcessingHeader.read(fp)
100 100
101 101 def __searchFiles(self,path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r"):
102 102
103 103 print "Searching files ..."
104 104
105 105 startUtSeconds = time.mktime(startDateTime.timetuple())
106 106 endUtSeconds = time.mktime(endDateTime.timetuple())
107 107
108 startYear = startDateTime.timetuple().tm_year
109 endYear = endDateTime.timetuple().tm_year
108 # startYear = startDateTime.timetuple().tm_year
109 # endYear = endDateTime.timetuple().tm_year
110 #
111 # startDoy = startDateTime.timetuple().tm_yday
112 # endDoy = endDateTime.timetuple().tm_yday
113 #
114 # yearRange = range(startYear,endYear+1)
115 #
116 # doyDoubleList = []
117 # if startYear == endYear:
118 # doyList = range(startDoy,endDoy+1)
119 # else:
120 # for year in yearRange:
121 # if (year == startYear):
122 # doyDoubleList.append(range(startDoy,365+1))
123 # elif (year == endYear):
124 # doyDoubleList.append(range(1,endDoy+1))
125 # else:
126 # doyDoubleList.append(range(1,365+1))
127 # doyList = []
128 # for list in doyDoubleList:
129 # doyList = doyList + list
130 #
131 # dirList = []
132 # for thisPath in os.listdir(path):
133 # if os.path.isdir(os.path.join(path,thisPath)):
134 # #dirList.append(os.path.join(path,thisPath))
135 # dirList.append(thisPath)
136 #
137 # pathList = []
138 # pathDict = {}
139 # for year in yearRange:
140 # for doy in doyList:
141 # match = fnmatch.filter(dirList, 'D' + '%4.4d%3.3d' % (year,doy))
142 # if len(match) == 0:
143 # match = fnmatch.filter(dirList, 'd' + '%4.4d%3.3d' % (year,doy))
144 # if len(match) == 0: continue
145 # if expLabel == '':
146 # pathList.append(os.path.join(path,match[0]))
147 # pathDict.setdefault(os.path.join(path,match[0]))
148 # pathDict[os.path.join(path,match[0])] = []
149 # else:
150 # pathList.append(os.path.join(path,os.path.join(match[0],expLabel)))
151 # pathDict.setdefault(os.path.join(path,os.path.join(match[0],expLabel)))
152 # pathDict[os.path.join(path,os.path.join(match[0],expLabel))] = []
110 153
111 startDoy = startDateTime.timetuple().tm_yday
112 endDoy = endDateTime.timetuple().tm_yday
113
114 yearRange = range(startYear,endYear+1)
115
116 doyDoubleList = []
117 if startYear == endYear:
118 doyList = range(startDoy,endDoy+1)
119 else:
120 for year in yearRange:
121 if (year == startYear):
122 doyDoubleList.append(range(startDoy,365+1))
123 elif (year == endYear):
124 doyDoubleList.append(range(1,endDoy+1))
125 else:
126 doyDoubleList.append(range(1,365+1))
127 doyList = []
128 for list in doyDoubleList:
129 doyList = doyList + list
130
131 doyPathList = []
154
155 dirList = []
132 156 for thisPath in os.listdir(path):
133 157 if os.path.isdir(os.path.join(path,thisPath)):
134 #doyPathList.append(os.path.join(path,thisPath))
135 doyPathList.append(thisPath)
136
158 dirList.append(thisPath)
159
137 160 pathList = []
138 pathDict = {}
139 for year in yearRange:
140 for doy in doyList:
141 match = fnmatch.filter(doyPathList, 'D' + '%4.4d%3.3d' % (year,doy))
142 if len(match) == 0:
143 match = fnmatch.filter(doyPathList, 'd' + '%4.4d%3.3d' % (year,doy))
144 if len(match) == 0: continue
145 if expLabel == '':
146 pathList.append(os.path.join(path,match[0]))
147 pathDict.setdefault(os.path.join(path,match[0]))
148 pathDict[os.path.join(path,match[0])] = []
149 else:
150 pathList.append(os.path.join(path,os.path.join(match[0],expLabel)))
151 pathDict.setdefault(os.path.join(path,os.path.join(match[0],expLabel)))
152 pathDict[os.path.join(path,os.path.join(match[0],expLabel))] = []
153 161
162 thisDateTime = startDateTime
154 163
164 while(thisDateTime <= endDateTime):
165 year = thisDateTime.timetuple().tm_year
166 doy = thisDateTime.timetuple().tm_yday
167
168 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
169 if len(match) == 0:
170 thisDateTime += datetime.timedelta(1)
171 continue
172
173 pathList.append(os.path.join(path,match[0],expLabel))
174 thisDateTime += datetime.timedelta(1)
175
155 176 filenameList = []
156 177 for thisPath in pathList:
157 178 fileList = glob.glob1(thisPath, "*%s" %ext)
158 179 #pathDict[thisPath].append(fileList)
159 180 fileList.sort()
160 181 for file in fileList:
161 182 filename = os.path.join(thisPath,file)
162 183 if self.isThisFileinRange(filename, startUtSeconds, endUtSeconds):
163 184 filenameList.append(filename)
164 185
165 186 self.filenameList = filenameList
166 187
167 188 return pathList, filenameList
168 189
169 190 def isThisFileinRange(self, filename, startUTSeconds=None, endUTSeconds=None):
170 191
171 192 try:
172 193 fp = open(filename,'rb')
173 194 except:
174 195 raise IOError, "The file %s can't be opened" %(filename)
175 196
176 197 if startUTSeconds==None:
177 198 startUTSeconds = self.startUTCSeconds
178 199
179 200 if endUTSeconds==None:
180 201 endUTSeconds = self.endUTCSeconds
181 202
182 203 m_BasicHeader = BasicHeader()
183 204
184 205 if not(m_BasicHeader.read(fp)):
185 206 return 0
186 207
187 208 fp.close()
188 209
189 210 if not ((startUTSeconds <= m_BasicHeader.utc) and (endUTSeconds >= m_BasicHeader.utc)):
190 211 return 0
191 212
192 213 return 1
193 214
194 215 def __readBasicHeader(self, fp=None):
195 216
196 217 if fp == None:
197 218 fp = self.__fp
198 219
199 220 self.m_BasicHeader.read(fp)
200 221
201 222 def __readFirstHeader(self):
202 223
203 224 self.__readBasicHeader()
204 225 self.__rdSystemHeader()
205 226 self.__rdRadarControllerHeader()
206 227 self.__rdProcessingHeader()
207 228 self.firstHeaderSize = self.m_BasicHeader.size
208 229
209 230 data_type=int(numpy.log2((self.m_ProcessingHeader.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
210 231 if data_type == 0:
211 232 tmp=numpy.dtype([('real','<i1'),('imag','<i1')])
212 233 elif data_type == 1:
213 234 tmp=numpy.dtype([('real','<i2'),('imag','<i2')])
214 235 elif data_type == 2:
215 236 tmp=numpy.dtype([('real','<i4'),('imag','<i4')])
216 237 elif data_type == 3:
217 238 tmp=numpy.dtype([('real','<i8'),('imag','<i8')])
218 239 elif data_type == 4:
219 240 tmp=numpy.dtype([('real','<f4'),('imag','<f4')])
220 241 elif data_type == 5:
221 242 tmp=numpy.dtype([('real','<f8'),('imag','<f8')])
222 243 else:
223 244 print 'no define data type'
224 245 tmp = 0
225 246
226 247 self.__dataType = tmp
227 248 self.__fileSizeByHeader = self.m_ProcessingHeader.dataBlocksPerFile * self.m_ProcessingHeader.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.m_ProcessingHeader.dataBlocksPerFile - 1)
228 249
229 250 def __setNextFileOnline(self):
230 251 return 0
231 252
232 253 def __setNextFileOffline(self):
233 254
234 255 idFile = self.__idFile
235 256 while(True):
236 257
237 258 idFile += 1
238 259
239 260 if not(idFile < len(self.filenameList)):
240 261 self.noMoreFiles = 1
241 262 return 0
242 263
243 264 filename = self.filenameList[idFile]
244 265 fileSize = os.path.getsize(filename)
245 266 fp = open(filename,'rb')
246 267
247 268 currentSize = fileSize - fp.tell()
248 269 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
249 270
250 271 if (currentSize < neededSize):
251 272 continue
252 273
253 274 break
254 275
255 276 self.__flagIsNewFile = 1
256 277 self.__idFile = idFile
257 278 self.filename = filename
258 279 self.fileSize = fileSize
259 280 self.__fp = fp
260 281
261 282 print 'Setting the file: %s'%self.filename
262 283
263 284 return 1
264 285
265 286 def __setNextFile(self):
266 287
267 288 if self.online:
268 289 return self.__setNextFileOnline()
269 290 else:
270 291 return self.__setNextFileOffline()
271 292
272 293 def __setNewBlock(self):
273 294
274 295 if self.__fp == None:
275 296 return 0
276 297
277 298 if self.__flagIsNewFile:
278 299 return 1
279 300
280 301 currentSize = self.fileSize - self.__fp.tell()
281 302 neededSize = self.m_ProcessingHeader.blockSize + self.basicHeaderSize
282 303
283 304 # Bloque Completo
284 305 if (currentSize >= neededSize):
285 306 self.__readBasicHeader()
286 307 return 1
287 308
288 309 if not(self.__setNextFile()):
289 310 return 0
290 311
291 312 self.__readFirstHeader()
292 313
293 314 deltaTime = self.m_BasicHeader.utc - self.__lastUTTime # check this
294 315
295 316 self.flagResetProcessing = 0
296 317 if deltaTime > self.__maxTimeStep:
297 318 self.flagResetProcessing = 1
298 319
299 320 return 1
300 321
301 322 def __readBlock(self):
302 323 """Lee el bloque de datos desde la posicion actual del puntero del archivo y
303 324 actualiza todos los parametros relacionados al bloque de datos (data, time,
304 325 etc). La data leida es almacenada en el buffer y el contador de datos leidos es
305 326 seteado a 0
306 327 """
307 328
308 329 pts2read = self.m_ProcessingHeader.profilesPerBlock*self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels
309 330
310 331 data = numpy.fromfile(self.__fp,self.__dataType,pts2read)
311 332
312 333 data = data.reshape((self.m_ProcessingHeader.profilesPerBlock, self.m_ProcessingHeader.numHeights, self.m_SystemHeader.numChannels))
313 334
314 335 self.__flagIsNewFile = 0
315 336
316 337 self.__buffer = data
317 338
318 339 self.__buffer_id = 0
319 340
320 341 def readNextBlock(self):
321 342
322 343 if not(self.__setNewBlock()):
323 344 return 0
324 345
325 346 self.__readBlock()
326 347
327 348 self.__lastUTTime = self.m_BasicHeader.utc
328 349
329 350 return 1
330 351
331 352 def __hasNotDataInBuffer(self):
332 353 if self.__buffer_id >= self.m_ProcessingHeader.profilesPerBlock:
333 354 return 1
334 355
335 356 return 0
336 357
337 358 def getData(self):
338 """Obtiene un unidad de datos del buffer de lectura y es copiada a la clase "Data"
359 """Obtiene un unidad de datos del buffer de lectura y es copiada a la clase "Voltage"
339 360 con todos los parametros asociados a este. cuando no hay datos en el buffer de
340 lectura es necesario hacer una nueva lectura de los bloques de datos
341 "__readBlock"
361 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
342 362 """
343 363 self.flagResetProcessing = 0
344 364
345 365 if self.__hasNotDataInBuffer():
346 366 self.readNextBlock()
347 367
348 368 if self.noMoreFiles == 1:
349 369 print 'Process finished'
350 370 return None
351 371
352 372 data = self.__buffer[self.__buffer_id,:,:]
353 373 time = 111
354 374
355 375 # self.m_Voltage.data = data
356 376 # self.m_Voltage.timeProfile = time
357 377 # self.m_Voltage.m_BasicHeader = self.m_BasicHeader.copy()
358 378 # self.m_Voltage.m_ProcessingHeader = self.m_ProcessingHeader.copy()
359 379 # self.m_Voltage.m_RadarControllerHeader = self.m_RadarControllerHeader.copy()
360 380 # self.m_Voltage.m_SystemHeader = self.m_systemHeader.copy()
361 381
362 382 self.__buffer_id += 1
363 383
364 384 #call setData - to Data Object
365 385
366 386 return data
367 387
368 388
369 389 def setup(self, path, startDateTime, endDateTime, set=None, expLabel = "", ext = ".r", online = 0):
370 390
371 391 if online == 0:
372 392 pathList, filenameList = self.__searchFiles(path, startDateTime, endDateTime, set, expLabel, ext)
373 393
374 394 if len(filenameList) == 0:
375 395 self.__fp = None
376 396 self.noMoreFiles = 1
377 397 print 'Do not exist files in range: %s - %s'%(startDateTime.ctime(), endDateTime.ctime())
378 398 return 0
379 399
380 400 # for thisFile in filenameList:
381 401 # print thisFile
382 402
383 403 self.__idFile = -1
384 404
385 405 if not(self.__setNextFile()):
386 406 print "No more files"
387 407 return 0
388 408
389 409 self.__readFirstHeader()
390 410
391 411 self.startUTCSeconds = time.mktime(startDateTime.timetuple())
392 412 self.endUTCSeconds = time.mktime(endDateTime.timetuple())
393 413
394 414 self.startYear = startDateTime.timetuple().tm_year
395 415 self.endYear = endDateTime.timetuple().tm_year
396 416
397 417 self.startDoy = startDateTime.timetuple().tm_yday
398 418 self.endDoy = endDateTime.timetuple().tm_yday
399 419 #call fillHeaderValues() - to Data Object
400 420
401 421 self.__pathList = pathList
402 422 self.filenameList = filenameList
403 423 self.online = online
404 424
405 425 class VoltageWriter(DataWriter):
406 426
407 427 m_BasicHeader= BasicHeader()
408 428
409 429
410 430 m_SystemHeader = SystemHeader()
411 431
412 432
413 433 m_RadarControllerHeader = RadarControllerHeader()
414 434
415 435
416 436 m_ProcessingHeader = ProcessingHeader()
417 437
418 438 m_Voltage = None
419 439
420 440 def __init__(self, m_Voltage):
421 441
422 442 self.m_Voltage = m_Voltage
423 443
424 444
425 445 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now