##// END OF EJS Templates
schain-jc
RSS
Busqueda de archivos dentro del directorio indicado (sin busqueda de subdirectorios) activando el flag de lectura 'walk'
Miguel Valdez -
r224:b8cf5fb6064c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -38,35 +38,40 class ParameterConf():
38
38
39 def getValue(self):
39 def getValue(self):
40
40
41 value = self.value
42
41 if self.format == 'list':
43 if self.format == 'list':
42 strList = self.value.split(',')
44 strList = value.split(',')
43 return strList
45 return strList
44
46
45 if self.format == 'intlist':
47 if self.format == 'intlist':
46 strList = self.value.split(',')
48 strList = value.split(',')
47 intList = [int(x) for x in strList]
49 intList = [int(x) for x in strList]
48 return intList
50 return intList
49
51
50 if self.format == 'floatlist':
52 if self.format == 'floatlist':
51 strList = self.value.split(',')
53 strList = value.split(',')
52 floatList = [float(x) for x in strList]
54 floatList = [float(x) for x in strList]
53 return floatList
55 return floatList
54
56
55 if self.format == 'date':
57 if self.format == 'date':
56 strList = self.value.split('/')
58 strList = value.split('/')
57 intList = [int(x) for x in strList]
59 intList = [int(x) for x in strList]
58 date = datetime.date(intList[0], intList[1], intList[2])
60 date = datetime.date(intList[0], intList[1], intList[2])
59 return date
61 return date
60
62
61 if self.format == 'time':
63 if self.format == 'time':
62 strList = self.value.split(':')
64 strList = value.split(':')
63 intList = [int(x) for x in strList]
65 intList = [int(x) for x in strList]
64 time = datetime.time(intList[0], intList[1], intList[2])
66 time = datetime.time(intList[0], intList[1], intList[2])
65 return time
67 return time
66
68
69 if self.format == 'bool':
70 value = int(value)
71
67 func = eval(self.format)
72 func = eval(self.format)
68
73
69 return func(self.value)
74 return func(value)
70
75
71 def setup(self, id, name, value, format='str'):
76 def setup(self, id, name, value, format='str'):
72
77
@@ -359,9 +364,6 class ReadUnitConf(ProcUnitConf):
359 endDate = None
364 endDate = None
360 startTime = None
365 startTime = None
361 endTime = None
366 endTime = None
362 online = None
363 expLabel = None
364 delay = None
365
367
366 ELEMENTNAME = 'ReadUnit'
368 ELEMENTNAME = 'ReadUnit'
367
369
@@ -379,7 +381,7 class ReadUnitConf(ProcUnitConf):
379
381
380 return self.ELEMENTNAME
382 return self.ELEMENTNAME
381
383
382 def setup(self, id, name, datatype, path, startDate, endDate, startTime, endTime, online=0, expLabel='', delay=60):
384 def setup(self, id, name, datatype, path, startDate, endDate, startTime, endTime, **kwargs):
383
385
384 self.id = id
386 self.id = id
385 self.name = name
387 self.name = name
@@ -390,13 +392,10 class ReadUnitConf(ProcUnitConf):
390 self.endDate = endDate
392 self.endDate = endDate
391 self.startTime = startTime
393 self.startTime = startTime
392 self.endTime = endTime
394 self.endTime = endTime
393 self.online = online
394 self.expLabel = expLabel
395 self.delay = delay
396
395
397 self.addRunOperation()
396 self.addRunOperation(**kwargs)
398
397
399 def addRunOperation(self):
398 def addRunOperation(self, **kwargs):
400
399
401 opObj = self.addOperation(name = 'run', optype = 'self')
400 opObj = self.addOperation(name = 'run', optype = 'self')
402
401
@@ -405,14 +404,14 class ReadUnitConf(ProcUnitConf):
405 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
404 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
406 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
405 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
407 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
406 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
408 opObj.addParameter(name='expLabel' , value=self.expLabel, format='str')
407
409 opObj.addParameter(name='online' , value=self.online, format='int')
408 for key, value in kwargs.items():
410 opObj.addParameter(name='delay' , value=self.delay, format='float')
409 opObj.addParameter(name=key, value=value, format=type(value).__name__)
411
410
412 return opObj
411 return opObj
413
412
414
413
415 class Controller():
414 class Project():
416
415
417 id = None
416 id = None
418 name = None
417 name = None
@@ -420,7 +419,7 class Controller():
420 # readUnitConfObjList = None
419 # readUnitConfObjList = None
421 procUnitConfObjDict = None
420 procUnitConfObjDict = None
422
421
423 ELEMENTNAME = 'Controller'
422 ELEMENTNAME = 'Project'
424
423
425 def __init__(self):
424 def __init__(self):
426
425
@@ -447,13 +446,13 class Controller():
447 self.name = name
446 self.name = name
448 self.description = description
447 self.description = description
449
448
450 def addReadUnit(self, datatype, path, startDate='', endDate='', startTime='', endTime='', online=0, expLabel='', delay=60):
449 def addReadUnit(self, datatype, path, startDate='', endDate='', startTime='', endTime='', **kwargs):
451
450
452 id = self.__getNewId()
451 id = self.__getNewId()
453 name = '%sReader' %(datatype)
452 name = '%sReader' %(datatype)
454
453
455 readUnitConfObj = ReadUnitConf()
454 readUnitConfObj = ReadUnitConf()
456 readUnitConfObj.setup(id, name, datatype, path, startDate, endDate, startTime, endTime, online, expLabel, delay)
455 readUnitConfObj.setup(id, name, datatype, path, startDate, endDate, startTime, endTime, **kwargs)
457
456
458 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
457 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
459
458
@@ -473,7 +472,7 class Controller():
473
472
474 def makeXml(self):
473 def makeXml(self):
475
474
476 projectElement = Element('Controller')
475 projectElement = Element('Project')
477 projectElement.set('id', str(self.id))
476 projectElement.set('id', str(self.id))
478 projectElement.set('name', self.name)
477 projectElement.set('name', self.name)
479 projectElement.set('description', self.description)
478 projectElement.set('description', self.description)
@@ -527,7 +526,7 class Controller():
527
526
528 def printattr(self):
527 def printattr(self):
529
528
530 print "Controller[%s]: name = %s, description = %s" %(self.id,
529 print "Project[%s]: name = %s, description = %s" %(self.id,
531 self.name,
530 self.name,
532 self.description)
531 self.description)
533
532
@@ -589,7 +588,7 if __name__ == '__main__':
589 desc = "Segundo Test"
588 desc = "Segundo Test"
590 filename = "schain.xml"
589 filename = "schain.xml"
591
590
592 controllerObj = Controller()
591 controllerObj = Project()
593
592
594 controllerObj.setup(id = '191', name='test01', description=desc)
593 controllerObj.setup(id = '191', name='test01', description=desc)
595
594
@@ -599,7 +598,8 if __name__ == '__main__':
599 endDate='2012/12/31',
598 endDate='2012/12/31',
600 startTime='00:00:00',
599 startTime='00:00:00',
601 endTime='23:59:59',
600 endTime='23:59:59',
602 online=0)
601 online=1,
602 walk=1)
603
603
604 # opObj00 = readUnitConfObj.addOperation(name='printInfo')
604 # opObj00 = readUnitConfObj.addOperation(name='printInfo')
605
605
@@ -617,6 +617,8 if __name__ == '__main__':
617
617
618 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
618 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
619 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
619 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
620 # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
621
620
622
621 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
623 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
622 opObj11.addParameter(name='idfigure', value='1', format='int')
624 opObj11.addParameter(name='idfigure', value='1', format='int')
@@ -624,21 +626,28 if __name__ == '__main__':
624 opObj11.addParameter(name='zmin', value='40', format='int')
626 opObj11.addParameter(name='zmin', value='40', format='int')
625 opObj11.addParameter(name='zmax', value='90', format='int')
627 opObj11.addParameter(name='zmax', value='90', format='int')
626 opObj11.addParameter(name='showprofile', value='1', format='int')
628 opObj11.addParameter(name='showprofile', value='1', format='int')
627
628 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
629
630 opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='other')
631 opObj12.addParameter(name='n', value='2', format='int')
632 opObj12.addParameter(name='overlapping', value='1', format='int')
633
629
634 procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
630 # opObj11 = procUnitConfObj1.addOperation(name='CrossSpectraPlot', optype='other')
635 procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
631 # opObj11.addParameter(name='idfigure', value='2', format='int')
636
632 # opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
637 opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='other')
633 # opObj11.addParameter(name='zmin', value='40', format='int')
638 opObj11.addParameter(name='idfigure', value='2', format='int')
634 # opObj11.addParameter(name='zmax', value='90', format='int')
639 opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
635
640 opObj11.addParameter(name='zmin', value='40', format='int')
636
641 opObj11.addParameter(name='zmax', value='90', format='int')
637 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
638 #
639 # opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='other')
640 # opObj12.addParameter(name='n', value='2', format='int')
641 # opObj12.addParameter(name='overlapping', value='1', format='int')
642 #
643 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
644 # procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
645 #
646 # opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='other')
647 # opObj11.addParameter(name='idfigure', value='2', format='int')
648 # opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
649 # opObj11.addParameter(name='zmin', value='40', format='int')
650 # opObj11.addParameter(name='zmax', value='90', format='int')
642 # opObj11.addParameter(name='showprofile', value='1', format='int')
651 # opObj11.addParameter(name='showprofile', value='1', format='int')
643
652
644 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
653 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
Show file before | Show file after | Hide comments
@@ -255,27 +255,10 def createPcolor(ax, x, y, z, xmin, xmax, ymin, ymax, zmin, zmax,
255
255
256 if XAxisAsTime:
256 if XAxisAsTime:
257
257
258 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime("%H:%M:%S"))
258 func = lambda x, pos: ('%s') %(datetime.datetime.utcfromtimestamp(x).strftime("%H:%M:%S"))
259 ax.xaxis.set_major_formatter(FuncFormatter(func))
259 ax.xaxis.set_major_formatter(FuncFormatter(func))
260 ax.xaxis.set_major_locator(LinearLocator(7))
260 ax.xaxis.set_major_locator(LinearLocator(7))
261
261
262 # seconds = numpy.array([xmin, xmax])
263 # datesList = map(datetime.datetime.fromtimestamp, seconds)
264 # ax.set_xlim([datesList[0],datesList[-1]])
265 # ax.xaxis.set_major_locator(MinuteLocator(numpy.arange(0,61,10)))
266 # ax.xaxis.set_minor_locator(SecondLocator(numpy.arange(0,61,60)))
267 # ax.xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
268 # xdateList = map(datetime.datetime.fromtimestamp, x)
269 # xdate = matplotlib.dates.date2num(xdateList)
270 # x = xdate
271
272 # labels = []
273 # for item in ax.xaxis.get_ticklabels():
274 # stri = item.get_text()
275 # text = datetime.datetime.fromtimestamp(float(stri))
276 # labels.append(text)
277 #
278 # ax.xaxis.set_ticklabels(labels)
279 return imesh
262 return imesh
280
263
281 def pcolor(imesh, z, xlabel='', ylabel='', title=''):
264 def pcolor(imesh, z, xlabel='', ylabel='', title=''):
@@ -290,9 +273,6 def pcolor(imesh, z, xlabel='', ylabel='', title=''):
290
273
291 def addpcolor(ax, x, y, z, zmin, zmax, xlabel='', ylabel='', title=''):
274 def addpcolor(ax, x, y, z, zmin, zmax, xlabel='', ylabel='', title=''):
292
275
293 # xdateList = map(datetime.datetime.fromtimestamp, x)
294 # xdate = matplotlib.dates.date2num(xdateList)
295
296 printLabels(ax, xlabel, ylabel, title)
276 printLabels(ax, xlabel, ylabel, title)
297
277
298 imesh = ax.pcolormesh(x,y,z.T,vmin=zmin,vmax=zmax)
278 imesh = ax.pcolormesh(x,y,z.T,vmin=zmin,vmax=zmax)
Show file before | Show file after | Hide comments
@@ -7,6 +7,7 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
7 import os, sys
7 import os, sys
8 import copy
8 import copy
9 import numpy
9 import numpy
10 import datetime
10
11
11 from jroheaderIO import SystemHeader, RadarControllerHeader
12 from jroheaderIO import SystemHeader, RadarControllerHeader
12
13
@@ -211,6 +212,11 class JROData:
211
212
212 def getDatatime(self):
213 def getDatatime(self):
213
214
215 datatime = datetime.datetime.utcfromtimestamp(self.utctime)
216 return datatime
217
218 def getTimeRange(self):
219
214 datatime = []
220 datatime = []
215
221
216 datatime.append(self.utctime)
222 datatime.append(self.utctime)
@@ -238,10 +244,9 class JROData:
238
244
239 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
245 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
240 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
246 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
241
242 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
247 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
243
244 noise = property(getNoise, "I'm the 'nHeights' property.")
248 noise = property(getNoise, "I'm the 'nHeights' property.")
249 datatime = property(getDatatime, "I'm the 'datatime' property")
245
250
246 class Voltage(JROData):
251 class Voltage(JROData):
247
252
Show file before | Show file after | Hide comments
@@ -74,6 +74,48 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
74
74
75 return 1
75 return 1
76
76
77 def isFileinThisTime(filename, startTime, endTime):
78 """
79 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
80
81 Inputs:
82 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
83
84 startTime : tiempo inicial del rango seleccionado en formato datetime.time
85
86 endTime : tiempo final del rango seleccionado en formato datetime.time
87
88 Return:
89 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
90 fecha especificado, de lo contrario retorna False.
91
92 Excepciones:
93 Si el archivo no existe o no puede ser abierto
94 Si la cabecera no puede ser leida.
95
96 """
97
98
99 try:
100 fp = open(filename,'rb')
101 except:
102 raise IOError, "The file %s can't be opened" %(filename)
103
104 basicHeaderObj = BasicHeader()
105 sts = basicHeaderObj.read(fp)
106 fp.close()
107
108 thisTime = basicHeaderObj.datatime.time()
109
110 if not(sts):
111 print "Skipping the file %s because it has not a valid header" %(filename)
112 return 0
113
114 if not ((startTime <= thisTime) and (endTime > thisTime)):
115 return 0
116
117 return 1
118
77 def getlastFileFromPath(path, ext):
119 def getlastFileFromPath(path, ext):
78 """
120 """
79 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
121 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
@@ -97,10 +139,13 def getlastFileFromPath(path, ext):
97 year = int(file[1:5])
139 year = int(file[1:5])
98 doy = int(file[5:8])
140 doy = int(file[5:8])
99
141
100 if (os.path.splitext(file)[-1].upper() != ext.upper()) : continue
142
101 except:
143 except:
102 continue
144 continue
103
145
146 if (os.path.splitext(file)[-1].lower() != ext.lower()):
147 continue
148
104 validFilelist.append(file)
149 validFilelist.append(file)
105
150
106 if validFilelist:
151 if validFilelist:
@@ -119,6 +164,8 def checkForRealPath(path, year, doy, set, ext):
119 nombre correcto del file es .../.../D2009307/P2009307367.ext
164 nombre correcto del file es .../.../D2009307/P2009307367.ext
120
165
121 Entonces la funcion prueba con las siguientes combinaciones
166 Entonces la funcion prueba con las siguientes combinaciones
167 .../.../y2009307367.ext
168 .../.../Y2009307367.ext
122 .../.../x2009307/y2009307367.ext
169 .../.../x2009307/y2009307367.ext
123 .../.../x2009307/Y2009307367.ext
170 .../.../x2009307/Y2009307367.ext
124 .../.../X2009307/y2009307367.ext
171 .../.../X2009307/y2009307367.ext
@@ -130,25 +177,30 def checkForRealPath(path, year, doy, set, ext):
130 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
177 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
131 para el filename
178 para el filename
132 """
179 """
133 filepath = None
180 fullfilename = None
134 find_flag = False
181 find_flag = False
135 filename = None
182 filename = None
136
183
184 prefixDirList = [None,'d','D']
137 if ext.lower() == ".r": #voltage
185 if ext.lower() == ".r": #voltage
138 header1 = "dD"
186 prefixFileList = ['d','D']
139 header2 = "dD"
140 elif ext.lower() == ".pdata": #spectra
187 elif ext.lower() == ".pdata": #spectra
141 header1 = "dD"
188 prefixFileList = ['p','P']
142 header2 = "pP"
143 else:
189 else:
144 return None, filename
190 return None, filename
191
192 #barrido por las combinaciones posibles
193 for prefixDir in prefixDirList:
194 thispath = path
195 if prefixDir != None:
196 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
197 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
198
199 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
200 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
201 fullfilename = os.path.join( thispath, filename ) #formo el path completo
145
202
146 for dir in header1: #barrido por las dos combinaciones posibles de "D"
203 if os.path.exists( fullfilename ): #verifico que exista
147 for fil in header2: #barrido por las dos combinaciones posibles de "D"
148 doypath = "%s%04d%03d" % ( dir, year, doy ) #formo el nombre del directorio xYYYYDDD (x=d o x=D)
149 filename = "%s%04d%03d%03d%s" % ( fil, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
150 filepath = os.path.join( path, doypath, filename ) #formo el path completo
151 if os.path.exists( filepath ): #verifico que exista
152 find_flag = True
204 find_flag = True
153 break
205 break
154 if find_flag:
206 if find_flag:
@@ -157,7 +209,7 def checkForRealPath(path, year, doy, set, ext):
157 if not(find_flag):
209 if not(find_flag):
158 return None, filename
210 return None, filename
159
211
160 return filepath, filename
212 return fullfilename, filename
161
213
162 class JRODataIO:
214 class JRODataIO:
163
215
@@ -273,54 +325,50 class JRODataReader(JRODataIO, ProcessingUnit):
273 startTime=datetime.time(0,0,0),
325 startTime=datetime.time(0,0,0),
274 endTime=datetime.time(23,59,59),
326 endTime=datetime.time(23,59,59),
275 set=None,
327 set=None,
276 expLabel="",
328 expLabel='',
277 ext=".r"):
329 ext='.r',
278 dirList = []
330 walk=True):
279 for thisPath in os.listdir(path):
280 if os.path.isdir(os.path.join(path,thisPath)):
281 dirList.append(thisPath)
282
283 if not(dirList):
284 return None, None
285
286 pathList = []
287 dateList = []
288
331
289 thisDate = startDate
332 pathList = []
290
333
291 while(thisDate <= endDate):
334 if not walk:
292 year = thisDate.timetuple().tm_year
335 pathList.append(path)
293 doy = thisDate.timetuple().tm_yday
294
336
295 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
337 else:
296 if len(match) == 0:
338 dirList = []
297 thisDate += datetime.timedelta(1)
339 for thisPath in os.listdir(path):
298 continue
340 if os.path.isdir(os.path.join(path,thisPath)):
341 dirList.append(thisPath)
342
343 if not(dirList):
344 return None, None
345
346 thisDate = startDate
299
347
300 pathList.append(os.path.join(path,match[0],expLabel))
348 while(thisDate <= endDate):
301 dateList.append(thisDate)
349 year = thisDate.timetuple().tm_year
302 thisDate += datetime.timedelta(1)
350 doy = thisDate.timetuple().tm_yday
351
352 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
353 if len(match) == 0:
354 thisDate += datetime.timedelta(1)
355 continue
356
357 pathList.append(os.path.join(path,match[0],expLabel))
358 thisDate += datetime.timedelta(1)
359
303
360
304 filenameList = []
361 filenameList = []
305 for index in range(len(pathList)):
362 for thisPath in pathList:
306
363
307 thisPath = pathList[index]
308 fileList = glob.glob1(thisPath, "*%s" %ext)
364 fileList = glob.glob1(thisPath, "*%s" %ext)
309 fileList.sort()
365 fileList.sort()
310
366
311 #Busqueda de datos en el rango de horas indicados
312 thisDate = dateList[index]
313 startDT = datetime.datetime.combine(thisDate, startTime)
314 endDT = datetime.datetime.combine(thisDate, endTime)
315
316 startUtSeconds = time.mktime(startDT.timetuple())
317 endUtSeconds = time.mktime(endDT.timetuple())
318
319 for file in fileList:
367 for file in fileList:
320
368
321 filename = os.path.join(thisPath,file)
369 filename = os.path.join(thisPath,file)
322
370
323 if isThisFileinRange(filename, startUtSeconds, endUtSeconds):
371 if isFileinThisTime(filename, startTime, endTime):
324 filenameList.append(filename)
372 filenameList.append(filename)
325
373
326 if not(filenameList):
374 if not(filenameList):
@@ -330,7 +378,7 class JRODataReader(JRODataIO, ProcessingUnit):
330
378
331 return pathList, filenameList
379 return pathList, filenameList
332
380
333 def __searchFilesOnLine(self, path, startDate=None, endDate=None, startTime=None, endTime=None, expLabel = "", ext = None):
381 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True):
334
382
335 """
383 """
336 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
384 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
@@ -339,21 +387,11 class JRODataReader(JRODataIO, ProcessingUnit):
339 Input:
387 Input:
340 path : carpeta donde estan contenidos los files que contiene data
388 path : carpeta donde estan contenidos los files que contiene data
341
389
342 startDate : Fecha inicial. Rechaza todos los directorios donde
343 file end time < startDate (obejto datetime.date)
344
345 endDate : Fecha final. Rechaza todos los directorios donde
346 file start time > endDate (obejto datetime.date)
347
348 startTime : Tiempo inicial. Rechaza todos los archivos donde
349 file end time < startTime (obejto datetime.time)
350
351 endTime : Tiempo final. Rechaza todos los archivos donde
352 file start time > endTime (obejto datetime.time)
353
354 expLabel : Nombre del subexperimento (subfolder)
390 expLabel : Nombre del subexperimento (subfolder)
355
391
356 ext : extension de los files
392 ext : extension de los files
393
394 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
357
395
358 Return:
396 Return:
359 directory : eL directorio donde esta el file encontrado
397 directory : eL directorio donde esta el file encontrado
@@ -365,60 +403,38 class JRODataReader(JRODataIO, ProcessingUnit):
365
403
366 """
404 """
367 dirList = []
405 dirList = []
368 pathList = []
369 directory = None
370
406
371 #Filtra solo los directorios
407 if walk:
372 for thisPath in os.listdir(path):
373 if os.path.isdir(os.path.join(path, thisPath)):
374 dirList.append(thisPath)
375
376 if not(dirList):
377 return None, None, None, None, None
378
379 dirList = sorted( dirList, key=str.lower )
380
381 if startDate:
382 startDateTime = datetime.datetime.combine(startDate, startTime)
383 thisDateTime = startDateTime
384 if endDate == None: endDateTime = startDateTime
385 else: endDateTime = datetime.datetime.combine(endDate, endTime)
386
408
387 while(thisDateTime <= endDateTime):
409 #Filtra solo los directorios
388 year = thisDateTime.timetuple().tm_year
410 for thisPath in os.listdir(path):
389 doy = thisDateTime.timetuple().tm_yday
411 if os.path.isdir(os.path.join(path, thisPath)):
390
412 dirList.append(thisPath)
391 match = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy))
413
392 if len(match) == 0:
414 if not(dirList):
393 thisDateTime += datetime.timedelta(1)
394 continue
395
396 pathList.append(os.path.join(path,match[0], expLabel))
397 thisDateTime += datetime.timedelta(1)
398
399 if not(pathList):
400 print "\tNo files in range: %s - %s" %(startDateTime.ctime(), endDateTime.ctime())
401 return None, None, None, None, None
415 return None, None, None, None, None
402
416
403 directory = pathList[0]
417 dirList = sorted( dirList, key=str.lower )
404
418
419 doypath = dirList[-1]
420 fullpath = os.path.join(path, doypath, expLabel)
421
405 else:
422 else:
406 directory = dirList[-1]
423 fullpath = path
407 directory = os.path.join(path,directory)
408
424
409 filename = getlastFileFromPath(directory, ext)
425 filename = getlastFileFromPath(fullpath, ext)
410
426
411 if not(filename):
427 if not(filename):
412 return None, None, None, None, None
428 return None, None, None, None, None
413
429
414 if not(self.__verifyFile(os.path.join(directory, filename))):
430 if not(self.__verifyFile(os.path.join(fullpath, filename))):
415 return None, None, None, None, None
431 return None, None, None, None, None
416
432
417 year = int( filename[1:5] )
433 year = int( filename[1:5] )
418 doy = int( filename[5:8] )
434 doy = int( filename[5:8] )
419 set = int( filename[8:11] )
435 set = int( filename[8:11] )
420
436
421 return directory, filename, year, doy, set
437 return fullpath, filename, year, doy, set
422
438
423
439
424
440
@@ -480,9 +496,9 class JRODataReader(JRODataIO, ProcessingUnit):
480 self.set += 1
496 self.set += 1
481
497
482 #busca el 1er file disponible
498 #busca el 1er file disponible
483 file, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
499 fullfilename, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
484 if file:
500 if fullfilename:
485 if self.__verifyFile(file, False):
501 if self.__verifyFile(fullfilename, False):
486 fileOk_flag = True
502 fileOk_flag = True
487
503
488 #si no encuentra un file entonces espera y vuelve a buscar
504 #si no encuentra un file entonces espera y vuelve a buscar
@@ -501,9 +517,9 class JRODataReader(JRODataIO, ProcessingUnit):
501 else:
517 else:
502 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
518 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
503
519
504 file, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
520 fullfilename, filename = checkForRealPath( self.path, self.year, self.doy, self.set, self.ext )
505 if file:
521 if fullfilename:
506 if self.__verifyFile(file):
522 if self.__verifyFile(fullfilename):
507 fileOk_flag = True
523 fileOk_flag = True
508 break
524 break
509
525
@@ -520,13 +536,13 class JRODataReader(JRODataIO, ProcessingUnit):
520 self.doy += 1
536 self.doy += 1
521
537
522 if fileOk_flag:
538 if fileOk_flag:
523 self.fileSize = os.path.getsize( file )
539 self.fileSize = os.path.getsize( fullfilename )
524 self.filename = file
540 self.filename = fullfilename
525 self.flagIsNewFile = 1
541 self.flagIsNewFile = 1
526 if self.fp != None: self.fp.close()
542 if self.fp != None: self.fp.close()
527 self.fp = open(file, 'rb')
543 self.fp = open(fullfilename, 'rb')
528 self.flagNoMoreFiles = 0
544 self.flagNoMoreFiles = 0
529 print 'Setting the file: %s' % file
545 print 'Setting the file: %s' % fullfilename
530 else:
546 else:
531 self.fileSize = 0
547 self.fileSize = 0
532 self.filename = None
548 self.filename = None
@@ -555,7 +571,11 class JRODataReader(JRODataIO, ProcessingUnit):
555 return 1
571 return 1
556
572
557 def __waitNewBlock(self):
573 def __waitNewBlock(self):
558 #si es OnLine y ademas aun no se han leido un bloque completo entonces se espera por uno valido
574 """
575 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
576
577 Si el modo de lectura es OffLine siempre retorn 0
578 """
559 if not self.online:
579 if not self.online:
560 return 0
580 return 0
561
581
@@ -586,6 +606,7 class JRODataReader(JRODataIO, ProcessingUnit):
586 return 0
606 return 0
587
607
588 def __setNewBlock(self):
608 def __setNewBlock(self):
609
589 if self.fp == None:
610 if self.fp == None:
590 return 0
611 return 0
591
612
@@ -738,7 +759,8 class JRODataReader(JRODataIO, ProcessingUnit):
738 expLabel = "",
759 expLabel = "",
739 ext = None,
760 ext = None,
740 online = False,
761 online = False,
741 delay = 60):
762 delay = 60,
763 walk = True):
742
764
743 if path == None:
765 if path == None:
744 raise ValueError, "The path is not valid"
766 raise ValueError, "The path is not valid"
@@ -748,17 +770,17 class JRODataReader(JRODataIO, ProcessingUnit):
748
770
749 if online:
771 if online:
750 print "Searching files in online mode..."
772 print "Searching files in online mode..."
751 doypath, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext)
773
752
774 for nTries in range( self.nTries ):
753 if not(doypath):
775 fullpath, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk)
754 for nTries in range( self.nTries ):
776
755 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
777 if fullpath:
756 time.sleep( self.delay )
778 break
757 doypath, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext)
779
758 if doypath:
780 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
759 break
781 time.sleep( self.delay )
760
782
761 if not(doypath):
783 if not(fullpath):
762 print "There 'isn't valied files in %s" % path
784 print "There 'isn't valied files in %s" % path
763 return None
785 return None
764
786
@@ -769,7 +791,10 class JRODataReader(JRODataIO, ProcessingUnit):
769
791
770 else:
792 else:
771 print "Searching files in offline mode ..."
793 print "Searching files in offline mode ..."
772 pathList, filenameList = self.__searchFilesOffLine(path, startDate, endDate, startTime, endTime, set, expLabel, ext)
794 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
795 startTime=startTime, endTime=endTime,
796 set=set, expLabel=expLabel, ext=ext,
797 walk=walk)
773
798
774 if not(pathList):
799 if not(pathList):
775 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
800 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
@@ -1000,12 +1025,12 class JRODataWriter(JRODataIO, Operation):
1000 timeTuple = time.localtime( self.dataOut.dataUtcTime)
1025 timeTuple = time.localtime( self.dataOut.dataUtcTime)
1001 subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1026 subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1002
1027
1003 doypath = os.path.join( path, subfolder )
1028 fullpath = os.path.join( path, subfolder )
1004 if not( os.path.exists(doypath) ):
1029 if not( os.path.exists(fullpath) ):
1005 os.mkdir(doypath)
1030 os.mkdir(fullpath)
1006 self.setFile = -1 #inicializo mi contador de seteo
1031 self.setFile = -1 #inicializo mi contador de seteo
1007 else:
1032 else:
1008 filesList = os.listdir( doypath )
1033 filesList = os.listdir( fullpath )
1009 if len( filesList ) > 0:
1034 if len( filesList ) > 0:
1010 filesList = sorted( filesList, key=str.lower )
1035 filesList = sorted( filesList, key=str.lower )
1011 filen = filesList[-1]
1036 filen = filesList[-1]
@@ -2433,9 +2458,9 class SpectraHeisWriter():
2433 #folder='D%4.4d%3.3d'%(name.tm_year,name.tm_yday)
2458 #folder='D%4.4d%3.3d'%(name.tm_year,name.tm_yday)
2434 subfolder = 'D%4.4d%3.3d' % (name.tm_year,name.tm_yday)
2459 subfolder = 'D%4.4d%3.3d' % (name.tm_year,name.tm_yday)
2435
2460
2436 doypath = os.path.join( self.wrpath, subfolder )
2461 fullpath = os.path.join( self.wrpath, subfolder )
2437 if not( os.path.exists(doypath) ):
2462 if not( os.path.exists(fullpath) ):
2438 os.mkdir(doypath)
2463 os.mkdir(fullpath)
2439 self.setFile += 1
2464 self.setFile += 1
2440 file = 'D%4.4d%3.3d%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
2465 file = 'D%4.4d%3.3d%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
2441
2466
Show file before | Show file after | Hide comments
@@ -6,6 +6,7 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
6 import sys
6 import sys
7 import numpy
7 import numpy
8 import copy
8 import copy
9 import datetime
9
10
10 class Header:
11 class Header:
11
12
@@ -37,6 +38,7 class BasicHeader(Header):
37 dstFlag = None
38 dstFlag = None
38 errorCount = None
39 errorCount = None
39 struct = None
40 struct = None
41 datatime = None
40
42
41 def __init__(self):
43 def __init__(self):
42
44
@@ -72,6 +74,7 class BasicHeader(Header):
72 self.dstFlag = int(header['nDstflag'][0])
74 self.dstFlag = int(header['nDstflag'][0])
73 self.errorCount = int(header['nErrorCount'][0])
75 self.errorCount = int(header['nErrorCount'][0])
74
76
77 self.datatime = datetime.datetime.utcfromtimestamp(self.utc)
75 except Exception, e:
78 except Exception, e:
76 print "BasicHeader: " + e
79 print "BasicHeader: " + e
77 return 0
80 return 0
Show file before | Show file after | Hide comments
@@ -79,6 +79,9 class CrossSpectraPlot(Figure):
79 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
79 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
80 pairsIndexList.append(dataOut.pairsList.index(pair))
80 pairsIndexList.append(dataOut.pairsList.index(pair))
81
81
82 if pairIndexList == []:
83 return
84
82 x = dataOut.getVelRange(1)
85 x = dataOut.getVelRange(1)
83 y = dataOut.getHeiRange()
86 y = dataOut.getHeiRange()
84 z = 10.*numpy.log10(dataOut.data_spc[:,:,:])
87 z = 10.*numpy.log10(dataOut.data_spc[:,:,:])
@@ -104,7 +107,7 class CrossSpectraPlot(Figure):
104
107
105 self.__isConfig = True
108 self.__isConfig = True
106
109
107 thisDatetime = datetime.datetime.fromtimestamp(dataOut.utctime)
110 thisDatetime = dataOut.datatime
108 title = "Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
111 title = "Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
109 xlabel = "Velocity (m/s)"
112 xlabel = "Velocity (m/s)"
110 ylabel = "Range (Km)"
113 ylabel = "Range (Km)"
@@ -223,7 +226,7 class RTIPlot(Figure):
223
226
224 def __getTimeLim(self, x, xmin, xmax):
227 def __getTimeLim(self, x, xmin, xmax):
225
228
226 thisdatetime = datetime.datetime.fromtimestamp(numpy.min(x))
229 thisdatetime = datetime.datetime.utcfromtimestamp(numpy.min(x))
227 thisdate = datetime.datetime.combine(thisdatetime.date(), datetime.time(0,0,0))
230 thisdate = datetime.datetime.combine(thisdatetime.date(), datetime.time(0,0,0))
228
231
229 ####################################################
232 ####################################################
@@ -284,7 +287,7 class RTIPlot(Figure):
284
287
285 tmin = None
288 tmin = None
286 tmax = None
289 tmax = None
287 x = dataOut.getDatatime()
290 x = dataOut.getTimeRange()
288 y = dataOut.getHeiRange()
291 y = dataOut.getHeiRange()
289 z = 10.*numpy.log10(dataOut.data_spc[channelIndexList,:,:])
292 z = 10.*numpy.log10(dataOut.data_spc[channelIndexList,:,:])
290 avg = numpy.average(z, axis=1)
293 avg = numpy.average(z, axis=1)
@@ -308,7 +311,7 class RTIPlot(Figure):
308
311
309 self.__isConfig = True
312 self.__isConfig = True
310
313
311 thisDatetime = datetime.datetime.fromtimestamp(dataOut.utctime)
314 thisDatetime = dataOut.datatime
312 title = "RTI: %s" %(thisDatetime.strftime("%d-%b-%Y"))
315 title = "RTI: %s" %(thisDatetime.strftime("%d-%b-%Y"))
313 xlabel = "Velocity (m/s)"
316 xlabel = "Velocity (m/s)"
314 ylabel = "Range (Km)"
317 ylabel = "Range (Km)"
@@ -457,7 +460,7 class SpectraPlot(Figure):
457
460
458 self.__isConfig = True
461 self.__isConfig = True
459
462
460 thisDatetime = datetime.datetime.fromtimestamp(dataOut.utctime)
463 thisDatetime = dataOut.datatime
461 title = "Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
464 title = "Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
462 xlabel = "Velocity (m/s)"
465 xlabel = "Velocity (m/s)"
463 ylabel = "Range (Km)"
466 ylabel = "Range (Km)"
@@ -564,7 +567,7 class Scope(Figure):
564 self.__isConfig = True
567 self.__isConfig = True
565
568
566
569
567 thisDatetime = datetime.datetime.fromtimestamp(dataOut.utctime)
570 thisDatetime = dataOut.datatime
568 title = "Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
571 title = "Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
569 xlabel = "Range (Km)"
572 xlabel = "Range (Km)"
570 ylabel = "Intensity"
573 ylabel = "Intensity"
General Comments 0
You need to be logged in to leave comments. Login now