##// END OF EJS Templates
schain
RSS
cambios para online offline
José Chávez -
r1019:42311705b9c8
parent child
Show More
Side by Side Unified
Context file:
r1019:42311705b9c8 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1730 +1,1786
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 #import h5py
13 #import h5py
14 import traceback
14 import traceback
15
15
16 try:
16 try:
17 from gevent import sleep
17 from gevent import sleep
18 except:
18 except:
19 from time import sleep
19 from time import sleep
20
20
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23
23
24 LOCALTIME = True
24 LOCALTIME = True
25
25
26 def isNumber(cad):
26 def isNumber(cad):
27 """
27 """
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29
29
30 Excepciones:
30 Excepciones:
31 Si un determinado string no puede ser convertido a numero
31 Si un determinado string no puede ser convertido a numero
32 Input:
32 Input:
33 str, string al cual se le analiza para determinar si convertible a un numero o no
33 str, string al cual se le analiza para determinar si convertible a un numero o no
34
34
35 Return:
35 Return:
36 True : si el string es uno numerico
36 True : si el string es uno numerico
37 False : no es un string numerico
37 False : no es un string numerico
38 """
38 """
39 try:
39 try:
40 float( cad )
40 float( cad )
41 return True
41 return True
42 except:
42 except:
43 return False
43 return False
44
44
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
49 Inputs:
49 Inputs:
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51
51
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 segundos contados desde 01/01/1970.
53 segundos contados desde 01/01/1970.
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 segundos contados desde 01/01/1970.
55 segundos contados desde 01/01/1970.
56
56
57 Return:
57 Return:
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 fecha especificado, de lo contrario retorna False.
59 fecha especificado, de lo contrario retorna False.
60
60
61 Excepciones:
61 Excepciones:
62 Si el archivo no existe o no puede ser abierto
62 Si el archivo no existe o no puede ser abierto
63 Si la cabecera no puede ser leida.
63 Si la cabecera no puede ser leida.
64
64
65 """
65 """
66 basicHeaderObj = BasicHeader(LOCALTIME)
66 basicHeaderObj = BasicHeader(LOCALTIME)
67
67
68 try:
68 try:
69 fp = open(filename,'rb')
69 fp = open(filename,'rb')
70 except IOError:
70 except IOError:
71 print "The file %s can't be opened" %(filename)
71 print "The file %s can't be opened" %(filename)
72 return 0
72 return 0
73
73
74 sts = basicHeaderObj.read(fp)
74 sts = basicHeaderObj.read(fp)
75 fp.close()
75 fp.close()
76
76
77 if not(sts):
77 if not(sts):
78 print "Skipping the file %s because it has not a valid header" %(filename)
78 print "Skipping the file %s because it has not a valid header" %(filename)
79 return 0
79 return 0
80
80
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 return 0
82 return 0
83
83
84 return 1
84 return 1
85
85
86 def isTimeInRange(thisTime, startTime, endTime):
86 def isTimeInRange(thisTime, startTime, endTime):
87
87
88 if endTime >= startTime:
88 if endTime >= startTime:
89 if (thisTime < startTime) or (thisTime > endTime):
89 if (thisTime < startTime) or (thisTime > endTime):
90 return 0
90 return 0
91
91
92 return 1
92 return 1
93 else:
93 else:
94 if (thisTime < startTime) and (thisTime > endTime):
94 if (thisTime < startTime) and (thisTime > endTime):
95 return 0
95 return 0
96
96
97 return 1
97 return 1
98
98
99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 """
100 """
101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102
102
103 Inputs:
103 Inputs:
104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105
105
106 startDate : fecha inicial del rango seleccionado en formato datetime.date
106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107
107
108 endDate : fecha final del rango seleccionado en formato datetime.date
108 endDate : fecha final del rango seleccionado en formato datetime.date
109
109
110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111
111
112 endTime : tiempo final del rango seleccionado en formato datetime.time
112 endTime : tiempo final del rango seleccionado en formato datetime.time
113
113
114 Return:
114 Return:
115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 fecha especificado, de lo contrario retorna False.
116 fecha especificado, de lo contrario retorna False.
117
117
118 Excepciones:
118 Excepciones:
119 Si el archivo no existe o no puede ser abierto
119 Si el archivo no existe o no puede ser abierto
120 Si la cabecera no puede ser leida.
120 Si la cabecera no puede ser leida.
121
121
122 """
122 """
123
123
124
124
125 try:
125 try:
126 fp = open(filename,'rb')
126 fp = open(filename,'rb')
127 except IOError:
127 except IOError:
128 print "The file %s can't be opened" %(filename)
128 print "The file %s can't be opened" %(filename)
129 return None
129 return None
130
130
131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 systemHeaderObj = SystemHeader()
132 systemHeaderObj = SystemHeader()
133 radarControllerHeaderObj = RadarControllerHeader()
133 radarControllerHeaderObj = RadarControllerHeader()
134 processingHeaderObj = ProcessingHeader()
134 processingHeaderObj = ProcessingHeader()
135
135
136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137
137
138 sts = firstBasicHeaderObj.read(fp)
138 sts = firstBasicHeaderObj.read(fp)
139
139
140 if not(sts):
140 if not(sts):
141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 return None
142 return None
143
143
144 if not systemHeaderObj.read(fp):
144 if not systemHeaderObj.read(fp):
145 return None
145 return None
146
146
147 if not radarControllerHeaderObj.read(fp):
147 if not radarControllerHeaderObj.read(fp):
148 return None
148 return None
149
149
150 if not processingHeaderObj.read(fp):
150 if not processingHeaderObj.read(fp):
151 return None
151 return None
152
152
153 filesize = os.path.getsize(filename)
153 filesize = os.path.getsize(filename)
154
154
155 offset = processingHeaderObj.blockSize + 24 #header size
155 offset = processingHeaderObj.blockSize + 24 #header size
156
156
157 if filesize <= offset:
157 if filesize <= offset:
158 print "[Reading] %s: This file has not enough data" %filename
158 print "[Reading] %s: This file has not enough data" %filename
159 return None
159 return None
160
160
161 fp.seek(-offset, 2)
161 fp.seek(-offset, 2)
162
162
163 sts = lastBasicHeaderObj.read(fp)
163 sts = lastBasicHeaderObj.read(fp)
164
164
165 fp.close()
165 fp.close()
166
166
167 thisDatetime = lastBasicHeaderObj.datatime
167 thisDatetime = lastBasicHeaderObj.datatime
168 thisTime_last_block = thisDatetime.time()
168 thisTime_last_block = thisDatetime.time()
169
169
170 thisDatetime = firstBasicHeaderObj.datatime
170 thisDatetime = firstBasicHeaderObj.datatime
171 thisDate = thisDatetime.date()
171 thisDate = thisDatetime.date()
172 thisTime_first_block = thisDatetime.time()
172 thisTime_first_block = thisDatetime.time()
173
173
174 #General case
174 #General case
175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 #-----------o----------------------------o-----------
176 #-----------o----------------------------o-----------
177 # startTime endTime
177 # startTime endTime
178
178
179 if endTime >= startTime:
179 if endTime >= startTime:
180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 return None
181 return None
182
182
183 return thisDatetime
183 return thisDatetime
184
184
185 #If endTime < startTime then endTime belongs to the next day
185 #If endTime < startTime then endTime belongs to the next day
186
186
187
187
188 #<<<<<<<<<<<o o>>>>>>>>>>>
188 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #-----------o----------------------------o-----------
189 #-----------o----------------------------o-----------
190 # endTime startTime
190 # endTime startTime
191
191
192 if (thisDate == startDate) and (thisTime_last_block < startTime):
192 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 return None
193 return None
194
194
195 if (thisDate == endDate) and (thisTime_first_block > endTime):
195 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 return None
196 return None
197
197
198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 return None
199 return None
200
200
201 return thisDatetime
201 return thisDatetime
202
202
203 def isFolderInDateRange(folder, startDate=None, endDate=None):
203 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 """
204 """
205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206
206
207 Inputs:
207 Inputs:
208 folder : nombre completo del directorio.
208 folder : nombre completo del directorio.
209 Su formato deberia ser "/path_root/?YYYYDDD"
209 Su formato deberia ser "/path_root/?YYYYDDD"
210
210
211 siendo:
211 siendo:
212 YYYY : Anio (ejemplo 2015)
212 YYYY : Anio (ejemplo 2015)
213 DDD : Dia del anio (ejemplo 305)
213 DDD : Dia del anio (ejemplo 305)
214
214
215 startDate : fecha inicial del rango seleccionado en formato datetime.date
215 startDate : fecha inicial del rango seleccionado en formato datetime.date
216
216
217 endDate : fecha final del rango seleccionado en formato datetime.date
217 endDate : fecha final del rango seleccionado en formato datetime.date
218
218
219 Return:
219 Return:
220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 fecha especificado, de lo contrario retorna False.
221 fecha especificado, de lo contrario retorna False.
222 Excepciones:
222 Excepciones:
223 Si el directorio no tiene el formato adecuado
223 Si el directorio no tiene el formato adecuado
224 """
224 """
225
225
226 basename = os.path.basename(folder)
226 basename = os.path.basename(folder)
227
227
228 if not isRadarFolder(basename):
228 if not isRadarFolder(basename):
229 print "The folder %s has not the rigth format" %folder
229 print "The folder %s has not the rigth format" %folder
230 return 0
230 return 0
231
231
232 if startDate and endDate:
232 if startDate and endDate:
233 thisDate = getDateFromRadarFolder(basename)
233 thisDate = getDateFromRadarFolder(basename)
234
234
235 if thisDate < startDate:
235 if thisDate < startDate:
236 return 0
236 return 0
237
237
238 if thisDate > endDate:
238 if thisDate > endDate:
239 return 0
239 return 0
240
240
241 return 1
241 return 1
242
242
243 def isFileInDateRange(filename, startDate=None, endDate=None):
243 def isFileInDateRange(filename, startDate=None, endDate=None):
244 """
244 """
245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246
246
247 Inputs:
247 Inputs:
248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249
249
250 Su formato deberia ser "?YYYYDDDsss"
250 Su formato deberia ser "?YYYYDDDsss"
251
251
252 siendo:
252 siendo:
253 YYYY : Anio (ejemplo 2015)
253 YYYY : Anio (ejemplo 2015)
254 DDD : Dia del anio (ejemplo 305)
254 DDD : Dia del anio (ejemplo 305)
255 sss : set
255 sss : set
256
256
257 startDate : fecha inicial del rango seleccionado en formato datetime.date
257 startDate : fecha inicial del rango seleccionado en formato datetime.date
258
258
259 endDate : fecha final del rango seleccionado en formato datetime.date
259 endDate : fecha final del rango seleccionado en formato datetime.date
260
260
261 Return:
261 Return:
262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 fecha especificado, de lo contrario retorna False.
263 fecha especificado, de lo contrario retorna False.
264 Excepciones:
264 Excepciones:
265 Si el archivo no tiene el formato adecuado
265 Si el archivo no tiene el formato adecuado
266 """
266 """
267
267
268 basename = os.path.basename(filename)
268 basename = os.path.basename(filename)
269
269
270 if not isRadarFile(basename):
270 if not isRadarFile(basename):
271 print "The filename %s has not the rigth format" %filename
271 print "The filename %s has not the rigth format" %filename
272 return 0
272 return 0
273
273
274 if startDate and endDate:
274 if startDate and endDate:
275 thisDate = getDateFromRadarFile(basename)
275 thisDate = getDateFromRadarFile(basename)
276
276
277 if thisDate < startDate:
277 if thisDate < startDate:
278 return 0
278 return 0
279
279
280 if thisDate > endDate:
280 if thisDate > endDate:
281 return 0
281 return 0
282
282
283 return 1
283 return 1
284
284
285 def getFileFromSet(path, ext, set):
285 def getFileFromSet(path, ext, set):
286 validFilelist = []
286 validFilelist = []
287 fileList = os.listdir(path)
287 fileList = os.listdir(path)
288
288
289 # 0 1234 567 89A BCDE
289 # 0 1234 567 89A BCDE
290 # H YYYY DDD SSS .ext
290 # H YYYY DDD SSS .ext
291
291
292 for thisFile in fileList:
292 for thisFile in fileList:
293 try:
293 try:
294 year = int(thisFile[1:5])
294 year = int(thisFile[1:5])
295 doy = int(thisFile[5:8])
295 doy = int(thisFile[5:8])
296 except:
296 except:
297 continue
297 continue
298
298
299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 continue
300 continue
301
301
302 validFilelist.append(thisFile)
302 validFilelist.append(thisFile)
303
303
304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305
305
306 if len(myfile)!= 0:
306 if len(myfile)!= 0:
307 return myfile[0]
307 return myfile[0]
308 else:
308 else:
309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 print 'the filename %s does not exist'%filename
310 print 'the filename %s does not exist'%filename
311 print '...going to the last file: '
311 print '...going to the last file: '
312
312
313 if validFilelist:
313 if validFilelist:
314 validFilelist = sorted( validFilelist, key=str.lower )
314 validFilelist = sorted( validFilelist, key=str.lower )
315 return validFilelist[-1]
315 return validFilelist[-1]
316
316
317 return None
317 return None
318
318
319 def getlastFileFromPath(path, ext):
319 def getlastFileFromPath(path, ext):
320 """
320 """
321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323
323
324 Input:
324 Input:
325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 ext : extension de los files contenidos en una carpeta
326 ext : extension de los files contenidos en una carpeta
327
327
328 Return:
328 Return:
329 El ultimo file de una determinada carpeta, no se considera el path.
329 El ultimo file de una determinada carpeta, no se considera el path.
330 """
330 """
331 validFilelist = []
331 validFilelist = []
332 fileList = os.listdir(path)
332 fileList = os.listdir(path)
333
333
334 # 0 1234 567 89A BCDE
334 # 0 1234 567 89A BCDE
335 # H YYYY DDD SSS .ext
335 # H YYYY DDD SSS .ext
336
336
337 for thisFile in fileList:
337 for thisFile in fileList:
338
338
339 year = thisFile[1:5]
339 year = thisFile[1:5]
340 if not isNumber(year):
340 if not isNumber(year):
341 continue
341 continue
342
342
343 doy = thisFile[5:8]
343 doy = thisFile[5:8]
344 if not isNumber(doy):
344 if not isNumber(doy):
345 continue
345 continue
346
346
347 year = int(year)
347 year = int(year)
348 doy = int(doy)
348 doy = int(doy)
349
349
350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 continue
351 continue
352
352
353 validFilelist.append(thisFile)
353 validFilelist.append(thisFile)
354
354
355 if validFilelist:
355 if validFilelist:
356 validFilelist = sorted( validFilelist, key=str.lower )
356 validFilelist = sorted( validFilelist, key=str.lower )
357 return validFilelist[-1]
357 return validFilelist[-1]
358
358
359 return None
359 return None
360
360
361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 """
362 """
363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 el path exacto de un determinado file.
365 el path exacto de un determinado file.
366
366
367 Example :
367 Example :
368 nombre correcto del file es .../.../D2009307/P2009307367.ext
368 nombre correcto del file es .../.../D2009307/P2009307367.ext
369
369
370 Entonces la funcion prueba con las siguientes combinaciones
370 Entonces la funcion prueba con las siguientes combinaciones
371 .../.../y2009307367.ext
371 .../.../y2009307367.ext
372 .../.../Y2009307367.ext
372 .../.../Y2009307367.ext
373 .../.../x2009307/y2009307367.ext
373 .../.../x2009307/y2009307367.ext
374 .../.../x2009307/Y2009307367.ext
374 .../.../x2009307/Y2009307367.ext
375 .../.../X2009307/y2009307367.ext
375 .../.../X2009307/y2009307367.ext
376 .../.../X2009307/Y2009307367.ext
376 .../.../X2009307/Y2009307367.ext
377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378
378
379 Return:
379 Return:
380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 para el filename
382 para el filename
383 """
383 """
384 fullfilename = None
384 fullfilename = None
385 find_flag = False
385 find_flag = False
386 filename = None
386 filename = None
387
387
388 prefixDirList = [None,'d','D']
388 prefixDirList = [None,'d','D']
389 if ext.lower() == ".r": #voltage
389 if ext.lower() == ".r": #voltage
390 prefixFileList = ['d','D']
390 prefixFileList = ['d','D']
391 elif ext.lower() == ".pdata": #spectra
391 elif ext.lower() == ".pdata": #spectra
392 prefixFileList = ['p','P']
392 prefixFileList = ['p','P']
393 else:
393 else:
394 return None, filename
394 return None, filename
395
395
396 #barrido por las combinaciones posibles
396 #barrido por las combinaciones posibles
397 for prefixDir in prefixDirList:
397 for prefixDir in prefixDirList:
398 thispath = path
398 thispath = path
399 if prefixDir != None:
399 if prefixDir != None:
400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 if foldercounter == 0:
401 if foldercounter == 0:
402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 else:
403 else:
404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408
408
409 if os.path.exists( fullfilename ): #verifico que exista
409 if os.path.exists( fullfilename ): #verifico que exista
410 find_flag = True
410 find_flag = True
411 break
411 break
412 if find_flag:
412 if find_flag:
413 break
413 break
414
414
415 if not(find_flag):
415 if not(find_flag):
416 return None, filename
416 return None, filename
417
417
418 return fullfilename, filename
418 return fullfilename, filename
419
419
420 def isRadarFolder(folder):
420 def isRadarFolder(folder):
421 try:
421 try:
422 year = int(folder[1:5])
422 year = int(folder[1:5])
423 doy = int(folder[5:8])
423 doy = int(folder[5:8])
424 except:
424 except:
425 return 0
425 return 0
426
426
427 return 1
427 return 1
428
428
429 def isRadarFile(file):
429 def isRadarFile(file):
430 try:
430 try:
431 year = int(file[1:5])
431 year = int(file[1:5])
432 doy = int(file[5:8])
432 doy = int(file[5:8])
433 set = int(file[8:11])
433 set = int(file[8:11])
434 except:
434 except:
435 return 0
435 return 0
436
436
437 return 1
437 return 1
438
438
439 def getDateFromRadarFile(file):
439 def getDateFromRadarFile(file):
440 try:
440 try:
441 year = int(file[1:5])
441 year = int(file[1:5])
442 doy = int(file[5:8])
442 doy = int(file[5:8])
443 set = int(file[8:11])
443 set = int(file[8:11])
444 except:
444 except:
445 return None
445 return None
446
446
447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 return thisDate
448 return thisDate
449
449
450 def getDateFromRadarFolder(folder):
450 def getDateFromRadarFolder(folder):
451 try:
451 try:
452 year = int(folder[1:5])
452 year = int(folder[1:5])
453 doy = int(folder[5:8])
453 doy = int(folder[5:8])
454 except:
454 except:
455 return None
455 return None
456
456
457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 return thisDate
458 return thisDate
459
459
460 class JRODataIO:
460 class JRODataIO:
461
461
462 c = 3E8
462 c = 3E8
463
463
464 isConfig = False
464 isConfig = False
465
465
466 basicHeaderObj = None
466 basicHeaderObj = None
467
467
468 systemHeaderObj = None
468 systemHeaderObj = None
469
469
470 radarControllerHeaderObj = None
470 radarControllerHeaderObj = None
471
471
472 processingHeaderObj = None
472 processingHeaderObj = None
473
473
474 dtype = None
474 dtype = None
475
475
476 pathList = []
476 pathList = []
477
477
478 filenameList = []
478 filenameList = []
479
479
480 filename = None
480 filename = None
481
481
482 ext = None
482 ext = None
483
483
484 flagIsNewFile = 1
484 flagIsNewFile = 1
485
485
486 flagDiscontinuousBlock = 0
486 flagDiscontinuousBlock = 0
487
487
488 flagIsNewBlock = 0
488 flagIsNewBlock = 0
489
489
490 fp = None
490 fp = None
491
491
492 firstHeaderSize = 0
492 firstHeaderSize = 0
493
493
494 basicHeaderSize = 24
494 basicHeaderSize = 24
495
495
496 versionFile = 1103
496 versionFile = 1103
497
497
498 fileSize = None
498 fileSize = None
499
499
500 # ippSeconds = None
500 # ippSeconds = None
501
501
502 fileSizeByHeader = None
502 fileSizeByHeader = None
503
503
504 fileIndex = None
504 fileIndex = None
505
505
506 profileIndex = None
506 profileIndex = None
507
507
508 blockIndex = None
508 blockIndex = None
509
509
510 nTotalBlocks = None
510 nTotalBlocks = None
511
511
512 maxTimeStep = 30
512 maxTimeStep = 30
513
513
514 lastUTTime = None
514 lastUTTime = None
515
515
516 datablock = None
516 datablock = None
517
517
518 dataOut = None
518 dataOut = None
519
519
520 blocksize = None
520 blocksize = None
521
521
522 getByBlock = False
522 getByBlock = False
523
523
524 def __init__(self):
524 def __init__(self):
525
525
526 raise NotImplementedError
526 raise NotImplementedError
527
527
528 def run(self):
528 def run(self):
529
529
530 raise NotImplementedError
530 raise NotImplementedError
531
531
532 def getDtypeWidth(self):
532 def getDtypeWidth(self):
533
533
534 dtype_index = get_dtype_index(self.dtype)
534 dtype_index = get_dtype_index(self.dtype)
535 dtype_width = get_dtype_width(dtype_index)
535 dtype_width = get_dtype_width(dtype_index)
536
536
537 return dtype_width
537 return dtype_width
538
538
539 class JRODataReader(JRODataIO):
539 class JRODataReader(JRODataIO):
540
540
541
541 firstTime = True
542 online = 0
542 online = 0
543
543
544 realtime = 0
544 realtime = 0
545
545
546 nReadBlocks = 0
546 nReadBlocks = 0
547
547
548 delay = 10 #number of seconds waiting a new file
548 delay = 10 #number of seconds waiting a new file
549
549
550 nTries = 3 #quantity tries
550 nTries = 3 #quantity tries
551
551
552 nFiles = 3 #number of files for searching
552 nFiles = 3 #number of files for searching
553
553
554 path = None
554 path = None
555
555
556 foldercounter = 0
556 foldercounter = 0
557
557
558 flagNoMoreFiles = 0
558 flagNoMoreFiles = 0
559
559
560 datetimeList = []
560 datetimeList = []
561
561
562 __isFirstTimeOnline = 1
562 __isFirstTimeOnline = 1
563
563
564 __printInfo = True
564 __printInfo = True
565
565
566 profileIndex = None
566 profileIndex = None
567
567
568 nTxs = 1
568 nTxs = 1
569
569
570 txIndex = None
570 txIndex = None
571
571
572 #Added--------------------
572 #Added--------------------
573
573
574 selBlocksize = None
574 selBlocksize = None
575
575
576 selBlocktime = None
576 selBlocktime = None
577
577
578
578 onlineWithDate = False
579 def __init__(self):
579 def __init__(self):
580
580
581 """
581 """
582 This class is used to find data files
582 This class is used to find data files
583
583
584 Example:
584 Example:
585 reader = JRODataReader()
585 reader = JRODataReader()
586 fileList = reader.findDataFiles()
586 fileList = reader.findDataFiles()
587
587
588 """
588 """
589 pass
589 pass
590
590
591
591
592 def createObjByDefault(self):
592 def createObjByDefault(self):
593 """
593 """
594
594
595 """
595 """
596 raise NotImplementedError
596 raise NotImplementedError
597
597
598 def getBlockDimension(self):
598 def getBlockDimension(self):
599
599
600 raise NotImplementedError
600 raise NotImplementedError
601
601
602 def __searchFilesOffLine(self,
602 def __searchFilesOffLine(self,
603 path,
603 path,
604 startDate=None,
604 startDate=None,
605 endDate=None,
605 endDate=None,
606 startTime=datetime.time(0,0,0),
606 startTime=datetime.time(0,0,0),
607 endTime=datetime.time(23,59,59),
607 endTime=datetime.time(23,59,59),
608 set=None,
608 set=None,
609 expLabel='',
609 expLabel='',
610 ext='.r',
610 ext='.r',
611 walk=True):
611 walk=True):
612
613 self.filenameList = []
612 self.filenameList = []
614 self.datetimeList = []
613 self.datetimeList = []
615
614
616 pathList = []
615 pathList = []
617
616
618 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
617 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
619
618
620 if dateList == []:
619 if dateList == []:
621 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
620 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
622 return None, None
621 return None, None
623
622
624 if len(dateList) > 1:
623 if len(dateList) > 1:
625 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
624 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
626 else:
625 else:
627 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
626 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
628
627
629 filenameList = []
628 filenameList = []
630 datetimeList = []
629 datetimeList = []
631
630
632 for thisPath in pathList:
631 for thisPath in pathList:
633 # thisPath = pathList[pathDict[file]]
632 # thisPath = pathList[pathDict[file]]
634
633
635 fileList = glob.glob1(thisPath, "*%s" %ext)
634 fileList = glob.glob1(thisPath, "*%s" %ext)
636 fileList.sort()
635 fileList.sort()
637
636
638 for file in fileList:
637 for file in fileList:
639
638
640 filename = os.path.join(thisPath,file)
639 filename = os.path.join(thisPath,file)
641
640
642 if not isFileInDateRange(filename, startDate, endDate):
641 if not isFileInDateRange(filename, startDate, endDate):
643 continue
642 continue
644
643
645 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
644 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
646
645
647 if not(thisDatetime):
646 if not(thisDatetime):
648 continue
647 continue
649
648
650 filenameList.append(filename)
649 filenameList.append(filename)
651 datetimeList.append(thisDatetime)
650 datetimeList.append(thisDatetime)
652
651
653 if not(filenameList):
652 if not(filenameList):
654 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
653 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
655 return None, None
654 return None, None
656
655
657 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
656 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
658 print
657 print
659
658
660 for i in range(len(filenameList)):
659 for i in range(len(filenameList)):
661 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
660 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
662
661
663 self.filenameList = filenameList
662 self.filenameList = filenameList
664 self.datetimeList = datetimeList
663 self.datetimeList = datetimeList
665
664
665
666 return pathList, filenameList
666 return pathList, filenameList
667
667
668 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None, startDate=None, endDate=None):
668 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
669
669
670 """
670 """
671 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
671 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
672 devuelve el archivo encontrado ademas de otros datos.
672 devuelve el archivo encontrado ademas de otros datos.
673
674 Input:
675 path : carpeta donde estan contenidos los files que contiene data
676
677 expLabel : Nombre del subexperimento (subfolder)
678
679 ext : extension de los files
680
681 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
682
683 Return:
684 directory : eL directorio donde esta el file encontrado
685 filename : el ultimo file de una determinada carpeta
686 year : el anho
687 doy : el numero de dia del anho
688 set : el set del archivo
689
673
674 Input:
675 path : carpeta donde estan contenidos los files que contiene data
676
677 expLabel : Nombre del subexperimento (subfolder)
678
679 ext : extension de los files
680
681 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
682
683 Return:
684 directory : eL directorio donde esta el file encontrado
685 filename : el ultimo file de una determinada carpeta
686 year : el anho
687 doy : el numero de dia del anho
688 set : el set del archivo
689
690
690
691 """
691 """
692 pathList = None
693 filenameList = None
692 if not os.path.isdir(path):
694 if not os.path.isdir(path):
693 return None, None, None, None, None, None
695 return None, None, None, None, None, None
694
696
695 dirList = []
697 dirList = []
696
698
697 if not walk:
699 if not walk:
698 fullpath = path
700 fullpath = path
699 foldercounter = 0
701 foldercounter = 0
700 else:
702 else:
701 # Filtra solo los directorios
703 # Filtra solo los directorios
702 for thisPath in os.listdir(path):
704 for thisPath in os.listdir(path):
703 if not os.path.isdir(os.path.join(path,thisPath)):
705 if not os.path.isdir(os.path.join(path,thisPath)):
704 continue
706 continue
705 if not isRadarFolder(thisPath):
707 if not isRadarFolder(thisPath):
706 continue
708 continue
707
709
708 dirList.append(thisPath)
710 dirList.append(thisPath)
709
711
710 if not(dirList):
712 if not(dirList):
711 return None, None, None, None, None, None
713 return None, None, None, None, None, None
712
714
713 dirList = sorted( dirList, key=str.lower )
715 dirList = sorted( dirList, key=str.lower )
714
716
715 doypath = dirList[-1]
717 doypath = dirList[-1]
716 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
718 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
717 fullpath = os.path.join(path, doypath, expLabel)
719 fullpath = os.path.join(path, doypath, expLabel)
718
720
719
721
720 print "[Reading] %s folder was found: " %(fullpath )
722 print "[Reading] %s folder was found: " %(fullpath )
721
723
722 if set == None:
724 if set == None:
723 filename = getlastFileFromPath(fullpath, ext)
725 filename = getlastFileFromPath(fullpath, ext)
724 else:
726 else:
725 filename = getFileFromSet(fullpath, ext, set)
727 filename = getFileFromSet(fullpath, ext, set)
726
728
727 if not(filename):
729 if not(filename):
728 return None, None, None, None, None, None
730 return None, None, None, None, None, None
729
731
730 print "[Reading] %s file was found" %(filename)
732 print "[Reading] %s file was found" %(filename)
731
733
732 if not(self.__verifyFile(os.path.join(fullpath, filename))):
734 if not(self.__verifyFile(os.path.join(fullpath, filename))):
733 return None, None, None, None, None, None
735 return None, None, None, None, None, None
734
736
735 year = int( filename[1:5] )
737 year = int( filename[1:5] )
736 doy = int( filename[5:8] )
738 doy = int( filename[5:8] )
737 set = int( filename[8:11] )
739 set = int( filename[8:11] )
738
740
739 return fullpath, foldercounter, filename, year, doy, set
741 return fullpath, foldercounter, filename, year, doy, set
740
742
741 def __setNextFileOffline(self):
743 def __setNextFileOffline(self):
742
744
743 idFile = self.fileIndex
745 idFile = self.fileIndex
744
746
745 while (True):
747 while (True):
746 idFile += 1
748 idFile += 1
747 if not(idFile < len(self.filenameList)):
749 if not(idFile < len(self.filenameList)):
748 self.flagNoMoreFiles = 1
750 self.flagNoMoreFiles = 1
749 # print "[Reading] No more Files"
751 # print "[Reading] No more Files"
750 return 0
752 return 0
751
753
752 filename = self.filenameList[idFile]
754 filename = self.filenameList[idFile]
753
755
754 if not(self.__verifyFile(filename)):
756 if not(self.__verifyFile(filename)):
755 continue
757 continue
756
758
757 fileSize = os.path.getsize(filename)
759 fileSize = os.path.getsize(filename)
758 fp = open(filename,'rb')
760 fp = open(filename,'rb')
759 break
761 break
760
762
761 self.flagIsNewFile = 1
763 self.flagIsNewFile = 1
762 self.fileIndex = idFile
764 self.fileIndex = idFile
763 self.filename = filename
765 self.filename = filename
764 self.fileSize = fileSize
766 self.fileSize = fileSize
765 self.fp = fp
767 self.fp = fp
766
768
767 # print "[Reading] Setting the file: %s"%self.filename
769 #print "[Reading] Setting the file: %s"%self.filename
768
770
769 return 1
771 return 1
770
772
771 def __setNextFileOnline(self):
773 def __setNextFileOnline(self):
772 """
774 """
773 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
775 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
774 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
776 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
775 siguientes.
777 siguientes.
778
779 Affected:
780 self.flagIsNewFile
781 self.filename
782 self.fileSize
783 self.fp
784 self.set
785 self.flagNoMoreFiles
786
787 Return:
788 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
789 1 : si el file fue abierto con exito y esta listo a ser leido
776
790
777 Affected:
791 Excepciones:
778 self.flagIsNewFile
792 Si un determinado file no puede ser abierto
779 self.filename
793 """
780 self.fileSize
781 self.fp
782 self.set
783 self.flagNoMoreFiles
784
785 Return:
786 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
787 1 : si el file fue abierto con exito y esta listo a ser leido
788
794
789 Excepciones:
790 Si un determinado file no puede ser abierto
791 """
792 nFiles = 0
795 nFiles = 0
793 fileOk_flag = False
796 fileOk_flag = False
794 firstTime_flag = True
797 firstTime_flag = True
795
798
796 self.set += 1
799 self.set += 1
797
800
798 if self.set > 999:
801 if self.set > 999:
799 self.set = 0
802 self.set = 0
800 self.foldercounter += 1
803 self.foldercounter += 1
801
804
802 #busca el 1er file disponible
805 #busca el 1er file disponible
803 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
806 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
804 if fullfilename:
807 if fullfilename:
805 if self.__verifyFile(fullfilename, False):
808 if self.__verifyFile(fullfilename, False):
806 fileOk_flag = True
809 fileOk_flag = True
807
810
808 #si no encuentra un file entonces espera y vuelve a buscar
811 #si no encuentra un file entonces espera y vuelve a buscar
809 if not(fileOk_flag):
812 if not(fileOk_flag):
810 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
813 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
811
814
812 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
815 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
813 tries = self.nTries
816 tries = self.nTries
814 else:
817 else:
815 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
818 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
816
819
817 for nTries in range( tries ):
820 for nTries in range( tries ):
818 if firstTime_flag:
821 if firstTime_flag:
819 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
822 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
820 sleep( self.delay )
823 sleep( self.delay )
821 else:
824 else:
822 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
825 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
823
826
824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
827 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 if fullfilename:
828 if fullfilename:
826 if self.__verifyFile(fullfilename):
829 if self.__verifyFile(fullfilename):
827 fileOk_flag = True
830 fileOk_flag = True
828 break
831 break
829
832
830 if fileOk_flag:
833 if fileOk_flag:
831 break
834 break
832
835
833 firstTime_flag = False
836 firstTime_flag = False
834
837
835 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
838 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
836 self.set += 1
839 self.set += 1
837
840
838 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
841 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
839 self.set = 0
842 self.set = 0
840 self.doy += 1
843 self.doy += 1
841 self.foldercounter = 0
844 self.foldercounter = 0
842
845
843 if fileOk_flag:
846 if fileOk_flag:
844 self.fileSize = os.path.getsize( fullfilename )
847 self.fileSize = os.path.getsize( fullfilename )
845 self.filename = fullfilename
848 self.filename = fullfilename
846 self.flagIsNewFile = 1
849 self.flagIsNewFile = 1
847 if self.fp != None: self.fp.close()
850 if self.fp != None: self.fp.close()
848 self.fp = open(fullfilename, 'rb')
851 self.fp = open(fullfilename, 'rb')
849 self.flagNoMoreFiles = 0
852 self.flagNoMoreFiles = 0
850 # print '[Reading] Setting the file: %s' % fullfilename
853 # print '[Reading] Setting the file: %s' % fullfilename
851 else:
854 else:
852 self.fileSize = 0
855 self.fileSize = 0
853 self.filename = None
856 self.filename = None
854 self.flagIsNewFile = 0
857 self.flagIsNewFile = 0
855 self.fp = None
858 self.fp = None
856 self.flagNoMoreFiles = 1
859 self.flagNoMoreFiles = 1
857 # print '[Reading] No more files to read'
860 # print '[Reading] No more files to read'
858
861
859 return fileOk_flag
862 return fileOk_flag
860
863
861 def setNextFile(self):
864 def setNextFile(self):
862 if self.fp != None:
865 if self.fp != None:
863 self.fp.close()
866 self.fp.close()
864
865 if self.online:
867 if self.online:
866 newFile = self.__setNextFileOnline()
868 newFile = self.__setNextFileOnline()
867 else:
869 else:
868 newFile = self.__setNextFileOffline()
870 newFile = self.__setNextFileOffline()
869
870 if not(newFile):
871 if not(newFile):
872 if self.onlineWithDate is True:
873 self.onlineWithDate=False
874 self.online = True
875 self.firstTime = False
876 self.setup(
877 path=self.path,
878 startDate=self.startDate,
879 endDate=self.endDate,
880 startTime=self.startTime ,
881 endTime=self.endTime,
882 set=self.set,
883 expLabel=self.expLabel,
884 ext=self.ext,
885 online=self.online,
886 delay=self.delay,
887 walk=self.walk,
888 getblock=self.getblock,
889 nTxs=self.nTxs,
890 realtime=self.realtime,
891 blocksize=self.blocksize,
892 blocktime=self.blocktime
893 )
894 return 1
871 print '[Reading] No more files to read'
895 print '[Reading] No more files to read'
872 return 0
896 return 0
873
897
874 print '[Reading] Setting the file: %s' % self.filename
898 print '[Reading] Setting the file: %s' % self.filename
875
899
876 self.__readFirstHeader()
900 self.__readFirstHeader()
877 self.nReadBlocks = 0
901 self.nReadBlocks = 0
878 return 1
902 return 1
879
903
880 def __waitNewBlock(self):
904 def __waitNewBlock(self):
881 """
905 """
882 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
906 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
883
907
884 Si el modo de lectura es OffLine siempre retorn 0
908 Si el modo de lectura es OffLine siempre retorn 0
885 """
909 """
886 if not self.online:
910 if not self.online:
887 return 0
911 return 0
888
912
889 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
913 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
890 return 0
914 return 0
891
915
892 currentPointer = self.fp.tell()
916 currentPointer = self.fp.tell()
893
917
894 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
918 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
895
919
896 for nTries in range( self.nTries ):
920 for nTries in range( self.nTries ):
897
921
898 self.fp.close()
922 self.fp.close()
899 self.fp = open( self.filename, 'rb' )
923 self.fp = open( self.filename, 'rb' )
900 self.fp.seek( currentPointer )
924 self.fp.seek( currentPointer )
901
925
902 self.fileSize = os.path.getsize( self.filename )
926 self.fileSize = os.path.getsize( self.filename )
903 currentSize = self.fileSize - currentPointer
927 currentSize = self.fileSize - currentPointer
904
928
905 if ( currentSize >= neededSize ):
929 if ( currentSize >= neededSize ):
906 self.basicHeaderObj.read(self.fp)
930 self.basicHeaderObj.read(self.fp)
907 return 1
931 return 1
908
932
909 if self.fileSize == self.fileSizeByHeader:
933 if self.fileSize == self.fileSizeByHeader:
910 # self.flagEoF = True
934 # self.flagEoF = True
911 return 0
935 return 0
912
936
913 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
937 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
914 sleep( self.delay )
938 sleep( self.delay )
915
939
916
940
917 return 0
941 return 0
918
942
919 def waitDataBlock(self,pointer_location):
943 def waitDataBlock(self,pointer_location):
920
944
921 currentPointer = pointer_location
945 currentPointer = pointer_location
922
946
923 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
947 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
924
948
925 for nTries in range( self.nTries ):
949 for nTries in range( self.nTries ):
926 self.fp.close()
950 self.fp.close()
927 self.fp = open( self.filename, 'rb' )
951 self.fp = open( self.filename, 'rb' )
928 self.fp.seek( currentPointer )
952 self.fp.seek( currentPointer )
929
953
930 self.fileSize = os.path.getsize( self.filename )
954 self.fileSize = os.path.getsize( self.filename )
931 currentSize = self.fileSize - currentPointer
955 currentSize = self.fileSize - currentPointer
932
956
933 if ( currentSize >= neededSize ):
957 if ( currentSize >= neededSize ):
934 return 1
958 return 1
935
959
936 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
960 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
937 sleep( self.delay )
961 sleep( self.delay )
938
962
939 return 0
963 return 0
940
964
941 def __jumpToLastBlock(self):
965 def __jumpToLastBlock(self):
942
966
943 if not(self.__isFirstTimeOnline):
967 if not(self.__isFirstTimeOnline):
944 return
968 return
945
969
946 csize = self.fileSize - self.fp.tell()
970 csize = self.fileSize - self.fp.tell()
947 blocksize = self.processingHeaderObj.blockSize
971 blocksize = self.processingHeaderObj.blockSize
948
972
949 #salta el primer bloque de datos
973 #salta el primer bloque de datos
950 if csize > self.processingHeaderObj.blockSize:
974 if csize > self.processingHeaderObj.blockSize:
951 self.fp.seek(self.fp.tell() + blocksize)
975 self.fp.seek(self.fp.tell() + blocksize)
952 else:
976 else:
953 return
977 return
954
978
955 csize = self.fileSize - self.fp.tell()
979 csize = self.fileSize - self.fp.tell()
956 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
980 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
957 while True:
981 while True:
958
982
959 if self.fp.tell()<self.fileSize:
983 if self.fp.tell()<self.fileSize:
960 self.fp.seek(self.fp.tell() + neededsize)
984 self.fp.seek(self.fp.tell() + neededsize)
961 else:
985 else:
962 self.fp.seek(self.fp.tell() - neededsize)
986 self.fp.seek(self.fp.tell() - neededsize)
963 break
987 break
964
988
965 # csize = self.fileSize - self.fp.tell()
989 # csize = self.fileSize - self.fp.tell()
966 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
990 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
967 # factor = int(csize/neededsize)
991 # factor = int(csize/neededsize)
968 # if factor > 0:
992 # if factor > 0:
969 # self.fp.seek(self.fp.tell() + factor*neededsize)
993 # self.fp.seek(self.fp.tell() + factor*neededsize)
970
994
971 self.flagIsNewFile = 0
995 self.flagIsNewFile = 0
972 self.__isFirstTimeOnline = 0
996 self.__isFirstTimeOnline = 0
973
997
974 def __setNewBlock(self):
998 def __setNewBlock(self):
975
999
976 if self.fp == None:
1000 if self.fp == None:
977 return 0
1001 return 0
978
1002
979 # if self.online:
1003 # if self.online:
980 # self.__jumpToLastBlock()
1004 # self.__jumpToLastBlock()
981
1005
982 if self.flagIsNewFile:
1006 if self.flagIsNewFile:
983 self.lastUTTime = self.basicHeaderObj.utc
1007 self.lastUTTime = self.basicHeaderObj.utc
984 return 1
1008 return 1
985
1009
986 if self.realtime:
1010 if self.realtime:
987 self.flagDiscontinuousBlock = 1
1011 self.flagDiscontinuousBlock = 1
988 if not(self.setNextFile()):
1012 if not(self.setNextFile()):
989 return 0
1013 return 0
990 else:
1014 else:
991 return 1
1015 return 1
992
1016
993 currentSize = self.fileSize - self.fp.tell()
1017 currentSize = self.fileSize - self.fp.tell()
994 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1018 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995
1019
996 if (currentSize >= neededSize):
1020 if (currentSize >= neededSize):
997 self.basicHeaderObj.read(self.fp)
1021 self.basicHeaderObj.read(self.fp)
998 self.lastUTTime = self.basicHeaderObj.utc
1022 self.lastUTTime = self.basicHeaderObj.utc
999 return 1
1023 return 1
1000
1024
1001 if self.__waitNewBlock():
1025 if self.__waitNewBlock():
1002 self.lastUTTime = self.basicHeaderObj.utc
1026 self.lastUTTime = self.basicHeaderObj.utc
1003 return 1
1027 return 1
1004
1028
1005 if not(self.setNextFile()):
1029 if not(self.setNextFile()):
1006 return 0
1030 return 0
1007
1031
1008 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1032 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1009 self.lastUTTime = self.basicHeaderObj.utc
1033 self.lastUTTime = self.basicHeaderObj.utc
1010
1034
1011 self.flagDiscontinuousBlock = 0
1035 self.flagDiscontinuousBlock = 0
1012
1036
1013 if deltaTime > self.maxTimeStep:
1037 if deltaTime > self.maxTimeStep:
1014 self.flagDiscontinuousBlock = 1
1038 self.flagDiscontinuousBlock = 1
1015
1039
1016 return 1
1040 return 1
1017
1041
1018 def readNextBlock(self):
1042 def readNextBlock(self):
1019
1043
1020 #Skip block out of startTime and endTime
1044 #Skip block out of startTime and endTime
1021 while True:
1045 while True:
1022 if not(self.__setNewBlock()):
1046 if not(self.__setNewBlock()):
1023 return 0
1047 return 0
1024
1025 if not(self.readBlock()):
1048 if not(self.readBlock()):
1026 return 0
1049 return 0
1027
1028 self.getBasicHeader()
1050 self.getBasicHeader()
1029
1030 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1051 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1031
1052
1032 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1053 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1033 self.processingHeaderObj.dataBlocksPerFile,
1054 self.processingHeaderObj.dataBlocksPerFile,
1034 self.dataOut.datatime.ctime())
1055 self.dataOut.datatime.ctime())
1035 continue
1056 continue
1036
1057
1037 break
1058 break
1038
1039 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1059 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1040 self.processingHeaderObj.dataBlocksPerFile,
1060 self.processingHeaderObj.dataBlocksPerFile,
1041 self.dataOut.datatime.ctime())
1061 self.dataOut.datatime.ctime())
1042 return 1
1062 return 1
1043
1063
1044 def __readFirstHeader(self):
1064 def __readFirstHeader(self):
1045
1065
1046 self.basicHeaderObj.read(self.fp)
1066 self.basicHeaderObj.read(self.fp)
1047 self.systemHeaderObj.read(self.fp)
1067 self.systemHeaderObj.read(self.fp)
1048 self.radarControllerHeaderObj.read(self.fp)
1068 self.radarControllerHeaderObj.read(self.fp)
1049 self.processingHeaderObj.read(self.fp)
1069 self.processingHeaderObj.read(self.fp)
1050
1070
1051 self.firstHeaderSize = self.basicHeaderObj.size
1071 self.firstHeaderSize = self.basicHeaderObj.size
1052
1072
1053 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1073 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1054 if datatype == 0:
1074 if datatype == 0:
1055 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1075 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1056 elif datatype == 1:
1076 elif datatype == 1:
1057 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1077 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1058 elif datatype == 2:
1078 elif datatype == 2:
1059 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1079 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1060 elif datatype == 3:
1080 elif datatype == 3:
1061 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1081 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1062 elif datatype == 4:
1082 elif datatype == 4:
1063 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1083 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1064 elif datatype == 5:
1084 elif datatype == 5:
1065 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1085 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1066 else:
1086 else:
1067 raise ValueError, 'Data type was not defined'
1087 raise ValueError, 'Data type was not defined'
1068
1088
1069 self.dtype = datatype_str
1089 self.dtype = datatype_str
1070 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1090 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1071 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1091 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1072 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1092 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1073 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1093 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1074 self.getBlockDimension()
1094 self.getBlockDimension()
1075
1095
1076 def __verifyFile(self, filename, msgFlag=True):
1096 def __verifyFile(self, filename, msgFlag=True):
1077
1097
1078 msg = None
1098 msg = None
1079
1099
1080 try:
1100 try:
1081 fp = open(filename, 'rb')
1101 fp = open(filename, 'rb')
1082 except IOError:
1102 except IOError:
1083
1103
1084 if msgFlag:
1104 if msgFlag:
1085 print "[Reading] File %s can't be opened" % (filename)
1105 print "[Reading] File %s can't be opened" % (filename)
1086
1106
1087 return False
1107 return False
1088
1108
1089 currentPosition = fp.tell()
1109 currentPosition = fp.tell()
1090 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1110 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1091
1111
1092 if neededSize == 0:
1112 if neededSize == 0:
1093 basicHeaderObj = BasicHeader(LOCALTIME)
1113 basicHeaderObj = BasicHeader(LOCALTIME)
1094 systemHeaderObj = SystemHeader()
1114 systemHeaderObj = SystemHeader()
1095 radarControllerHeaderObj = RadarControllerHeader()
1115 radarControllerHeaderObj = RadarControllerHeader()
1096 processingHeaderObj = ProcessingHeader()
1116 processingHeaderObj = ProcessingHeader()
1097
1117
1098 if not( basicHeaderObj.read(fp) ):
1118 if not( basicHeaderObj.read(fp) ):
1099 fp.close()
1119 fp.close()
1100 return False
1120 return False
1101
1121
1102 if not( systemHeaderObj.read(fp) ):
1122 if not( systemHeaderObj.read(fp) ):
1103 fp.close()
1123 fp.close()
1104 return False
1124 return False
1105
1125
1106 if not( radarControllerHeaderObj.read(fp) ):
1126 if not( radarControllerHeaderObj.read(fp) ):
1107 fp.close()
1127 fp.close()
1108 return False
1128 return False
1109
1129
1110 if not( processingHeaderObj.read(fp) ):
1130 if not( processingHeaderObj.read(fp) ):
1111 fp.close()
1131 fp.close()
1112 return False
1132 return False
1113
1133
1114 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1134 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1115 else:
1135 else:
1116 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1136 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1117
1137
1118 fp.close()
1138 fp.close()
1119
1139
1120 fileSize = os.path.getsize(filename)
1140 fileSize = os.path.getsize(filename)
1121 currentSize = fileSize - currentPosition
1141 currentSize = fileSize - currentPosition
1122
1142
1123 if currentSize < neededSize:
1143 if currentSize < neededSize:
1124 if msgFlag and (msg != None):
1144 if msgFlag and (msg != None):
1125 print msg
1145 print msg
1126 return False
1146 return False
1127
1147
1128 return True
1148 return True
1129
1149
1130 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1150 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1131
1151
1132 path_empty = True
1152 path_empty = True
1133
1153
1134 dateList = []
1154 dateList = []
1135 pathList = []
1155 pathList = []
1136
1156
1137 multi_path = path.split(',')
1157 multi_path = path.split(',')
1138
1158
1139 if not walk:
1159 if not walk:
1140
1160
1141 for single_path in multi_path:
1161 for single_path in multi_path:
1142
1162
1143 if not os.path.isdir(single_path):
1163 if not os.path.isdir(single_path):
1144 continue
1164 continue
1145
1165
1146 fileList = glob.glob1(single_path, "*"+ext)
1166 fileList = glob.glob1(single_path, "*"+ext)
1147
1167
1148 if not fileList:
1168 if not fileList:
1149 continue
1169 continue
1150
1170
1151 path_empty = False
1171 path_empty = False
1152
1172
1153 fileList.sort()
1173 fileList.sort()
1154
1174
1155 for thisFile in fileList:
1175 for thisFile in fileList:
1156
1176
1157 if not os.path.isfile(os.path.join(single_path, thisFile)):
1177 if not os.path.isfile(os.path.join(single_path, thisFile)):
1158 continue
1178 continue
1159
1179
1160 if not isRadarFile(thisFile):
1180 if not isRadarFile(thisFile):
1161 continue
1181 continue
1162
1182
1163 if not isFileInDateRange(thisFile, startDate, endDate):
1183 if not isFileInDateRange(thisFile, startDate, endDate):
1164 continue
1184 continue
1165
1185
1166 thisDate = getDateFromRadarFile(thisFile)
1186 thisDate = getDateFromRadarFile(thisFile)
1167
1187
1168 if thisDate in dateList:
1188 if thisDate in dateList:
1169 continue
1189 continue
1170
1190
1171 dateList.append(thisDate)
1191 dateList.append(thisDate)
1172 pathList.append(single_path)
1192 pathList.append(single_path)
1173
1193
1174 else:
1194 else:
1175 for single_path in multi_path:
1195 for single_path in multi_path:
1176
1196
1177 if not os.path.isdir(single_path):
1197 if not os.path.isdir(single_path):
1178 continue
1198 continue
1179
1199
1180 dirList = []
1200 dirList = []
1181
1201
1182 for thisPath in os.listdir(single_path):
1202 for thisPath in os.listdir(single_path):
1183
1203
1184 if not os.path.isdir(os.path.join(single_path,thisPath)):
1204 if not os.path.isdir(os.path.join(single_path,thisPath)):
1185 continue
1205 continue
1186
1206
1187 if not isRadarFolder(thisPath):
1207 if not isRadarFolder(thisPath):
1188 continue
1208 continue
1189
1209
1190 if not isFolderInDateRange(thisPath, startDate, endDate):
1210 if not isFolderInDateRange(thisPath, startDate, endDate):
1191 continue
1211 continue
1192
1212
1193 dirList.append(thisPath)
1213 dirList.append(thisPath)
1194
1214
1195 if not dirList:
1215 if not dirList:
1196 continue
1216 continue
1197
1217
1198 dirList.sort()
1218 dirList.sort()
1199
1219
1200 for thisDir in dirList:
1220 for thisDir in dirList:
1201
1221
1202 datapath = os.path.join(single_path, thisDir, expLabel)
1222 datapath = os.path.join(single_path, thisDir, expLabel)
1203 fileList = glob.glob1(datapath, "*"+ext)
1223 fileList = glob.glob1(datapath, "*"+ext)
1204
1224
1205 if not fileList:
1225 if not fileList:
1206 continue
1226 continue
1207
1227
1208 path_empty = False
1228 path_empty = False
1209
1229
1210 thisDate = getDateFromRadarFolder(thisDir)
1230 thisDate = getDateFromRadarFolder(thisDir)
1211
1231
1212 pathList.append(datapath)
1232 pathList.append(datapath)
1213 dateList.append(thisDate)
1233 dateList.append(thisDate)
1214
1234
1215 dateList.sort()
1235 dateList.sort()
1216
1236
1217 if walk:
1237 if walk:
1218 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1238 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1219 else:
1239 else:
1220 pattern_path = multi_path[0]
1240 pattern_path = multi_path[0]
1221
1241
1222 if path_empty:
1242 if path_empty:
1223 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1243 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1224 else:
1244 else:
1225 if not dateList:
1245 if not dateList:
1226 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1246 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1227
1247
1228 if include_path:
1248 if include_path:
1229 return dateList, pathList
1249 return dateList, pathList
1230
1250
1231 return dateList
1251 return dateList
1232
1252
1233 def setup(self,
1253 def setup(self,
1234 path=None,
1254 path=None,
1235 startDate=None,
1255 startDate=None,
1236 endDate=None,
1256 endDate=None,
1237 startTime=datetime.time(0,0,0),
1257 startTime=datetime.time(0,0,0),
1238 endTime=datetime.time(23,59,59),
1258 endTime=datetime.time(23,59,59),
1239 set=None,
1259 set=None,
1240 expLabel = "",
1260 expLabel = "",
1241 ext = None,
1261 ext = None,
1242 online = False,
1262 online = False,
1243 delay = 60,
1263 delay = 60,
1244 walk = True,
1264 walk = True,
1245 getblock = False,
1265 getblock = False,
1246 nTxs = 1,
1266 nTxs = 1,
1247 realtime=False,
1267 realtime=False,
1248 blocksize=None,
1268 blocksize=None,
1249 blocktime=None):
1269 blocktime=None):
1250
1270
1251 if path == None:
1271 if path == None:
1252 raise ValueError, "[Reading] The path is not valid"
1272 raise ValueError, "[Reading] The path is not valid"
1273
1253
1274
1254 if ext == None:
1275 if ext == None:
1255 ext = self.ext
1276 ext = self.ext
1277
1278 self.path = path
1279 self.startDate = startDate
1280 self.endDate = endDate
1281 self.startTime = startTime
1282 self.endTime = endTime
1283 self.set = set
1284 self.expLabel = expLabel
1285 self.ext = ext
1286 self.online = online
1287 self.delay = delay
1288 self.walk = walk
1289 self.getblock = getblock
1290 self.nTxs = nTxs
1291 self.realtime = realtime
1292 self.blocksize = blocksize
1293 self.blocktime = blocktime
1294
1295
1296 if self.firstTime is True:
1297 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1298 startTime=startTime, endTime=endTime,
1299 set=set, expLabel=expLabel, ext=ext,
1300 walk=walk)
1301 filenameList = filenameList[:-1]
1302
1303 if pathList is not None and filenameList is not None and online:
1304 self.onlineWithDate = True
1305 online = False
1306 self.fileIndex = -1
1307 self.pathList = pathList
1308 self.filenameList = filenameList
1309 file_name = os.path.basename(filenameList[-1])
1310 basename, ext = os.path.splitext(file_name)
1311 last_set = int(basename[-3:])
1256
1312
1257 if online:
1313 if online:
1258 print "[Reading] Searching files in online mode..."
1314 print "[Reading] Searching files in online mode..."
1259
1315
1260 for nTries in range( self.nTries ):
1316 for nTries in range(self.nTries):
1261 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1262 expLabel=expLabel,
1318 expLabel=expLabel,
1263 ext=ext,
1319 ext=ext,
1264 walk=walk,
1320 walk=walk,
1265 startDate=startDate,
1321 startDate=startDate,
1266 endDate=endDate,
1322 startTime=startTime,
1267 startTime=startTime, endTime=endTime,
1268 set=set)
1323 set=set)
1269
1324
1270 if fullpath:
1325 if fullpath:
1271 break
1326 break
1272
1273 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1327 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1274 sleep( self.delay )
1328 sleep( self.delay )
1275
1329
1276 if not(fullpath):
1330 if not(fullpath):
1277 print "[Reading] There 'isn't any valid file in %s" % path
1331 print "[Reading] There 'isn't any valid file in %s" % path
1278 return
1332 return
1279
1333
1280 self.year = year
1334 self.year = year
1281 self.doy = doy
1335 self.doy = doy
1282 self.set = set - 1
1336 self.set = set - 1
1283 self.path = path
1337 self.path = path
1284 self.foldercounter = foldercounter
1338 self.foldercounter = foldercounter
1285 last_set = None
1339 last_set = None
1286
1340
1287 else:
1341 else:
1288 print "[Reading] Searching files in offline mode ..."
1342 print "[Reading] Searching files in offline mode ..."
1289 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1343 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1290 startTime=startTime, endTime=endTime,
1344 startTime=startTime, endTime=endTime,
1291 set=set, expLabel=expLabel, ext=ext,
1345 set=set, expLabel=expLabel, ext=ext,
1292 walk=walk)
1346 walk=walk)
1293
1347
1294 if not(pathList):
1348 if not(pathList):
1295 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1349 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1296 # datetime.datetime.combine(startDate,startTime).ctime(),
1350 # datetime.datetime.combine(startDate,startTime).ctime(),
1297 # datetime.datetime.combine(endDate,endTime).ctime())
1351 # datetime.datetime.combine(endDate,endTime).ctime())
1298
1352
1299 # sys.exit(-1)
1353 # sys.exit(-1)
1300
1354
1301 self.fileIndex = -1
1355 self.fileIndex = -1
1302 self.pathList = []
1356 self.pathList = []
1303 self.filenameList = []
1357 self.filenameList = []
1304 return
1358 return
1305
1359
1306 self.fileIndex = -1
1360 self.fileIndex = -1
1307 self.pathList = pathList
1361 self.pathList = pathList
1308 self.filenameList = filenameList
1362 self.filenameList = filenameList
1309 file_name = os.path.basename(filenameList[-1])
1363 file_name = os.path.basename(filenameList[-1])
1310 basename, ext = os.path.splitext(file_name)
1364 basename, ext = os.path.splitext(file_name)
1311 last_set = int(basename[-3:])
1365 last_set = int(basename[-3:])
1312
1366
1367
1313 self.online = online
1368 self.online = online
1314 self.realtime = realtime
1369 self.realtime = realtime
1315 self.delay = delay
1370 self.delay = delay
1316 ext = ext.lower()
1371 ext = ext.lower()
1317 self.ext = ext
1372 self.ext = ext
1318 self.getByBlock = getblock
1373 self.getByBlock = getblock
1319 self.nTxs = nTxs
1374 self.nTxs = nTxs
1320 self.startTime = startTime
1375 self.startTime = startTime
1321 self.endTime = endTime
1376 self.endTime = endTime
1377
1322
1378
1323 #Added-----------------
1379 #Added-----------------
1324 self.selBlocksize = blocksize
1380 self.selBlocksize = blocksize
1325 self.selBlocktime = blocktime
1381 self.selBlocktime = blocktime
1326
1382
1327
1383
1328 if not(self.setNextFile()):
1384 if not(self.setNextFile()):
1329 if (startDate!=None) and (endDate!=None):
1385 if (startDate!=None) and (endDate!=None):
1330 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1386 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1331 elif startDate != None:
1387 elif startDate != None:
1332 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1388 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1333 else:
1389 else:
1334 print "[Reading] No files"
1390 print "[Reading] No files"
1335
1391
1336 self.fileIndex = -1
1392 self.fileIndex = -1
1337 self.pathList = []
1393 self.pathList = []
1338 self.filenameList = []
1394 self.filenameList = []
1339 return
1395 return
1340
1396
1341 # self.getBasicHeader()
1397 # self.getBasicHeader()
1342
1398
1343 if last_set != None:
1399 if last_set != None:
1344 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1400 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1345 return
1401 return
1346
1402
1347 def getBasicHeader(self):
1403 def getBasicHeader(self):
1348
1404
1349 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1405 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1350
1406
1351 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1407 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1352
1408
1353 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1409 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1354
1410
1355 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1411 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1356
1412
1357 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1413 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1358
1414
1359 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1415 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1360
1416
1361 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1417 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1362
1418
1363 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1419 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1364
1420
1365
1421
1366 def getFirstHeader(self):
1422 def getFirstHeader(self):
1367
1423
1368 raise NotImplementedError
1424 raise NotImplementedError
1369
1425
1370 def getData(self):
1426 def getData(self):
1371
1427
1372 raise NotImplementedError
1428 raise NotImplementedError
1373
1429
1374 def hasNotDataInBuffer(self):
1430 def hasNotDataInBuffer(self):
1375
1431
1376 raise NotImplementedError
1432 raise NotImplementedError
1377
1433
1378 def readBlock(self):
1434 def readBlock(self):
1379
1435
1380 raise NotImplementedError
1436 raise NotImplementedError
1381
1437
1382 def isEndProcess(self):
1438 def isEndProcess(self):
1383
1439
1384 return self.flagNoMoreFiles
1440 return self.flagNoMoreFiles
1385
1441
1386 def printReadBlocks(self):
1442 def printReadBlocks(self):
1387
1443
1388 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1444 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1389
1445
1390 def printTotalBlocks(self):
1446 def printTotalBlocks(self):
1391
1447
1392 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1448 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1393
1449
1394 def printNumberOfBlock(self):
1450 def printNumberOfBlock(self):
1395
1451
1396 if self.flagIsNewBlock:
1452 if self.flagIsNewBlock:
1397 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1453 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1398 self.processingHeaderObj.dataBlocksPerFile,
1454 self.processingHeaderObj.dataBlocksPerFile,
1399 self.dataOut.datatime.ctime())
1455 self.dataOut.datatime.ctime())
1400
1456
1401 def printInfo(self):
1457 def printInfo(self):
1402
1458
1403 if self.__printInfo == False:
1459 if self.__printInfo == False:
1404 return
1460 return
1405
1461
1406 self.basicHeaderObj.printInfo()
1462 self.basicHeaderObj.printInfo()
1407 self.systemHeaderObj.printInfo()
1463 self.systemHeaderObj.printInfo()
1408 self.radarControllerHeaderObj.printInfo()
1464 self.radarControllerHeaderObj.printInfo()
1409 self.processingHeaderObj.printInfo()
1465 self.processingHeaderObj.printInfo()
1410
1466
1411 self.__printInfo = False
1467 self.__printInfo = False
1412
1468
1413
1469
1414 def run(self, **kwargs):
1470 def run(self, **kwargs):
1415
1471
1416 if not(self.isConfig):
1472 if not(self.isConfig):
1417
1473
1418 # self.dataOut = dataOut
1474 # self.dataOut = dataOut
1419 self.setup(**kwargs)
1475 self.setup(**kwargs)
1420 self.isConfig = True
1476 self.isConfig = True
1421
1477
1422 self.getData()
1478 self.getData()
1423
1479
1424 class JRODataWriter(JRODataIO):
1480 class JRODataWriter(JRODataIO):
1425
1481
1426 """
1482 """
1427 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1483 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1428 de los datos siempre se realiza por bloques.
1484 de los datos siempre se realiza por bloques.
1429 """
1485 """
1430
1486
1431 blockIndex = 0
1487 blockIndex = 0
1432
1488
1433 path = None
1489 path = None
1434
1490
1435 setFile = None
1491 setFile = None
1436
1492
1437 profilesPerBlock = None
1493 profilesPerBlock = None
1438
1494
1439 blocksPerFile = None
1495 blocksPerFile = None
1440
1496
1441 nWriteBlocks = 0
1497 nWriteBlocks = 0
1442
1498
1443 fileDate = None
1499 fileDate = None
1444
1500
1445 def __init__(self, dataOut=None):
1501 def __init__(self, dataOut=None):
1446 raise NotImplementedError
1502 raise NotImplementedError
1447
1503
1448
1504
1449 def hasAllDataInBuffer(self):
1505 def hasAllDataInBuffer(self):
1450 raise NotImplementedError
1506 raise NotImplementedError
1451
1507
1452
1508
1453 def setBlockDimension(self):
1509 def setBlockDimension(self):
1454 raise NotImplementedError
1510 raise NotImplementedError
1455
1511
1456
1512
1457 def writeBlock(self):
1513 def writeBlock(self):
1458 raise NotImplementedError
1514 raise NotImplementedError
1459
1515
1460
1516
1461 def putData(self):
1517 def putData(self):
1462 raise NotImplementedError
1518 raise NotImplementedError
1463
1519
1464
1520
1465 def getProcessFlags(self):
1521 def getProcessFlags(self):
1466
1522
1467 processFlags = 0
1523 processFlags = 0
1468
1524
1469 dtype_index = get_dtype_index(self.dtype)
1525 dtype_index = get_dtype_index(self.dtype)
1470 procflag_dtype = get_procflag_dtype(dtype_index)
1526 procflag_dtype = get_procflag_dtype(dtype_index)
1471
1527
1472 processFlags += procflag_dtype
1528 processFlags += procflag_dtype
1473
1529
1474 if self.dataOut.flagDecodeData:
1530 if self.dataOut.flagDecodeData:
1475 processFlags += PROCFLAG.DECODE_DATA
1531 processFlags += PROCFLAG.DECODE_DATA
1476
1532
1477 if self.dataOut.flagDeflipData:
1533 if self.dataOut.flagDeflipData:
1478 processFlags += PROCFLAG.DEFLIP_DATA
1534 processFlags += PROCFLAG.DEFLIP_DATA
1479
1535
1480 if self.dataOut.code is not None:
1536 if self.dataOut.code is not None:
1481 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1537 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1482
1538
1483 if self.dataOut.nCohInt > 1:
1539 if self.dataOut.nCohInt > 1:
1484 processFlags += PROCFLAG.COHERENT_INTEGRATION
1540 processFlags += PROCFLAG.COHERENT_INTEGRATION
1485
1541
1486 if self.dataOut.type == "Spectra":
1542 if self.dataOut.type == "Spectra":
1487 if self.dataOut.nIncohInt > 1:
1543 if self.dataOut.nIncohInt > 1:
1488 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1544 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1489
1545
1490 if self.dataOut.data_dc is not None:
1546 if self.dataOut.data_dc is not None:
1491 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1547 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1492
1548
1493 if self.dataOut.flagShiftFFT:
1549 if self.dataOut.flagShiftFFT:
1494 processFlags += PROCFLAG.SHIFT_FFT_DATA
1550 processFlags += PROCFLAG.SHIFT_FFT_DATA
1495
1551
1496 return processFlags
1552 return processFlags
1497
1553
1498 def setBasicHeader(self):
1554 def setBasicHeader(self):
1499
1555
1500 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1556 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1501 self.basicHeaderObj.version = self.versionFile
1557 self.basicHeaderObj.version = self.versionFile
1502 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1558 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1503
1559
1504 utc = numpy.floor(self.dataOut.utctime)
1560 utc = numpy.floor(self.dataOut.utctime)
1505 milisecond = (self.dataOut.utctime - utc)* 1000.0
1561 milisecond = (self.dataOut.utctime - utc)* 1000.0
1506
1562
1507 self.basicHeaderObj.utc = utc
1563 self.basicHeaderObj.utc = utc
1508 self.basicHeaderObj.miliSecond = milisecond
1564 self.basicHeaderObj.miliSecond = milisecond
1509 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1565 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1510 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1566 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1511 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1567 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1512
1568
1513 def setFirstHeader(self):
1569 def setFirstHeader(self):
1514 """
1570 """
1515 Obtiene una copia del First Header
1571 Obtiene una copia del First Header
1516
1572
1517 Affected:
1573 Affected:
1518
1574
1519 self.basicHeaderObj
1575 self.basicHeaderObj
1520 self.systemHeaderObj
1576 self.systemHeaderObj
1521 self.radarControllerHeaderObj
1577 self.radarControllerHeaderObj
1522 self.processingHeaderObj self.
1578 self.processingHeaderObj self.
1523
1579
1524 Return:
1580 Return:
1525 None
1581 None
1526 """
1582 """
1527
1583
1528 raise NotImplementedError
1584 raise NotImplementedError
1529
1585
1530 def __writeFirstHeader(self):
1586 def __writeFirstHeader(self):
1531 """
1587 """
1532 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1588 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1533
1589
1534 Affected:
1590 Affected:
1535 __dataType
1591 __dataType
1536
1592
1537 Return:
1593 Return:
1538 None
1594 None
1539 """
1595 """
1540
1596
1541 # CALCULAR PARAMETROS
1597 # CALCULAR PARAMETROS
1542
1598
1543 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1599 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1544 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1600 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1545
1601
1546 self.basicHeaderObj.write(self.fp)
1602 self.basicHeaderObj.write(self.fp)
1547 self.systemHeaderObj.write(self.fp)
1603 self.systemHeaderObj.write(self.fp)
1548 self.radarControllerHeaderObj.write(self.fp)
1604 self.radarControllerHeaderObj.write(self.fp)
1549 self.processingHeaderObj.write(self.fp)
1605 self.processingHeaderObj.write(self.fp)
1550
1606
1551 def __setNewBlock(self):
1607 def __setNewBlock(self):
1552 """
1608 """
1553 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1609 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1554
1610
1555 Return:
1611 Return:
1556 0 : si no pudo escribir nada
1612 0 : si no pudo escribir nada
1557 1 : Si escribio el Basic el First Header
1613 1 : Si escribio el Basic el First Header
1558 """
1614 """
1559 if self.fp == None:
1615 if self.fp == None:
1560 self.setNextFile()
1616 self.setNextFile()
1561
1617
1562 if self.flagIsNewFile:
1618 if self.flagIsNewFile:
1563 return 1
1619 return 1
1564
1620
1565 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1621 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1566 self.basicHeaderObj.write(self.fp)
1622 self.basicHeaderObj.write(self.fp)
1567 return 1
1623 return 1
1568
1624
1569 if not( self.setNextFile() ):
1625 if not( self.setNextFile() ):
1570 return 0
1626 return 0
1571
1627
1572 return 1
1628 return 1
1573
1629
1574
1630
1575 def writeNextBlock(self):
1631 def writeNextBlock(self):
1576 """
1632 """
1577 Selecciona el bloque siguiente de datos y los escribe en un file
1633 Selecciona el bloque siguiente de datos y los escribe en un file
1578
1634
1579 Return:
1635 Return:
1580 0 : Si no hizo pudo escribir el bloque de datos
1636 0 : Si no hizo pudo escribir el bloque de datos
1581 1 : Si no pudo escribir el bloque de datos
1637 1 : Si no pudo escribir el bloque de datos
1582 """
1638 """
1583 if not( self.__setNewBlock() ):
1639 if not( self.__setNewBlock() ):
1584 return 0
1640 return 0
1585
1641
1586 self.writeBlock()
1642 self.writeBlock()
1587
1643
1588 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1644 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1589 self.processingHeaderObj.dataBlocksPerFile)
1645 self.processingHeaderObj.dataBlocksPerFile)
1590
1646
1591 return 1
1647 return 1
1592
1648
1593 def setNextFile(self):
1649 def setNextFile(self):
1594 """
1650 """
1595 Determina el siguiente file que sera escrito
1651 Determina el siguiente file que sera escrito
1596
1652
1597 Affected:
1653 Affected:
1598 self.filename
1654 self.filename
1599 self.subfolder
1655 self.subfolder
1600 self.fp
1656 self.fp
1601 self.setFile
1657 self.setFile
1602 self.flagIsNewFile
1658 self.flagIsNewFile
1603
1659
1604 Return:
1660 Return:
1605 0 : Si el archivo no puede ser escrito
1661 0 : Si el archivo no puede ser escrito
1606 1 : Si el archivo esta listo para ser escrito
1662 1 : Si el archivo esta listo para ser escrito
1607 """
1663 """
1608 ext = self.ext
1664 ext = self.ext
1609 path = self.path
1665 path = self.path
1610
1666
1611 if self.fp != None:
1667 if self.fp != None:
1612 self.fp.close()
1668 self.fp.close()
1613
1669
1614 timeTuple = time.localtime( self.dataOut.utctime)
1670 timeTuple = time.localtime( self.dataOut.utctime)
1615 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1671 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1616
1672
1617 fullpath = os.path.join( path, subfolder )
1673 fullpath = os.path.join( path, subfolder )
1618 setFile = self.setFile
1674 setFile = self.setFile
1619
1675
1620 if not( os.path.exists(fullpath) ):
1676 if not( os.path.exists(fullpath) ):
1621 os.mkdir(fullpath)
1677 os.mkdir(fullpath)
1622 setFile = -1 #inicializo mi contador de seteo
1678 setFile = -1 #inicializo mi contador de seteo
1623 else:
1679 else:
1624 filesList = os.listdir( fullpath )
1680 filesList = os.listdir( fullpath )
1625 if len( filesList ) > 0:
1681 if len( filesList ) > 0:
1626 filesList = sorted( filesList, key=str.lower )
1682 filesList = sorted( filesList, key=str.lower )
1627 filen = filesList[-1]
1683 filen = filesList[-1]
1628 # el filename debera tener el siguiente formato
1684 # el filename debera tener el siguiente formato
1629 # 0 1234 567 89A BCDE (hex)
1685 # 0 1234 567 89A BCDE (hex)
1630 # x YYYY DDD SSS .ext
1686 # x YYYY DDD SSS .ext
1631 if isNumber( filen[8:11] ):
1687 if isNumber( filen[8:11] ):
1632 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1688 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1633 else:
1689 else:
1634 setFile = -1
1690 setFile = -1
1635 else:
1691 else:
1636 setFile = -1 #inicializo mi contador de seteo
1692 setFile = -1 #inicializo mi contador de seteo
1637
1693
1638 setFile += 1
1694 setFile += 1
1639
1695
1640 #If this is a new day it resets some values
1696 #If this is a new day it resets some values
1641 if self.dataOut.datatime.date() > self.fileDate:
1697 if self.dataOut.datatime.date() > self.fileDate:
1642 setFile = 0
1698 setFile = 0
1643 self.nTotalBlocks = 0
1699 self.nTotalBlocks = 0
1644
1700
1645 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1701 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1646
1702
1647 filename = os.path.join( path, subfolder, filen )
1703 filename = os.path.join( path, subfolder, filen )
1648
1704
1649 fp = open( filename,'wb' )
1705 fp = open( filename,'wb' )
1650
1706
1651 self.blockIndex = 0
1707 self.blockIndex = 0
1652
1708
1653 #guardando atributos
1709 #guardando atributos
1654 self.filename = filename
1710 self.filename = filename
1655 self.subfolder = subfolder
1711 self.subfolder = subfolder
1656 self.fp = fp
1712 self.fp = fp
1657 self.setFile = setFile
1713 self.setFile = setFile
1658 self.flagIsNewFile = 1
1714 self.flagIsNewFile = 1
1659 self.fileDate = self.dataOut.datatime.date()
1715 self.fileDate = self.dataOut.datatime.date()
1660
1716
1661 self.setFirstHeader()
1717 self.setFirstHeader()
1662
1718
1663 print '[Writing] Opening file: %s'%self.filename
1719 print '[Writing] Opening file: %s'%self.filename
1664
1720
1665 self.__writeFirstHeader()
1721 self.__writeFirstHeader()
1666
1722
1667 return 1
1723 return 1
1668
1724
1669 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1725 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1670 """
1726 """
1671 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1727 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1672
1728
1673 Inputs:
1729 Inputs:
1674 path : directory where data will be saved
1730 path : directory where data will be saved
1675 profilesPerBlock : number of profiles per block
1731 profilesPerBlock : number of profiles per block
1676 set : initial file set
1732 set : initial file set
1677 datatype : An integer number that defines data type:
1733 datatype : An integer number that defines data type:
1678 0 : int8 (1 byte)
1734 0 : int8 (1 byte)
1679 1 : int16 (2 bytes)
1735 1 : int16 (2 bytes)
1680 2 : int32 (4 bytes)
1736 2 : int32 (4 bytes)
1681 3 : int64 (8 bytes)
1737 3 : int64 (8 bytes)
1682 4 : float32 (4 bytes)
1738 4 : float32 (4 bytes)
1683 5 : double64 (8 bytes)
1739 5 : double64 (8 bytes)
1684
1740
1685 Return:
1741 Return:
1686 0 : Si no realizo un buen seteo
1742 0 : Si no realizo un buen seteo
1687 1 : Si realizo un buen seteo
1743 1 : Si realizo un buen seteo
1688 """
1744 """
1689
1745
1690 if ext == None:
1746 if ext == None:
1691 ext = self.ext
1747 ext = self.ext
1692
1748
1693 self.ext = ext.lower()
1749 self.ext = ext.lower()
1694
1750
1695 self.path = path
1751 self.path = path
1696
1752
1697 if set is None:
1753 if set is None:
1698 self.setFile = -1
1754 self.setFile = -1
1699 else:
1755 else:
1700 self.setFile = set - 1
1756 self.setFile = set - 1
1701
1757
1702 self.blocksPerFile = blocksPerFile
1758 self.blocksPerFile = blocksPerFile
1703
1759
1704 self.profilesPerBlock = profilesPerBlock
1760 self.profilesPerBlock = profilesPerBlock
1705
1761
1706 self.dataOut = dataOut
1762 self.dataOut = dataOut
1707 self.fileDate = self.dataOut.datatime.date()
1763 self.fileDate = self.dataOut.datatime.date()
1708 #By default
1764 #By default
1709 self.dtype = self.dataOut.dtype
1765 self.dtype = self.dataOut.dtype
1710
1766
1711 if datatype is not None:
1767 if datatype is not None:
1712 self.dtype = get_numpy_dtype(datatype)
1768 self.dtype = get_numpy_dtype(datatype)
1713
1769
1714 if not(self.setNextFile()):
1770 if not(self.setNextFile()):
1715 print "[Writing] There isn't a next file"
1771 print "[Writing] There isn't a next file"
1716 return 0
1772 return 0
1717
1773
1718 self.setBlockDimension()
1774 self.setBlockDimension()
1719
1775
1720 return 1
1776 return 1
1721
1777
1722 def run(self, dataOut, **kwargs):
1778 def run(self, dataOut, **kwargs):
1723
1779
1724 if not(self.isConfig):
1780 if not(self.isConfig):
1725
1781
1726 self.setup(dataOut, **kwargs)
1782 self.setup(dataOut, **kwargs)
1727 self.isConfig = True
1783 self.isConfig = True
1728
1784
1729 self.putData()
1785 self.putData()
1730
1786
Show file before | Show file after | Hide comments
@@ -1,637 +1,637
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import numpy
7 import numpy
8
8
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13 # from _sha import blocksize
13 # from _sha import blocksize
14
14
15 class VoltageReader(JRODataReader, ProcessingUnit):
15 class VoltageReader(JRODataReader, ProcessingUnit):
16 """
16 """
17 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
17 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
19 perfiles*alturas*canales) son almacenados en la variable "buffer".
19 perfiles*alturas*canales) son almacenados en la variable "buffer".
20
20
21 perfiles * alturas * canales
21 perfiles * alturas * canales
22
22
23 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
23 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
24 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
24 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
25 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
25 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
26 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
26 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
27
27
28 Example:
28 Example:
29
29
30 dpath = "/home/myuser/data"
30 dpath = "/home/myuser/data"
31
31
32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33
33
34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35
35
36 readerObj = VoltageReader()
36 readerObj = VoltageReader()
37
37
38 readerObj.setup(dpath, startTime, endTime)
38 readerObj.setup(dpath, startTime, endTime)
39
39
40 while(True):
40 while(True):
41
41
42 #to get one profile
42 #to get one profile
43 profile = readerObj.getData()
43 profile = readerObj.getData()
44
44
45 #print the profile
45 #print the profile
46 print profile
46 print profile
47
47
48 #If you want to see all datablock
48 #If you want to see all datablock
49 print readerObj.datablock
49 print readerObj.datablock
50
50
51 if readerObj.flagNoMoreFiles:
51 if readerObj.flagNoMoreFiles:
52 break
52 break
53
53
54 """
54 """
55
55
56 ext = ".r"
56 ext = ".r"
57
57
58 optchar = "D"
58 optchar = "D"
59 dataOut = None
59 dataOut = None
60
60
61 def __init__(self):
61 def __init__(self):
62 """
62 """
63 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
63 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
64
64
65 Input:
65 Input:
66 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
66 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
67 almacenar un perfil de datos cada vez que se haga un requerimiento
67 almacenar un perfil de datos cada vez que se haga un requerimiento
68 (getData). El perfil sera obtenido a partir del buffer de datos,
68 (getData). El perfil sera obtenido a partir del buffer de datos,
69 si el buffer esta vacio se hara un nuevo proceso de lectura de un
69 si el buffer esta vacio se hara un nuevo proceso de lectura de un
70 bloque de datos.
70 bloque de datos.
71 Si este parametro no es pasado se creara uno internamente.
71 Si este parametro no es pasado se creara uno internamente.
72
72
73 Variables afectadas:
73 Variables afectadas:
74 self.dataOut
74 self.dataOut
75
75
76 Return:
76 Return:
77 None
77 None
78 """
78 """
79
79
80 ProcessingUnit.__init__(self)
80 ProcessingUnit.__init__(self)
81
81
82 self.isConfig = False
82 self.isConfig = False
83
83
84 self.datablock = None
84 self.datablock = None
85
85
86 self.utc = 0
86 self.utc = 0
87
87
88 self.ext = ".r"
88 self.ext = ".r"
89
89
90 self.optchar = "D"
90 self.optchar = "D"
91
91
92 self.basicHeaderObj = BasicHeader(LOCALTIME)
92 self.basicHeaderObj = BasicHeader(LOCALTIME)
93
93
94 self.systemHeaderObj = SystemHeader()
94 self.systemHeaderObj = SystemHeader()
95
95
96 self.radarControllerHeaderObj = RadarControllerHeader()
96 self.radarControllerHeaderObj = RadarControllerHeader()
97
97
98 self.processingHeaderObj = ProcessingHeader()
98 self.processingHeaderObj = ProcessingHeader()
99
99
100 self.online = 0
100 self.online = 0
101
101
102 self.fp = None
102 self.fp = None
103
103
104 self.idFile = None
104 self.idFile = None
105
105
106 self.dtype = None
106 self.dtype = None
107
107
108 self.fileSizeByHeader = None
108 self.fileSizeByHeader = None
109
109
110 self.filenameList = []
110 self.filenameList = []
111
111
112 self.filename = None
112 self.filename = None
113
113
114 self.fileSize = None
114 self.fileSize = None
115
115
116 self.firstHeaderSize = 0
116 self.firstHeaderSize = 0
117
117
118 self.basicHeaderSize = 24
118 self.basicHeaderSize = 24
119
119
120 self.pathList = []
120 self.pathList = []
121
121
122 self.filenameList = []
122 self.filenameList = []
123
123
124 self.lastUTTime = 0
124 self.lastUTTime = 0
125
125
126 self.maxTimeStep = 30
126 self.maxTimeStep = 30
127
127
128 self.flagNoMoreFiles = 0
128 self.flagNoMoreFiles = 0
129
129
130 self.set = 0
130 self.set = 0
131
131
132 self.path = None
132 self.path = None
133
133
134 self.profileIndex = 2**32-1
134 self.profileIndex = 2**32-1
135
135
136 self.delay = 3 #seconds
136 self.delay = 3 #seconds
137
137
138 self.nTries = 3 #quantity tries
138 self.nTries = 3 #quantity tries
139
139
140 self.nFiles = 3 #number of files for searching
140 self.nFiles = 3 #number of files for searching
141
141
142 self.nReadBlocks = 0
142 self.nReadBlocks = 0
143
143
144 self.flagIsNewFile = 1
144 self.flagIsNewFile = 1
145
145
146 self.__isFirstTimeOnline = 1
146 self.__isFirstTimeOnline = 1
147
147
148 # self.ippSeconds = 0
148 # self.ippSeconds = 0
149
149
150 self.flagDiscontinuousBlock = 0
150 self.flagDiscontinuousBlock = 0
151
151
152 self.flagIsNewBlock = 0
152 self.flagIsNewBlock = 0
153
153
154 self.nTotalBlocks = 0
154 self.nTotalBlocks = 0
155
155
156 self.blocksize = 0
156 self.blocksize = 0
157
157
158 self.dataOut = self.createObjByDefault()
158 self.dataOut = self.createObjByDefault()
159
159
160 self.nTxs = 1
160 self.nTxs = 1
161
161
162 self.txIndex = 0
162 self.txIndex = 0
163
163
164 def createObjByDefault(self):
164 def createObjByDefault(self):
165
165
166 dataObj = Voltage()
166 dataObj = Voltage()
167
167
168 return dataObj
168 return dataObj
169
169
170 def __hasNotDataInBuffer(self):
170 def __hasNotDataInBuffer(self):
171
171
172 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
172 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
173 return 1
173 return 1
174
174
175 return 0
175 return 0
176
176
177
177
178 def getBlockDimension(self):
178 def getBlockDimension(self):
179 """
179 """
180 Obtiene la cantidad de puntos a leer por cada bloque de datos
180 Obtiene la cantidad de puntos a leer por cada bloque de datos
181
181
182 Affected:
182 Affected:
183 self.blocksize
183 self.blocksize
184
184
185 Return:
185 Return:
186 None
186 None
187 """
187 """
188 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
188 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
189 self.blocksize = pts2read
189 self.blocksize = pts2read
190
190
191
191
192 def readBlock(self):
192 def readBlock(self):
193 """
193 """
194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
197 es seteado a 0
197 es seteado a 0
198
199 Inputs:
200 None
201
198
202 Return:
199 Inputs:
203 None
200 None
204
201
205 Affected:
202 Return:
206 self.profileIndex
203 None
207 self.datablock
208 self.flagIsNewFile
209 self.flagIsNewBlock
210 self.nTotalBlocks
211
204
212 Exceptions:
205 Affected:
213 Si un bloque leido no es un bloque valido
206 self.profileIndex
207 self.datablock
208 self.flagIsNewFile
209 self.flagIsNewBlock
210 self.nTotalBlocks
211
212 Exceptions:
213 Si un bloque leido no es un bloque valido
214 """
214 """
215 current_pointer_location = self.fp.tell()
215 current_pointer_location = self.fp.tell()
216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217
217
218 try:
218 try:
219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
220 except:
220 except:
221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222
222
223 if self.waitDataBlock(pointer_location=current_pointer_location):
223 if self.waitDataBlock(pointer_location=current_pointer_location):
224 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
224 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
225 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
225 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
226 # return 0
226 # return 0
227
227
228 #Dimensions : nChannels, nProfiles, nSamples
228 #Dimensions : nChannels, nProfiles, nSamples
229
229
230 junk = numpy.transpose(junk, (2,0,1))
230 junk = numpy.transpose(junk, (2,0,1))
231 self.datablock = junk['real'] + junk['imag']*1j
231 self.datablock = junk['real'] + junk['imag']*1j
232
232
233 self.profileIndex = 0
233 self.profileIndex = 0
234
234
235 self.flagIsNewFile = 0
235 self.flagIsNewFile = 0
236 self.flagIsNewBlock = 1
236 self.flagIsNewBlock = 1
237
237
238 self.nTotalBlocks += 1
238 self.nTotalBlocks += 1
239 self.nReadBlocks += 1
239 self.nReadBlocks += 1
240
240
241 return 1
241 return 1
242
242
243 def getFirstHeader(self):
243 def getFirstHeader(self):
244
244
245 self.getBasicHeader()
245 self.getBasicHeader()
246
246
247 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
247 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
248
248
249 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
249 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
250
250
251 if self.nTxs > 1:
251 if self.nTxs > 1:
252 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
252 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
253
253
254 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
254 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
255
255
256 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
256 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
257 #
257 #
258 # if self.radarControllerHeaderObj.code is not None:
258 # if self.radarControllerHeaderObj.code is not None:
259 #
259 #
260 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
260 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
261 #
261 #
262 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
262 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
263 #
263 #
264 # self.dataOut.code = self.radarControllerHeaderObj.code
264 # self.dataOut.code = self.radarControllerHeaderObj.code
265
265
266 self.dataOut.dtype = self.dtype
266 self.dataOut.dtype = self.dtype
267
267
268 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
268 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
269
269
270 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
270 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
271
271
272 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
272 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
273
273
274 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
274 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
275
275
276 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
276 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
277
277
278 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
278 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
279
279
280 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
280 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
281
281
282 def reshapeData(self):
282 def reshapeData(self):
283
283
284 if self.nTxs < 0:
284 if self.nTxs < 0:
285 return
285 return
286
286
287 if self.nTxs == 1:
287 if self.nTxs == 1:
288 return
288 return
289
289
290 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
290 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
291 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
291 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
292
292
293 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
293 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
294 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
294 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
295
295
296 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
296 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
297
297
298 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
298 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
299 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
299 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
300 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
300 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
301
301
302 return
302 return
303
303
304 def getData(self):
304 def getData(self):
305 """
305 """
306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
309 "readNextBlock"
309 "readNextBlock"
310
311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
312
313 Return:
314
315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
316 es igual al total de perfiles leidos desde el archivo.
317
318 Si self.getByBlock == False:
319
310
320 self.dataOut.data = buffer[:, thisProfile, :]
311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
321
322 shape = [nChannels, nHeis]
323
324 Si self.getByBlock == True:
325
312
326 self.dataOut.data = buffer[:, :, :]
313 Return:
327
328 shape = [nChannels, nProfiles, nHeis]
329
314
330 Variables afectadas:
315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
331 self.dataOut
316 es igual al total de perfiles leidos desde el archivo.
332 self.profileIndex
333
317
334 Affected:
318 Si self.getByBlock == False:
335 self.dataOut
319
336 self.profileIndex
320 self.dataOut.data = buffer[:, thisProfile, :]
337 self.flagDiscontinuousBlock
321
338 self.flagIsNewBlock
322 shape = [nChannels, nHeis]
323
324 Si self.getByBlock == True:
325
326 self.dataOut.data = buffer[:, :, :]
327
328 shape = [nChannels, nProfiles, nHeis]
329
330 Variables afectadas:
331 self.dataOut
332 self.profileIndex
333
334 Affected:
335 self.dataOut
336 self.profileIndex
337 self.flagDiscontinuousBlock
338 self.flagIsNewBlock
339 """
339 """
340
340
341 if self.flagNoMoreFiles:
341 if self.flagNoMoreFiles:
342 self.dataOut.flagNoData = True
342 self.dataOut.flagNoData = True
343 print 'Process finished'
343 print 'Process finished'
344 return 0
344 return 0
345
345
346 self.flagDiscontinuousBlock = 0
346 self.flagDiscontinuousBlock = 0
347 self.flagIsNewBlock = 0
347 self.flagIsNewBlock = 0
348
348
349 if self.__hasNotDataInBuffer():
349 if self.__hasNotDataInBuffer():
350
350
351 if not( self.readNextBlock() ):
351 if not( self.readNextBlock() ):
352 return 0
352 return 0
353
353
354 self.getFirstHeader()
354 self.getFirstHeader()
355
355
356 self.reshapeData()
356 self.reshapeData()
357
357
358 if self.datablock is None:
358 if self.datablock is None:
359 self.dataOut.flagNoData = True
359 self.dataOut.flagNoData = True
360 return 0
360 return 0
361
361
362 if not self.getByBlock:
362 if not self.getByBlock:
363
363
364 """
364 """
365 Return profile by profile
365 Return profile by profile
366
366
367 If nTxs > 1 then one profile is divided by nTxs and number of total
367 If nTxs > 1 then one profile is divided by nTxs and number of total
368 blocks is increased by nTxs (nProfiles *= nTxs)
368 blocks is increased by nTxs (nProfiles *= nTxs)
369 """
369 """
370 self.dataOut.flagDataAsBlock = False
370 self.dataOut.flagDataAsBlock = False
371 self.dataOut.data = self.datablock[:,self.profileIndex,:]
371 self.dataOut.data = self.datablock[:,self.profileIndex,:]
372 self.dataOut.profileIndex = self.profileIndex
372 self.dataOut.profileIndex = self.profileIndex
373
373
374 self.profileIndex += 1
374 self.profileIndex += 1
375
375
376 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
376 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
377 # """
377 # """
378 # Return all block
378 # Return all block
379 # """
379 # """
380 # self.dataOut.flagDataAsBlock = True
380 # self.dataOut.flagDataAsBlock = True
381 # self.dataOut.data = self.datablock
381 # self.dataOut.data = self.datablock
382 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
382 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
383 #
383 #
384 # self.profileIndex = self.dataOut.nProfiles
384 # self.profileIndex = self.dataOut.nProfiles
385
385
386 else:
386 else:
387 """
387 """
388 Return a block
388 Return a block
389 """
389 """
390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 if self.selBlocktime != None:
391 if self.selBlocktime != None:
392 if self.dataOut.nCohInt is not None:
392 if self.dataOut.nCohInt is not None:
393 nCohInt = self.dataOut.nCohInt
393 nCohInt = self.dataOut.nCohInt
394 else:
394 else:
395 nCohInt = 1
395 nCohInt = 1
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397
397
398 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
398 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
399 self.profileIndex += self.selBlocksize
399 self.profileIndex += self.selBlocksize
400 datasize = self.dataOut.data.shape[1]
400 datasize = self.dataOut.data.shape[1]
401
401
402 if datasize < self.selBlocksize:
402 if datasize < self.selBlocksize:
403 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
403 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
404 buffer[:,:datasize,:] = self.dataOut.data
404 buffer[:,:datasize,:] = self.dataOut.data
405
405
406 while datasize < self.selBlocksize: #Not enough profiles to fill the block
406 while datasize < self.selBlocksize: #Not enough profiles to fill the block
407 if not( self.readNextBlock() ):
407 if not( self.readNextBlock() ):
408 return 0
408 return 0
409 self.getFirstHeader()
409 self.getFirstHeader()
410 self.reshapeData()
410 self.reshapeData()
411 if self.datablock is None:
411 if self.datablock is None:
412 self.dataOut.flagNoData = True
412 self.dataOut.flagNoData = True
413 return 0
413 return 0
414 #stack data
414 #stack data
415 blockIndex = self.selBlocksize - datasize
415 blockIndex = self.selBlocksize - datasize
416 datablock1 = self.datablock[:,:blockIndex,:]
416 datablock1 = self.datablock[:,:blockIndex,:]
417
417
418 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
418 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
419 datasize += datablock1.shape[1]
419 datasize += datablock1.shape[1]
420
420
421 self.dataOut.data = buffer
421 self.dataOut.data = buffer
422 self.profileIndex = blockIndex
422 self.profileIndex = blockIndex
423
423
424 self.dataOut.flagDataAsBlock = True
424 self.dataOut.flagDataAsBlock = True
425 self.dataOut.nProfiles = self.dataOut.data.shape[1]
425 self.dataOut.nProfiles = self.dataOut.data.shape[1]
426
426
427 self.dataOut.flagNoData = False
427 self.dataOut.flagNoData = False
428
428
429 self.getBasicHeader()
429 self.getBasicHeader()
430
430
431 self.dataOut.realtime = self.online
431 self.dataOut.realtime = self.online
432
432
433 return self.dataOut.data
433 return self.dataOut.data
434
434
435 class VoltageWriter(JRODataWriter, Operation):
435 class VoltageWriter(JRODataWriter, Operation):
436 """
436 """
437 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
437 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
438 de los datos siempre se realiza por bloques.
438 de los datos siempre se realiza por bloques.
439 """
439 """
440
440
441 ext = ".r"
441 ext = ".r"
442
442
443 optchar = "D"
443 optchar = "D"
444
444
445 shapeBuffer = None
445 shapeBuffer = None
446
446
447
447
448 def __init__(self):
448 def __init__(self):
449 """
449 """
450 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
450 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
451
451
452 Affected:
452 Affected:
453 self.dataOut
453 self.dataOut
454
454
455 Return: None
455 Return: None
456 """
456 """
457 Operation.__init__(self)
457 Operation.__init__(self)
458
458
459 self.nTotalBlocks = 0
459 self.nTotalBlocks = 0
460
460
461 self.profileIndex = 0
461 self.profileIndex = 0
462
462
463 self.isConfig = False
463 self.isConfig = False
464
464
465 self.fp = None
465 self.fp = None
466
466
467 self.flagIsNewFile = 1
467 self.flagIsNewFile = 1
468
468
469 self.blockIndex = 0
469 self.blockIndex = 0
470
470
471 self.flagIsNewBlock = 0
471 self.flagIsNewBlock = 0
472
472
473 self.setFile = None
473 self.setFile = None
474
474
475 self.dtype = None
475 self.dtype = None
476
476
477 self.path = None
477 self.path = None
478
478
479 self.filename = None
479 self.filename = None
480
480
481 self.basicHeaderObj = BasicHeader(LOCALTIME)
481 self.basicHeaderObj = BasicHeader(LOCALTIME)
482
482
483 self.systemHeaderObj = SystemHeader()
483 self.systemHeaderObj = SystemHeader()
484
484
485 self.radarControllerHeaderObj = RadarControllerHeader()
485 self.radarControllerHeaderObj = RadarControllerHeader()
486
486
487 self.processingHeaderObj = ProcessingHeader()
487 self.processingHeaderObj = ProcessingHeader()
488
488
489 def hasAllDataInBuffer(self):
489 def hasAllDataInBuffer(self):
490 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
490 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
491 return 1
491 return 1
492 return 0
492 return 0
493
493
494
494
495 def setBlockDimension(self):
495 def setBlockDimension(self):
496 """
496 """
497 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
497 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
498
498
499 Affected:
499 Affected:
500 self.shape_spc_Buffer
500 self.shape_spc_Buffer
501 self.shape_cspc_Buffer
501 self.shape_cspc_Buffer
502 self.shape_dc_Buffer
502 self.shape_dc_Buffer
503
503
504 Return: None
504 Return: None
505 """
505 """
506 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
506 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
507 self.processingHeaderObj.nHeights,
507 self.processingHeaderObj.nHeights,
508 self.systemHeaderObj.nChannels)
508 self.systemHeaderObj.nChannels)
509
509
510 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
510 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
511 self.processingHeaderObj.profilesPerBlock,
511 self.processingHeaderObj.profilesPerBlock,
512 self.processingHeaderObj.nHeights),
512 self.processingHeaderObj.nHeights),
513 dtype=numpy.dtype('complex64'))
513 dtype=numpy.dtype('complex64'))
514
514
515 def writeBlock(self):
515 def writeBlock(self):
516 """
516 """
517 Escribe el buffer en el file designado
517 Escribe el buffer en el file designado
518
518
519 Affected:
519 Affected:
520 self.profileIndex
520 self.profileIndex
521 self.flagIsNewFile
521 self.flagIsNewFile
522 self.flagIsNewBlock
522 self.flagIsNewBlock
523 self.nTotalBlocks
523 self.nTotalBlocks
524 self.blockIndex
524 self.blockIndex
525
525
526 Return: None
526 Return: None
527 """
527 """
528 data = numpy.zeros( self.shapeBuffer, self.dtype )
528 data = numpy.zeros( self.shapeBuffer, self.dtype )
529
529
530 junk = numpy.transpose(self.datablock, (1,2,0))
530 junk = numpy.transpose(self.datablock, (1,2,0))
531
531
532 data['real'] = junk.real
532 data['real'] = junk.real
533 data['imag'] = junk.imag
533 data['imag'] = junk.imag
534
534
535 data = data.reshape( (-1) )
535 data = data.reshape( (-1) )
536
536
537 data.tofile( self.fp )
537 data.tofile( self.fp )
538
538
539 self.datablock.fill(0)
539 self.datablock.fill(0)
540
540
541 self.profileIndex = 0
541 self.profileIndex = 0
542 self.flagIsNewFile = 0
542 self.flagIsNewFile = 0
543 self.flagIsNewBlock = 1
543 self.flagIsNewBlock = 1
544
544
545 self.blockIndex += 1
545 self.blockIndex += 1
546 self.nTotalBlocks += 1
546 self.nTotalBlocks += 1
547
547
548 # print "[Writing] Block = %04d" %self.blockIndex
548 # print "[Writing] Block = %04d" %self.blockIndex
549
549
550 def putData(self):
550 def putData(self):
551 """
551 """
552 Setea un bloque de datos y luego los escribe en un file
552 Setea un bloque de datos y luego los escribe en un file
553
553
554 Affected:
554 Affected:
555 self.flagIsNewBlock
555 self.flagIsNewBlock
556 self.profileIndex
556 self.profileIndex
557
557
558 Return:
558 Return:
559 0 : Si no hay data o no hay mas files que puedan escribirse
559 0 : Si no hay data o no hay mas files que puedan escribirse
560 1 : Si se escribio la data de un bloque en un file
560 1 : Si se escribio la data de un bloque en un file
561 """
561 """
562 if self.dataOut.flagNoData:
562 if self.dataOut.flagNoData:
563 return 0
563 return 0
564
564
565 self.flagIsNewBlock = 0
565 self.flagIsNewBlock = 0
566
566
567 if self.dataOut.flagDiscontinuousBlock:
567 if self.dataOut.flagDiscontinuousBlock:
568 self.datablock.fill(0)
568 self.datablock.fill(0)
569 self.profileIndex = 0
569 self.profileIndex = 0
570 self.setNextFile()
570 self.setNextFile()
571
571
572 if self.profileIndex == 0:
572 if self.profileIndex == 0:
573 self.setBasicHeader()
573 self.setBasicHeader()
574
574
575 self.datablock[:,self.profileIndex,:] = self.dataOut.data
575 self.datablock[:,self.profileIndex,:] = self.dataOut.data
576
576
577 self.profileIndex += 1
577 self.profileIndex += 1
578
578
579 if self.hasAllDataInBuffer():
579 if self.hasAllDataInBuffer():
580 #if self.flagIsNewFile:
580 #if self.flagIsNewFile:
581 self.writeNextBlock()
581 self.writeNextBlock()
582 # self.setFirstHeader()
582 # self.setFirstHeader()
583
583
584 return 1
584 return 1
585
585
586 def __getBlockSize(self):
586 def __getBlockSize(self):
587 '''
587 '''
588 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
588 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
589 '''
589 '''
590
590
591 dtype_width = self.getDtypeWidth()
591 dtype_width = self.getDtypeWidth()
592
592
593 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
593 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
594
594
595 return blocksize
595 return blocksize
596
596
597 def setFirstHeader(self):
597 def setFirstHeader(self):
598
598
599 """
599 """
600 Obtiene una copia del First Header
600 Obtiene una copia del First Header
601
601
602 Affected:
602 Affected:
603 self.systemHeaderObj
603 self.systemHeaderObj
604 self.radarControllerHeaderObj
604 self.radarControllerHeaderObj
605 self.dtype
605 self.dtype
606
606
607 Return:
607 Return:
608 None
608 None
609 """
609 """
610
610
611 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
611 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
612 self.systemHeaderObj.nChannels = self.dataOut.nChannels
612 self.systemHeaderObj.nChannels = self.dataOut.nChannels
613 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
613 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
614
614
615 self.processingHeaderObj.dtype = 0 # Voltage
615 self.processingHeaderObj.dtype = 0 # Voltage
616 self.processingHeaderObj.blockSize = self.__getBlockSize()
616 self.processingHeaderObj.blockSize = self.__getBlockSize()
617 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
617 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
618 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
618 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
619 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
619 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
620 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
620 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
621 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
621 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
622 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
622 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
623
623
624 if self.dataOut.code is not None:
624 if self.dataOut.code is not None:
625 self.processingHeaderObj.code = self.dataOut.code
625 self.processingHeaderObj.code = self.dataOut.code
626 self.processingHeaderObj.nCode = self.dataOut.nCode
626 self.processingHeaderObj.nCode = self.dataOut.nCode
627 self.processingHeaderObj.nBaud = self.dataOut.nBaud
627 self.processingHeaderObj.nBaud = self.dataOut.nBaud
628
628
629 if self.processingHeaderObj.nWindows != 0:
629 if self.processingHeaderObj.nWindows != 0:
630 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
630 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
631 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
631 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
632 self.processingHeaderObj.nHeights = self.dataOut.nHeights
632 self.processingHeaderObj.nHeights = self.dataOut.nHeights
633 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
633 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
634
634
635 self.processingHeaderObj.processFlags = self.getProcessFlags()
635 self.processingHeaderObj.processFlags = self.getProcessFlags()
636
636
637 self.setBasicHeader() No newline at end of file
637 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -1,1012 +1,1013
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13
13
14 import time
14 import time
15
15
16 import threading
16 import threading
17 Thread = threading.Thread
17 Thread = threading.Thread
18
18
19 # try:
19 # try:
20 # from gevent import sleep
20 # from gevent import sleep
21 # except:
21 # except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25
25
26 class Remote(Thread):
26 class Remote(Thread):
27 """
27 """
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 used to upload or download files remotely.
29 used to upload or download files remotely.
30
30
31 Non-standard Python modules used:
31 Non-standard Python modules used:
32 None
32 None
33
33
34 Written by:
34 Written by:
35
35
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37
37
38 """
38 """
39
39
40 server = None
40 server = None
41 username = None
41 username = None
42 password = None
42 password = None
43 remotefolder = None
43 remotefolder = None
44
44
45 period = 60
45 period = 60
46 fileList = []
46 fileList = []
47 bussy = False
47 bussy = False
48
48
49 def __init__(self, server, username, password, remotefolder, period=60):
49 def __init__(self, server, username, password, remotefolder, period=60):
50
50
51 Thread.__init__(self)
51 Thread.__init__(self)
52
52
53 self.setDaemon(True)
53 self.setDaemon(True)
54
54
55 self.status = 0
55 self.status = 0
56
56
57 self.__server = server
57 self.__server = server
58 self.__username = username
58 self.__username = username
59 self.__password = password
59 self.__password = password
60 self.__remotefolder = remotefolder
60 self.__remotefolder = remotefolder
61
61
62 self.period = period
62 self.period = period
63
63
64 self.fileList = []
64 self.fileList = []
65 self.bussy = False
65 self.bussy = False
66
66
67 self.stopFlag = False
67 self.stopFlag = False
68
68
69 print "[Remote Server] Opening server: %s" %self.__server
69 print "[Remote Server] Opening server: %s" %self.__server
70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 print "[Remote Server] %s server was opened successfully" %self.__server
71 print "[Remote Server] %s server was opened successfully" %self.__server
72
72
73 self.close()
73 self.close()
74
74
75 self.mutex = threading.Lock()
75 self.mutex = threading.Lock()
76
76
77 def stop(self):
77 def stop(self):
78
78
79 self.stopFlag = True
79 self.stopFlag = True
80 self.join(10)
80 self.join(10)
81
81
82 def open(self):
82 def open(self):
83 """
83 """
84 Connect to server and create a connection class (FTP or SSH) to remote server.
84 Connect to server and create a connection class (FTP or SSH) to remote server.
85 """
85 """
86 raise NotImplementedError, "Implement this method in child class"
86 raise NotImplementedError, "Implement this method in child class"
87
87
88 def close(self):
88 def close(self):
89 """
89 """
90 Close connection to server
90 Close connection to server
91 """
91 """
92 raise NotImplementedError, "Implement this method in child class"
92 raise NotImplementedError, "Implement this method in child class"
93
93
94 def mkdir(self, remotefolder):
94 def mkdir(self, remotefolder):
95 """
95 """
96 Create a folder remotely
96 Create a folder remotely
97 """
97 """
98 raise NotImplementedError, "Implement this method in child class"
98 raise NotImplementedError, "Implement this method in child class"
99
99
100 def cd(self, remotefolder):
100 def cd(self, remotefolder):
101 """
101 """
102 Change working directory in remote server
102 Change working directory in remote server
103 """
103 """
104 raise NotImplementedError, "Implement this method in child class"
104 raise NotImplementedError, "Implement this method in child class"
105
105
106 def download(self, filename, localfolder=None):
106 def download(self, filename, localfolder=None):
107 """
107 """
108 Download a file from server to local host
108 Download a file from server to local host
109 """
109 """
110 raise NotImplementedError, "Implement this method in child class"
110 raise NotImplementedError, "Implement this method in child class"
111
111
112 def sendFile(self, fullfilename):
112 def sendFile(self, fullfilename):
113 """
113 """
114 sendFile method is used to upload a local file to the current directory in remote server
114 sendFile method is used to upload a local file to the current directory in remote server
115
115
116 Inputs:
116 Inputs:
117 fullfilename - full path name of local file to store in remote directory
117 fullfilename - full path name of local file to store in remote directory
118
118
119 Returns:
119 Returns:
120 0 in error case else 1
120 0 in error case else 1
121 """
121 """
122 raise NotImplementedError, "Implement this method in child class"
122 raise NotImplementedError, "Implement this method in child class"
123
123
124 def upload(self, fullfilename, remotefolder=None):
124 def upload(self, fullfilename, remotefolder=None):
125 """
125 """
126 upload method is used to upload a local file to remote directory. This method changes
126 upload method is used to upload a local file to remote directory. This method changes
127 working directory before sending a file.
127 working directory before sending a file.
128
128
129 Inputs:
129 Inputs:
130 fullfilename - full path name of local file to store in remote directory
130 fullfilename - full path name of local file to store in remote directory
131
131
132 remotefolder - remote directory
132 remotefolder - remote directory
133
133
134 Returns:
134 Returns:
135 0 in error case else 1
135 0 in error case else 1
136 """
136 """
137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
138
138
139 if not self.status:
139 if not self.status:
140 return 0
140 return 0
141
141
142 if remotefolder == None:
142 if remotefolder == None:
143 remotefolder = self.remotefolder
143 remotefolder = self.remotefolder
144
144
145 if not self.cd(remotefolder):
145 if not self.cd(remotefolder):
146 return 0
146 return 0
147
147
148 if not self.sendFile(fullfilename):
148 if not self.sendFile(fullfilename):
149 print "[Remote Server] Error uploading file %s" %fullfilename
149 print "[Remote Server] Error uploading file %s" %fullfilename
150 return 0
150 return 0
151
151
152 print "[Remote Server] upload finished successfully"
152 print "[Remote Server] upload finished successfully"
153
153
154 return 1
154 return 1
155
155
156 def delete(self, filename):
156 def delete(self, filename):
157 """
157 """
158 Remove a file from remote server
158 Remove a file from remote server
159 """
159 """
160 pass
160 pass
161
161
162 def updateFileList(self, fileList):
162 def updateFileList(self, fileList):
163 """
163 """
164 Remove a file from remote server
164 Remove a file from remote server
165 """
165 """
166
166
167 if fileList == self.fileList:
167 if fileList == self.fileList:
168 return 0
168 return 0
169
169
170 self.mutex.acquire()
170 self.mutex.acquire()
171 # init = time.time()
171 # init = time.time()
172 #
172 #
173 # while(self.bussy):
173 # while(self.bussy):
174 # sleep(0.1)
174 # sleep(0.1)
175 # if time.time() - init > 2*self.period:
175 # if time.time() - init > 2*self.period:
176 # return 0
176 # return 0
177
177
178 self.fileList = fileList
178 self.fileList = fileList
179 self.mutex.release()
179 self.mutex.release()
180 return 1
180 return 1
181
181
182 def run(self):
182 def run(self):
183
183
184 if not self.status:
184 if not self.status:
185 print "Finishing FTP service"
185 print "Finishing FTP service"
186 return
186 return
187
187
188 if not self.cd(self.remotefolder):
188 if not self.cd(self.remotefolder):
189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
190
190
191 while True:
191 while True:
192
192
193 for i in range(self.period):
193 for i in range(self.period):
194 if self.stopFlag:
194 if self.stopFlag:
195 break
195 break
196 sleep(1)
196 sleep(1)
197
197
198 if self.stopFlag:
198 if self.stopFlag:
199 break
199 break
200
200
201 # self.bussy = True
201 # self.bussy = True
202 self.mutex.acquire()
202 self.mutex.acquire()
203
203
204 print "[Remote Server] Opening %s" %self.__server
204 print "[Remote Server] Opening %s" %self.__server
205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 self.mutex.release()
206 self.mutex.release()
207 continue
207 continue
208
208
209 for thisFile in self.fileList:
209 for thisFile in self.fileList:
210 self.upload(thisFile, self.remotefolder)
210 self.upload(thisFile, self.remotefolder)
211
211
212 print "[Remote Server] Closing %s" %self.__server
212 print "[Remote Server] Closing %s" %self.__server
213 self.close()
213 self.close()
214
214
215 self.mutex.release()
215 self.mutex.release()
216 # self.bussy = False
216 # self.bussy = False
217
217
218 print "[Remote Server] Thread stopped successfully"
218 print "[Remote Server] Thread stopped successfully"
219
219
220 class FTPClient(Remote):
220 class FTPClient(Remote):
221
221
222 __ftpClientObj = None
222 __ftpClientObj = None
223
223
224 def __init__(self, server, username, password, remotefolder, period=60):
224 def __init__(self, server, username, password, remotefolder, period=60):
225 """
225 """
226 """
226 """
227 Remote.__init__(self, server, username, password, remotefolder, period)
227 Remote.__init__(self, server, username, password, remotefolder, period)
228
228
229 def open(self, server, username, password, remotefolder):
229 def open(self, server, username, password, remotefolder):
230
230
231 """
231 """
232 This method is used to set FTP parameters and establish a connection to remote server
232 This method is used to set FTP parameters and establish a connection to remote server
233
233
234 Inputs:
234 Inputs:
235 server - remote server IP Address
235 server - remote server IP Address
236
236
237 username - remote server Username
237 username - remote server Username
238
238
239 password - remote server password
239 password - remote server password
240
240
241 remotefolder - remote server current working directory
241 remotefolder - remote server current working directory
242
242
243 Return:
243 Return:
244 Boolean - Returns 1 if a connection has been established, 0 otherwise
244 Boolean - Returns 1 if a connection has been established, 0 otherwise
245
245
246 Affects:
246 Affects:
247 self.status - in case of error or fail connection this parameter is set to 0 else 1
247 self.status - in case of error or fail connection this parameter is set to 0 else 1
248
248
249 """
249 """
250
250
251 if server == None:
251 if server == None:
252 raise ValueError, "FTP server should be defined"
252 raise ValueError, "FTP server should be defined"
253
253
254 if username == None:
254 if username == None:
255 raise ValueError, "FTP username should be defined"
255 raise ValueError, "FTP username should be defined"
256
256
257 if password == None:
257 if password == None:
258 raise ValueError, "FTP password should be defined"
258 raise ValueError, "FTP password should be defined"
259
259
260 if remotefolder == None:
260 if remotefolder == None:
261 raise ValueError, "FTP remote folder should be defined"
261 raise ValueError, "FTP remote folder should be defined"
262
262
263 try:
263 try:
264 ftpClientObj = ftplib.FTP(server)
264 ftpClientObj = ftplib.FTP(server)
265 except ftplib.all_errors, e:
265 except ftplib.all_errors, e:
266 print "[FTP Server]: FTP server connection fail: %s" %server
266 print "[FTP Server]: FTP server connection fail: %s" %server
267 print "[FTP Server]:", e
267 print "[FTP Server]:", e
268 self.status = 0
268 self.status = 0
269 return 0
269 return 0
270
270
271 try:
271 try:
272 ftpClientObj.login(username, password)
272 ftpClientObj.login(username, password)
273 except ftplib.all_errors:
273 except ftplib.all_errors:
274 print "[FTP Server]: FTP username or password are incorrect"
274 print "[FTP Server]: FTP username or password are incorrect"
275 self.status = 0
275 self.status = 0
276 return 0
276 return 0
277
277
278 if remotefolder == None:
278 if remotefolder == None:
279 remotefolder = ftpClientObj.pwd()
279 remotefolder = ftpClientObj.pwd()
280 else:
280 else:
281 try:
281 try:
282 ftpClientObj.cwd(remotefolder)
282 ftpClientObj.cwd(remotefolder)
283 except ftplib.all_errors:
283 except ftplib.all_errors:
284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
285 remotefolder = ftpClientObj.pwd()
285 remotefolder = ftpClientObj.pwd()
286
286
287 self.server = server
287 self.server = server
288 self.username = username
288 self.username = username
289 self.password = password
289 self.password = password
290 self.remotefolder = remotefolder
290 self.remotefolder = remotefolder
291 self.__ftpClientObj = ftpClientObj
291 self.__ftpClientObj = ftpClientObj
292 self.status = 1
292 self.status = 1
293
293
294 return 1
294 return 1
295
295
296 def close(self):
296 def close(self):
297 """
297 """
298 Close connection to remote server
298 Close connection to remote server
299 """
299 """
300 if not self.status:
300 if not self.status:
301 return 0
301 return 0
302
302
303 self.__ftpClientObj.close()
303 self.__ftpClientObj.close()
304
304
305 def mkdir(self, remotefolder):
305 def mkdir(self, remotefolder):
306 """
306 """
307 mkdir is used to make a new directory in remote server
307 mkdir is used to make a new directory in remote server
308
308
309 Input:
309 Input:
310 remotefolder - directory name
310 remotefolder - directory name
311
311
312 Return:
312 Return:
313 0 in error case else 1
313 0 in error case else 1
314 """
314 """
315 if not self.status:
315 if not self.status:
316 return 0
316 return 0
317
317
318 try:
318 try:
319 self.__ftpClientObj.mkd(dirname)
319 self.__ftpClientObj.mkd(dirname)
320 except ftplib.all_errors:
320 except ftplib.all_errors:
321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
322 return 0
322 return 0
323
323
324 return 1
324 return 1
325
325
326 def cd(self, remotefolder):
326 def cd(self, remotefolder):
327 """
327 """
328 cd is used to change remote working directory on server
328 cd is used to change remote working directory on server
329
329
330 Input:
330 Input:
331 remotefolder - current working directory
331 remotefolder - current working directory
332
332
333 Affects:
333 Affects:
334 self.remotefolder
334 self.remotefolder
335
335
336 Return:
336 Return:
337 0 in case of error else 1
337 0 in case of error else 1
338 """
338 """
339 if not self.status:
339 if not self.status:
340 return 0
340 return 0
341
341
342 if remotefolder == self.remotefolder:
342 if remotefolder == self.remotefolder:
343 return 1
343 return 1
344
344
345 try:
345 try:
346 self.__ftpClientObj.cwd(remotefolder)
346 self.__ftpClientObj.cwd(remotefolder)
347 except ftplib.all_errors:
347 except ftplib.all_errors:
348 print '[FTP Server]: Error changing to %s' %remotefolder
348 print '[FTP Server]: Error changing to %s' %remotefolder
349 print '[FTP Server]: Trying to create remote folder'
349 print '[FTP Server]: Trying to create remote folder'
350
350
351 if not self.mkdir(remotefolder):
351 if not self.mkdir(remotefolder):
352 print '[FTP Server]: Remote folder could not be created'
352 print '[FTP Server]: Remote folder could not be created'
353 return 0
353 return 0
354
354
355 try:
355 try:
356 self.__ftpClientObj.cwd(remotefolder)
356 self.__ftpClientObj.cwd(remotefolder)
357 except ftplib.all_errors:
357 except ftplib.all_errors:
358 return 0
358 return 0
359
359
360 self.remotefolder = remotefolder
360 self.remotefolder = remotefolder
361
361
362 return 1
362 return 1
363
363
364 def sendFile(self, fullfilename):
364 def sendFile(self, fullfilename):
365
365
366 if not self.status:
366 if not self.status:
367 return 0
367 return 0
368
368
369 fp = open(fullfilename, 'rb')
369 fp = open(fullfilename, 'rb')
370
370
371 filename = os.path.basename(fullfilename)
371 filename = os.path.basename(fullfilename)
372
372
373 command = "STOR %s" %filename
373 command = "STOR %s" %filename
374
374
375 try:
375 try:
376 self.__ftpClientObj.storbinary(command, fp)
376 self.__ftpClientObj.storbinary(command, fp)
377 except ftplib.all_errors, e:
377 except ftplib.all_errors, e:
378 print "[FTP Server]:", e
378 print "[FTP Server]:", e
379 return 0
379 return 0
380
380
381 try:
381 try:
382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 except ftplib.all_errors, e:
383 except ftplib.all_errors, e:
384 print "[FTP Server]:", e
384 print "[FTP Server]:", e
385
385
386 fp.close()
386 fp.close()
387
387
388 return 1
388 return 1
389
389
390 class SSHClient(Remote):
390 class SSHClient(Remote):
391
391
392 __sshClientObj = None
392 __sshClientObj = None
393 __scpClientObj = None
393 __scpClientObj = None
394
394
395 def __init__(self, server, username, password, remotefolder, period=60):
395 def __init__(self, server, username, password, remotefolder, period=60):
396 """
396 """
397 """
397 """
398 Remote.__init__(self, server, username, password, remotefolder, period)
398 Remote.__init__(self, server, username, password, remotefolder, period)
399
399
400 def open(self, server, username, password, remotefolder, port=22):
400 def open(self, server, username, password, remotefolder, port=22):
401
401
402 """
402 """
403 This method is used to set SSH parameters and establish a connection to a remote server
403 This method is used to set SSH parameters and establish a connection to a remote server
404
404
405 Inputs:
405 Inputs:
406 server - remote server IP Address
406 server - remote server IP Address
407
407
408 username - remote server Username
408 username - remote server Username
409
409
410 password - remote server password
410 password - remote server password
411
411
412 remotefolder - remote server current working directory
412 remotefolder - remote server current working directory
413
413
414 Return: void
414 Return: void
415
415
416 Affects:
416 Affects:
417 self.status - in case of error or fail connection this parameter is set to 0 else 1
417 self.status - in case of error or fail connection this parameter is set to 0 else 1
418
418
419 """
419 """
420 import socket
420 import socket
421
421
422 if server == None:
422 if server == None:
423 raise ValueError, "SSH server should be defined"
423 raise ValueError, "SSH server should be defined"
424
424
425 if username == None:
425 if username == None:
426 raise ValueError, "SSH username should be defined"
426 raise ValueError, "SSH username should be defined"
427
427
428 if password == None:
428 if password == None:
429 raise ValueError, "SSH password should be defined"
429 raise ValueError, "SSH password should be defined"
430
430
431 if remotefolder == None:
431 if remotefolder == None:
432 raise ValueError, "SSH remote folder should be defined"
432 raise ValueError, "SSH remote folder should be defined"
433
433
434 sshClientObj = paramiko.SSHClient()
434 sshClientObj = paramiko.SSHClient()
435
435
436 sshClientObj.load_system_host_keys()
436 sshClientObj.load_system_host_keys()
437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
438
438
439 self.status = 0
439 self.status = 0
440 try:
440 try:
441 sshClientObj.connect(server, username=username, password=password, port=port)
441 sshClientObj.connect(server, username=username, password=password, port=port)
442 except paramiko.AuthenticationException, e:
442 except paramiko.AuthenticationException, e:
443 # print "SSH username or password are incorrect: %s"
443 # print "SSH username or password are incorrect: %s"
444 print "[SSH Server]:", e
444 print "[SSH Server]:", e
445 return 0
445 return 0
446 except SSHException, e:
446 except SSHException, e:
447 print "[SSH Server]:", e
447 print "[SSH Server]:", e
448 return 0
448 return 0
449 except socket.error:
449 except socket.error:
450 self.status = 0
450 self.status = 0
451 print "[SSH Server]:", e
451 print "[SSH Server]:", e
452 return 0
452 return 0
453
453
454 self.status = 1
454 self.status = 1
455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
456
456
457 if remotefolder == None:
457 if remotefolder == None:
458 remotefolder = self.pwd()
458 remotefolder = self.pwd()
459
459
460 self.server = server
460 self.server = server
461 self.username = username
461 self.username = username
462 self.password = password
462 self.password = password
463 self.__sshClientObj = sshClientObj
463 self.__sshClientObj = sshClientObj
464 self.__scpClientObj = scpClientObj
464 self.__scpClientObj = scpClientObj
465 self.status = 1
465 self.status = 1
466
466
467 if not self.cd(remotefolder):
467 if not self.cd(remotefolder):
468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
469 return 0
469 return 0
470
470
471 self.remotefolder = remotefolder
471 self.remotefolder = remotefolder
472
472
473 return 1
473 return 1
474
474
475 def close(self):
475 def close(self):
476 """
476 """
477 Close connection to remote server
477 Close connection to remote server
478 """
478 """
479 if not self.status:
479 if not self.status:
480 return 0
480 return 0
481
481
482 self.__scpClientObj.close()
482 self.__scpClientObj.close()
483 self.__sshClientObj.close()
483 self.__sshClientObj.close()
484
484
485 def __execute(self, command):
485 def __execute(self, command):
486 """
486 """
487 __execute a command on remote server
487 __execute a command on remote server
488
488
489 Input:
489 Input:
490 command - Exmaple 'ls -l'
490 command - Exmaple 'ls -l'
491
491
492 Return:
492 Return:
493 0 in error case else 1
493 0 in error case else 1
494 """
494 """
495 if not self.status:
495 if not self.status:
496 return 0
496 return 0
497
497
498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
499
499
500 result = stderr.readlines()
500 result = stderr.readlines()
501 if len(result) > 1:
501 if len(result) > 1:
502 return 0
502 return 0
503
503
504 result = stdout.readlines()
504 result = stdout.readlines()
505 if len(result) > 1:
505 if len(result) > 1:
506 return result[0][:-1]
506 return result[0][:-1]
507
507
508 return 1
508 return 1
509
509
510 def mkdir(self, remotefolder):
510 def mkdir(self, remotefolder):
511 """
511 """
512 mkdir is used to make a new directory in remote server
512 mkdir is used to make a new directory in remote server
513
513
514 Input:
514 Input:
515 remotefolder - directory name
515 remotefolder - directory name
516
516
517 Return:
517 Return:
518 0 in error case else 1
518 0 in error case else 1
519 """
519 """
520
520
521 command = 'mkdir %s' %remotefolder
521 command = 'mkdir %s' %remotefolder
522
522
523 return self.__execute(command)
523 return self.__execute(command)
524
524
525 def pwd(self):
525 def pwd(self):
526
526
527 command = 'pwd'
527 command = 'pwd'
528
528
529 return self.__execute(command)
529 return self.__execute(command)
530
530
531 def cd(self, remotefolder):
531 def cd(self, remotefolder):
532 """
532 """
533 cd is used to change remote working directory on server
533 cd is used to change remote working directory on server
534
534
535 Input:
535 Input:
536 remotefolder - current working directory
536 remotefolder - current working directory
537
537
538 Affects:
538 Affects:
539 self.remotefolder
539 self.remotefolder
540
540
541 Return:
541 Return:
542 0 in case of error else 1
542 0 in case of error else 1
543 """
543 """
544 if not self.status:
544 if not self.status:
545 return 0
545 return 0
546
546
547 if remotefolder == self.remotefolder:
547 if remotefolder == self.remotefolder:
548 return 1
548 return 1
549
549
550 chk_command = "cd %s; pwd" %remotefolder
550 chk_command = "cd %s; pwd" %remotefolder
551 mkdir_command = "mkdir %s" %remotefolder
551 mkdir_command = "mkdir %s" %remotefolder
552
552
553 if not self.__execute(chk_command):
553 if not self.__execute(chk_command):
554 if not self.__execute(mkdir_command):
554 if not self.__execute(mkdir_command):
555 self.remotefolder = None
555 self.remotefolder = None
556 return 0
556 return 0
557
557
558 self.remotefolder = remotefolder
558 self.remotefolder = remotefolder
559
559
560 return 1
560 return 1
561
561
562 def sendFile(self, fullfilename):
562 def sendFile(self, fullfilename):
563
563
564 if not self.status:
564 if not self.status:
565 return 0
565 return 0
566
566
567 try:
567 try:
568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
569 except scp.ScpError, e:
569 except scp.ScpError, e:
570 print "[SSH Server]", str(e)
570 print "[SSH Server]", str(e)
571 return 0
571 return 0
572
572
573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
574 command = 'chmod 775 %s' %remotefile
574 command = 'chmod 775 %s' %remotefile
575
575
576 return self.__execute(command)
576 return self.__execute(command)
577
577
578 class SendToServer(ProcessingUnit):
578 class SendToServer(ProcessingUnit):
579
579
580 def __init__(self):
580 def __init__(self):
581
581
582 ProcessingUnit.__init__(self)
582 ProcessingUnit.__init__(self)
583
583
584 self.isConfig = False
584 self.isConfig = False
585 self.clientObj = None
585 self.clientObj = None
586
586
587
587
588
588
589 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
589 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
590
590
591 self.clientObj = None
591 self.clientObj = None
592 self.localfolder = localfolder
592 self.localfolder = localfolder
593 self.ext = ext
593 self.ext = ext
594 self.period = period
594 self.period = period
595
595
596 if str.lower(protocol) == 'ftp':
596 if str.lower(protocol) == 'ftp':
597 self.clientObj = FTPClient(server, username, password, remotefolder, period)
597 self.clientObj = FTPClient(server, username, password, remotefolder, period)
598
598
599 if str.lower(protocol) == 'ssh':
599 if str.lower(protocol) == 'ssh':
600 self.clientObj = SSHClient(server, username, password, remotefolder, period)
600 self.clientObj = SSHClient(server, username, password, remotefolder, period)
601
601
602 if not self.clientObj:
602 if not self.clientObj:
603 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
603 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
604
604
605 self.clientObj.start()
605 self.clientObj.start()
606
606
607 def findFiles(self):
607 def findFiles(self):
608
608
609 if not type(self.localfolder) == list:
609 if not type(self.localfolder) == list:
610 folderList = [self.localfolder]
610 folderList = [self.localfolder]
611 else:
611 else:
612 folderList = self.localfolder
612 folderList = self.localfolder
613
613
614 #Remove duplicate items
614 #Remove duplicate items
615 folderList = list(set(folderList))
615 folderList = list(set(folderList))
616
616
617 fullfilenameList = []
617 fullfilenameList = []
618
618
619 for thisFolder in folderList:
619 for thisFolder in folderList:
620
620
621 print "[Remote Server]: Searching files on %s" %thisFolder
621 print "[Remote Server]: Searching files on %s" %thisFolder
622
622
623 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
623 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
624
624
625 if len(filenameList) < 1:
625 if len(filenameList) < 1:
626
626 continue
627 continue
627
628
628 for thisFile in filenameList:
629 for thisFile in filenameList:
629 fullfilename = os.path.join(thisFolder, thisFile)
630 fullfilename = os.path.join(thisFolder, thisFile)
630
631
631 if fullfilename in fullfilenameList:
632 if fullfilename in fullfilenameList:
632 continue
633 continue
633
634
634 #Only files modified in the last 30 minutes are considered
635 #Only files modified in the last 30 minutes are considered
635 if os.path.getmtime(fullfilename) < time.time() - 30*60:
636 if os.path.getmtime(fullfilename) < time.time() - 30*60:
636 continue
637 continue
637
638
638 fullfilenameList.append(fullfilename)
639 fullfilenameList.append(fullfilename)
639
640
640 return fullfilenameList
641 return fullfilenameList
641
642
642 def run(self, **kwargs):
643 def run(self, **kwargs):
643 if not self.isConfig:
644 if not self.isConfig:
644 self.init = time.time()
645 self.init = time.time()
645 self.setup(**kwargs)
646 self.setup(**kwargs)
646 self.isConfig = True
647 self.isConfig = True
647
648
648 if not self.clientObj.is_alive():
649 if not self.clientObj.is_alive():
649 print "[Remote Server]: Restarting connection "
650 print "[Remote Server]: Restarting connection "
650 self.setup(**kwargs)
651 self.setup(**kwargs)
651
652
652 if time.time() - self.init >= self.period:
653 if time.time() - self.init >= self.period:
653 fullfilenameList = self.findFiles()
654 fullfilenameList = self.findFiles()
654
655
655 if self.clientObj.updateFileList(fullfilenameList):
656 if self.clientObj.updateFileList(fullfilenameList):
656 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
657 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
657 self.init = time.time()
658 self.init = time.time()
658
659
659 def close(self):
660 def close(self):
660 print "[Remote Server] Stopping thread"
661 print "[Remote Server] Stopping thread"
661 self.clientObj.stop()
662 self.clientObj.stop()
662
663
663
664
664 class FTP(object):
665 class FTP(object):
665 """
666 """
666 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
667 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
667
668
668 Non-standard Python modules used: None
669 Non-standard Python modules used: None
669
670
670 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
671 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
671 """
672 """
672
673
673 def __init__(self,server = None, username=None, password=None, remotefolder=None):
674 def __init__(self,server = None, username=None, password=None, remotefolder=None):
674 """
675 """
675 This method is used to setting parameters for FTP and establishing connection to remote server
676 This method is used to setting parameters for FTP and establishing connection to remote server
676
677
677 Inputs:
678 Inputs:
678 server - remote server IP Address
679 server - remote server IP Address
679
680
680 username - remote server Username
681 username - remote server Username
681
682
682 password - remote server password
683 password - remote server password
683
684
684 remotefolder - remote server current working directory
685 remotefolder - remote server current working directory
685
686
686 Return: void
687 Return: void
687
688
688 Affects:
689 Affects:
689 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
690 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
690
691
691 self.folderList - sub-folder list of remote folder
692 self.folderList - sub-folder list of remote folder
692
693
693 self.fileList - file list of remote folder
694 self.fileList - file list of remote folder
694
695
695
696
696 """
697 """
697
698
698 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
699 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
699 server, username, password, remotefolder = self.parmsByDefault()
700 server, username, password, remotefolder = self.parmsByDefault()
700
701
701 self.server = server
702 self.server = server
702 self.username = username
703 self.username = username
703 self.password = password
704 self.password = password
704 self.remotefolder = remotefolder
705 self.remotefolder = remotefolder
705 self.file = None
706 self.file = None
706 self.ftp = None
707 self.ftp = None
707 self.status = 0
708 self.status = 0
708
709
709 try:
710 try:
710 self.ftp = ftplib.FTP(self.server)
711 self.ftp = ftplib.FTP(self.server)
711 self.ftp.login(self.username,self.password)
712 self.ftp.login(self.username,self.password)
712 self.ftp.cwd(self.remotefolder)
713 self.ftp.cwd(self.remotefolder)
713 # print 'Connect to FTP Server: Successfully'
714 # print 'Connect to FTP Server: Successfully'
714
715
715 except ftplib.all_errors:
716 except ftplib.all_errors:
716 print 'Error FTP Service'
717 print 'Error FTP Service'
717 self.status = 1
718 self.status = 1
718 return
719 return
719
720
720
721
721
722
722 self.dirList = []
723 self.dirList = []
723
724
724 try:
725 try:
725 self.dirList = self.ftp.nlst()
726 self.dirList = self.ftp.nlst()
726
727
727 except ftplib.error_perm, resp:
728 except ftplib.error_perm, resp:
728 if str(resp) == "550 No files found":
729 if str(resp) == "550 No files found":
729 print "no files in this directory"
730 print "no files in this directory"
730 self.status = 1
731 self.status = 1
731 return
732 return
732
733
733 except ftplib.all_errors:
734 except ftplib.all_errors:
734 print 'Error Displaying Dir-Files'
735 print 'Error Displaying Dir-Files'
735 self.status = 1
736 self.status = 1
736 return
737 return
737
738
738 self.fileList = []
739 self.fileList = []
739 self.folderList = []
740 self.folderList = []
740 #only for test
741 #only for test
741 for f in self.dirList:
742 for f in self.dirList:
742 name, ext = os.path.splitext(f)
743 name, ext = os.path.splitext(f)
743 if ext != '':
744 if ext != '':
744 self.fileList.append(f)
745 self.fileList.append(f)
745 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
746 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
746
747
747 def parmsByDefault(self):
748 def parmsByDefault(self):
748 server = 'jro-app.igp.gob.pe'
749 server = 'jro-app.igp.gob.pe'
749 username = 'wmaster'
750 username = 'wmaster'
750 password = 'mst2010vhf'
751 password = 'mst2010vhf'
751 remotefolder = '/home/wmaster/graficos'
752 remotefolder = '/home/wmaster/graficos'
752
753
753 return server, username, password, remotefolder
754 return server, username, password, remotefolder
754
755
755
756
756 def mkd(self,dirname):
757 def mkd(self,dirname):
757 """
758 """
758 mkd is used to make directory in remote server
759 mkd is used to make directory in remote server
759
760
760 Input:
761 Input:
761 dirname - directory name
762 dirname - directory name
762
763
763 Return:
764 Return:
764 1 in error case else 0
765 1 in error case else 0
765 """
766 """
766 try:
767 try:
767 self.ftp.mkd(dirname)
768 self.ftp.mkd(dirname)
768 except:
769 except:
769 print 'Error creating remote folder:%s'%dirname
770 print 'Error creating remote folder:%s'%dirname
770 return 1
771 return 1
771
772
772 return 0
773 return 0
773
774
774
775
775 def delete(self,filename):
776 def delete(self,filename):
776 """
777 """
777 delete is used to delete file in current working directory of remote server
778 delete is used to delete file in current working directory of remote server
778
779
779 Input:
780 Input:
780 filename - filename to delete in remote folder
781 filename - filename to delete in remote folder
781
782
782 Return:
783 Return:
783 1 in error case else 0
784 1 in error case else 0
784 """
785 """
785
786
786 try:
787 try:
787 self.ftp.delete(filename)
788 self.ftp.delete(filename)
788 except:
789 except:
789 print 'Error deleting remote file:%s'%filename
790 print 'Error deleting remote file:%s'%filename
790 return 1
791 return 1
791
792
792 return 0
793 return 0
793
794
794 def download(self,filename,localfolder):
795 def download(self,filename,localfolder):
795 """
796 """
796 download is used to downloading file from remote folder into local folder
797 download is used to downloading file from remote folder into local folder
797
798
798 Inputs:
799 Inputs:
799 filename - filename to donwload
800 filename - filename to donwload
800
801
801 localfolder - directory local to store filename
802 localfolder - directory local to store filename
802
803
803 Returns:
804 Returns:
804 self.status - 1 in error case else 0
805 self.status - 1 in error case else 0
805 """
806 """
806
807
807 self.status = 0
808 self.status = 0
808
809
809
810
810 if not(filename in self.fileList):
811 if not(filename in self.fileList):
811 print 'filename:%s not exists'%filename
812 print 'filename:%s not exists'%filename
812 self.status = 1
813 self.status = 1
813 return self.status
814 return self.status
814
815
815 newfilename = os.path.join(localfolder,filename)
816 newfilename = os.path.join(localfolder,filename)
816
817
817 self.file = open(newfilename, 'wb')
818 self.file = open(newfilename, 'wb')
818
819
819 try:
820 try:
820 print 'Download: ' + filename
821 print 'Download: ' + filename
821 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
822 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
822 print 'Download Complete'
823 print 'Download Complete'
823 except ftplib.all_errors:
824 except ftplib.all_errors:
824 print 'Error Downloading ' + filename
825 print 'Error Downloading ' + filename
825 self.status = 1
826 self.status = 1
826 return self.status
827 return self.status
827
828
828 self.file.close()
829 self.file.close()
829
830
830 return self.status
831 return self.status
831
832
832
833
833 def __handleDownload(self,block):
834 def __handleDownload(self,block):
834 """
835 """
835 __handleDownload is used to handle writing file
836 __handleDownload is used to handle writing file
836 """
837 """
837 self.file.write(block)
838 self.file.write(block)
838
839
839
840
840 def upload(self,filename,remotefolder=None):
841 def upload(self,filename,remotefolder=None):
841 """
842 """
842 upload is used to uploading local file to remote directory
843 upload is used to uploading local file to remote directory
843
844
844 Inputs:
845 Inputs:
845 filename - full path name of local file to store in remote directory
846 filename - full path name of local file to store in remote directory
846
847
847 remotefolder - remote directory
848 remotefolder - remote directory
848
849
849 Returns:
850 Returns:
850 self.status - 1 in error case else 0
851 self.status - 1 in error case else 0
851 """
852 """
852
853
853 if remotefolder == None:
854 if remotefolder == None:
854 remotefolder = self.remotefolder
855 remotefolder = self.remotefolder
855
856
856 self.status = 0
857 self.status = 0
857
858
858 try:
859 try:
859 self.ftp.cwd(remotefolder)
860 self.ftp.cwd(remotefolder)
860
861
861 self.file = open(filename, 'rb')
862 self.file = open(filename, 'rb')
862
863
863 (head, tail) = os.path.split(filename)
864 (head, tail) = os.path.split(filename)
864
865
865 command = "STOR " + tail
866 command = "STOR " + tail
866
867
867 print 'Uploading: ' + tail
868 print 'Uploading: ' + tail
868 self.ftp.storbinary(command, self.file)
869 self.ftp.storbinary(command, self.file)
869 print 'Upload Completed'
870 print 'Upload Completed'
870
871
871 except ftplib.all_errors:
872 except ftplib.all_errors:
872 print 'Error Uploading ' + tail
873 print 'Error Uploading ' + tail
873 self.status = 1
874 self.status = 1
874 return self.status
875 return self.status
875
876
876 self.file.close()
877 self.file.close()
877
878
878 #back to initial directory in __init__()
879 #back to initial directory in __init__()
879 self.ftp.cwd(self.remotefolder)
880 self.ftp.cwd(self.remotefolder)
880
881
881 return self.status
882 return self.status
882
883
883
884
884 def dir(self,remotefolder):
885 def dir(self,remotefolder):
885 """
886 """
886 dir is used to change working directory of remote server and get folder and file list
887 dir is used to change working directory of remote server and get folder and file list
887
888
888 Input:
889 Input:
889 remotefolder - current working directory
890 remotefolder - current working directory
890
891
891 Affects:
892 Affects:
892 self.fileList - file list of working directory
893 self.fileList - file list of working directory
893
894
894 Return:
895 Return:
895 infoList - list with filenames and size of file in bytes
896 infoList - list with filenames and size of file in bytes
896
897
897 self.folderList - folder list
898 self.folderList - folder list
898 """
899 """
899
900
900 self.remotefolder = remotefolder
901 self.remotefolder = remotefolder
901 print 'Change to ' + self.remotefolder
902 print 'Change to ' + self.remotefolder
902 try:
903 try:
903 self.ftp.cwd(remotefolder)
904 self.ftp.cwd(remotefolder)
904 except ftplib.all_errors:
905 except ftplib.all_errors:
905 print 'Error Change to ' + self.remotefolder
906 print 'Error Change to ' + self.remotefolder
906 infoList = None
907 infoList = None
907 self.folderList = None
908 self.folderList = None
908 return infoList,self.folderList
909 return infoList,self.folderList
909
910
910 self.dirList = []
911 self.dirList = []
911
912
912 try:
913 try:
913 self.dirList = self.ftp.nlst()
914 self.dirList = self.ftp.nlst()
914
915
915 except ftplib.error_perm, resp:
916 except ftplib.error_perm, resp:
916 if str(resp) == "550 No files found":
917 if str(resp) == "550 No files found":
917 print "no files in this directory"
918 print "no files in this directory"
918 infoList = None
919 infoList = None
919 self.folderList = None
920 self.folderList = None
920 return infoList,self.folderList
921 return infoList,self.folderList
921 except ftplib.all_errors:
922 except ftplib.all_errors:
922 print 'Error Displaying Dir-Files'
923 print 'Error Displaying Dir-Files'
923 infoList = None
924 infoList = None
924 self.folderList = None
925 self.folderList = None
925 return infoList,self.folderList
926 return infoList,self.folderList
926
927
927 infoList = []
928 infoList = []
928 self.fileList = []
929 self.fileList = []
929 self.folderList = []
930 self.folderList = []
930 for f in self.dirList:
931 for f in self.dirList:
931 name,ext = os.path.splitext(f)
932 name,ext = os.path.splitext(f)
932 if ext != '':
933 if ext != '':
933 self.fileList.append(f)
934 self.fileList.append(f)
934 value = (f,self.ftp.size(f))
935 value = (f,self.ftp.size(f))
935 infoList.append(value)
936 infoList.append(value)
936
937
937 if ext == '':
938 if ext == '':
938 self.folderList.append(f)
939 self.folderList.append(f)
939
940
940 return infoList,self.folderList
941 return infoList,self.folderList
941
942
942
943
943 def close(self):
944 def close(self):
944 """
945 """
945 close is used to close and end FTP connection
946 close is used to close and end FTP connection
946
947
947 Inputs: None
948 Inputs: None
948
949
949 Return: void
950 Return: void
950
951
951 """
952 """
952 self.ftp.close()
953 self.ftp.close()
953
954
954 class SendByFTP(Operation):
955 class SendByFTP(Operation):
955
956
956 def __init__(self):
957 def __init__(self):
957
958
958 self.status = 1
959 self.status = 1
959 self.counter = 0
960 self.counter = 0
960
961
961 def error_print(self, ValueError):
962 def error_print(self, ValueError):
962
963
963 print ValueError, 'Error FTP'
964 print ValueError, 'Error FTP'
964 print "don't worry the program is running..."
965 print "don't worry the program is running..."
965
966
966 def worker_ftp(self, server, username, password, remotefolder, filenameList):
967 def worker_ftp(self, server, username, password, remotefolder, filenameList):
967
968
968 self.ftpClientObj = FTP(server, username, password, remotefolder)
969 self.ftpClientObj = FTP(server, username, password, remotefolder)
969 for filename in filenameList:
970 for filename in filenameList:
970 self.ftpClientObj.upload(filename)
971 self.ftpClientObj.upload(filename)
971 self.ftpClientObj.close()
972 self.ftpClientObj.close()
972
973
973 def ftp_thread(self, server, username, password, remotefolder):
974 def ftp_thread(self, server, username, password, remotefolder):
974 if not(self.status):
975 if not(self.status):
975 return
976 return
976
977
977 import multiprocessing
978 import multiprocessing
978
979
979 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
980 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
980 p.start()
981 p.start()
981
982
982 p.join(3)
983 p.join(3)
983
984
984 if p.is_alive():
985 if p.is_alive():
985 p.terminate()
986 p.terminate()
986 p.join()
987 p.join()
987 print 'killing ftp process...'
988 print 'killing ftp process...'
988 self.status = 0
989 self.status = 0
989 return
990 return
990
991
991 self.status = 1
992 self.status = 1
992 return
993 return
993
994
994 def filterByExt(self, ext, localfolder):
995 def filterByExt(self, ext, localfolder):
995 fnameList = glob.glob1(localfolder,ext)
996 fnameList = glob.glob1(localfolder,ext)
996 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
997 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
997
998
998 if len(self.filenameList) == 0:
999 if len(self.filenameList) == 0:
999 self.status = 0
1000 self.status = 0
1000
1001
1001 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
1002 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
1002
1003
1003 self.counter += 1
1004 self.counter += 1
1004 if self.counter >= period:
1005 if self.counter >= period:
1005 self.filterByExt(ext, localfolder)
1006 self.filterByExt(ext, localfolder)
1006
1007
1007 self.ftp_thread(server, username, password, remotefolder)
1008 self.ftp_thread(server, username, password, remotefolder)
1008
1009
1009 self.counter = 0
1010 self.counter = 0
1010
1011
1011 self.status = 1
1012 self.status = 1
1012
1013
Show file before | Show file after | Hide comments
@@ -1,97 +1,99
1
1
2 import os, sys
2 import os, sys
3
3
4 path = os.path.split(os.getcwd())[0]
4 path = os.path.split(os.getcwd())[0]
5 path = os.path.split(path)[0]
5 path = os.path.split(path)[0]
6
6
7 sys.path.insert(0, path)
7 sys.path.insert(0, path)
8
8
9 from schainpy.controller import Project
9 from schainpy.controller import Project
10
10
11 controllerObj = Project()
11 controllerObj = Project()
12 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
12 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
13
13
14 #-------------------------------------- Setup -----------------------------------------
14 #-------------------------------------- Setup -----------------------------------------
15 #Verificar estas variables
15 #Verificar estas variables
16
16
17 #Path para los archivos
17 #Path para los archivos
18 # path = '/mnt/jars/2016_08/NOCHE'
18 # path = '/mnt/jars/2016_08/NOCHE'
19 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
19 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
20 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
20 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
21 path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
21 path = '/home/nanosat/data/jasmet'
22
22
23 #Path para los graficos
23 #Path para los graficos
24 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
24 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
25
25
26 #Path para los archivos HDF5 de meteoros
26 #Path para los archivos HDF5 de meteoros
27 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
27 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
28
28
29 #Fechas para busqueda de archivos
29 #Fechas para busqueda de archivos
30 startDate = '2016/08/29'
30 startDate = '2010/08/29'
31 endDate = '2016/09/11'
31 endDate = '2017/09/11'
32 #Horas para busqueda de archivos
32 #Horas para busqueda de archivos
33 startTime = '00:00:00'
33 startTime = '00:00:00'
34 endTime = '23:59:59'
34 endTime = '23:59:59'
35
35
36
36
37 #------------------------------ Voltage Reading Unit ----------------------------------
37 #------------------------------ Voltage Reading Unit ----------------------------------
38
38
39 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
39 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
40 path=path,
40 path=path,
41 startDate=startDate,
41 startDate=startDate,
42 endDate=endDate,
42 endDate=endDate,
43 startTime=startTime,
43 startTime=startTime,
44 endTime=endTime,
44 endTime=endTime,
45 online=0,
45 online=0,
46 delay=30,
46 delay=30,
47 walk=1,
47 walk=1,
48 getblock=1,
48 getblock=1,
49 blocktime=100)
49 blocktime=100)
50
50
51 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
51 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
52
52
53 #-------------------------- Voltage Processing Unit ------------------------------------
53 #-------------------------- Voltage Processing Unit ------------------------------------
54
54
55 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
55 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
56
56
57 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
57 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
58 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
58 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
59
59
60 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
60 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
61 opObj01.addParameter(name='frequency', value='30.e6', format='float')
61 opObj01.addParameter(name='frequency', value='30.e6', format='float')
62
62
63 opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
63 # opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
64 opObj01.addParameter(name='topLim', value='73', format='int')
64 # opObj01.addParameter(name='topLim', value='73', format='int')
65 opObj01.addParameter(name='botLim', value='71', format='int')
65 # opObj01.addParameter(name='botLim', value='71', format='int')
66
66
67 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
67 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
68
68
69 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
69 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
70 opObj03.addParameter(name='n', value='2', format='int')
70 opObj03.addParameter(name='n', value='2', format='int')
71
71
72 procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId())
73 opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
74 opObj11.addParameter(name='id', value='237', format='int')
75 opObj11.addParameter(name='xmin', value='9.0', format='float')
76 opObj11.addParameter(name='xmax', value='16.0', format='float')
77 opObj11.addParameter(name='zmin', value='15.0', format='float')
78 opObj11.addParameter(name='zmax', value='50.0', format='float')
79
72 #--------------------------- Parameters Processing Unit ------------------------------------
80 #--------------------------- Parameters Processing Unit ------------------------------------
73
81
74 procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
82 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
75 #
83 # #
76 opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
84 # opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
77 opObj10.addParameter(name='azimuth', value='45', format='float')
85 # opObj10.addParameter(name='azimuth', value='45', format='float')
78 opObj10.addParameter(name='hmin', value='60', format='float')
86 # opObj10.addParameter(name='hmin', value='60', format='float')
79 opObj10.addParameter(name='hmax', value='120', format='float')
87 # opObj10.addParameter(name='hmax', value='120', format='float')
80
88
81 opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
89 # opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
82 opObj12.addParameter(name='path', value=pathfile)
90 # opObj12.addParameter(name='path', value=pathfile)
83 opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
91 # opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
84 opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
92 # opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
85 opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
93 # opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
86 opObj12.addParameter(name='mode',value='2',format='int')
94 # opObj12.addParameter(name='mode',value='2',format='int')
87
95
88 #--------------------------------------------------------------------------------------------------
96 #--------------------------------------------------------------------------------------------------
89
90 print "Escribiendo el archivo XML"
91 controllerObj.writeXml("JASMET02.xml")
92 print "Leyendo el archivo XML"
93 controllerObj.readXml("JASMET02.xml")
94
95 controllerObj.createObjects()
97 controllerObj.createObjects()
96 controllerObj.connectObjects()
98 controllerObj.connectObjects()
97 controllerObj.run() No newline at end of file
99 controllerObj.run()
General Comments 0
You need to be logged in to leave comments. Login now