##// END OF EJS Templates
schain
RSS
casi merge con online_data_hour
José Chávez -
r1036:754fcef10e66 merge
parent child
Show More
Side by Side Unified
Context file:
r1036:754fcef10e66 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1813 +1,1976
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time, datetime
13 import time, datetime
14 import traceback
14 import traceback
15 import zmq
15 import zmq
16
16
17 try:
17 try:
18 from gevent import sleep
18 from gevent import sleep
19 except:
19 except:
20 from time import sleep
20 from time import sleep
21
21
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24
24
25 LOCALTIME = True
25 LOCALTIME = True
26
26
27 def isNumber(cad):
27 def isNumber(cad):
28 """
28 """
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30
30
31 Excepciones:
31 Excepciones:
32 Si un determinado string no puede ser convertido a numero
32 Si un determinado string no puede ser convertido a numero
33 Input:
33 Input:
34 str, string al cual se le analiza para determinar si convertible a un numero o no
34 str, string al cual se le analiza para determinar si convertible a un numero o no
35
35
36 Return:
36 Return:
37 True : si el string es uno numerico
37 True : si el string es uno numerico
38 False : no es un string numerico
38 False : no es un string numerico
39 """
39 """
40 try:
40 try:
41 float( cad )
41 float( cad )
42 return True
42 return True
43 except:
43 except:
44 return False
44 return False
45
45
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 """
47 """
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49
49
50 Inputs:
50 Inputs:
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52
52
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 segundos contados desde 01/01/1970.
54 segundos contados desde 01/01/1970.
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 segundos contados desde 01/01/1970.
56 segundos contados desde 01/01/1970.
57
57
58 Return:
58 Return:
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 fecha especificado, de lo contrario retorna False.
60 fecha especificado, de lo contrario retorna False.
61
61
62 Excepciones:
62 Excepciones:
63 Si el archivo no existe o no puede ser abierto
63 Si el archivo no existe o no puede ser abierto
64 Si la cabecera no puede ser leida.
64 Si la cabecera no puede ser leida.
65
65
66 """
66 """
67 basicHeaderObj = BasicHeader(LOCALTIME)
67 basicHeaderObj = BasicHeader(LOCALTIME)
68
68
69 try:
69 try:
70 fp = open(filename,'rb')
70 fp = open(filename,'rb')
71 except IOError:
71 except IOError:
72 print "The file %s can't be opened" %(filename)
72 print "The file %s can't be opened" %(filename)
73 return 0
73 return 0
74
74
75 sts = basicHeaderObj.read(fp)
75 sts = basicHeaderObj.read(fp)
76 fp.close()
76 fp.close()
77
77
78 if not(sts):
78 if not(sts):
79 print "Skipping the file %s because it has not a valid header" %(filename)
79 print "Skipping the file %s because it has not a valid header" %(filename)
80 return 0
80 return 0
81
81
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 return 0
83 return 0
84
84
85 return 1
85 return 1
86
86
87 def isTimeInRange(thisTime, startTime, endTime):
87 def isTimeInRange(thisTime, startTime, endTime):
88
88
89 if endTime >= startTime:
89 if endTime >= startTime:
90 if (thisTime < startTime) or (thisTime > endTime):
90 if (thisTime < startTime) or (thisTime > endTime):
91 return 0
91 return 0
92
92
93 return 1
93 return 1
94 else:
94 else:
95 if (thisTime < startTime) and (thisTime > endTime):
95 if (thisTime < startTime) and (thisTime > endTime):
96 return 0
96 return 0
97
97
98 return 1
98 return 1
99
99
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 """
101 """
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103
103
104 Inputs:
104 Inputs:
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106
106
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108
108
109 endDate : fecha final del rango seleccionado en formato datetime.date
109 endDate : fecha final del rango seleccionado en formato datetime.date
110
110
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112
112
113 endTime : tiempo final del rango seleccionado en formato datetime.time
113 endTime : tiempo final del rango seleccionado en formato datetime.time
114
114
115 Return:
115 Return:
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 fecha especificado, de lo contrario retorna False.
117 fecha especificado, de lo contrario retorna False.
118
118
119 Excepciones:
119 Excepciones:
120 Si el archivo no existe o no puede ser abierto
120 Si el archivo no existe o no puede ser abierto
121 Si la cabecera no puede ser leida.
121 Si la cabecera no puede ser leida.
122
122
123 """
123 """
124
124
125
125
126 try:
126 try:
127 fp = open(filename,'rb')
127 fp = open(filename,'rb')
128 except IOError:
128 except IOError:
129 print "The file %s can't be opened" %(filename)
129 print "The file %s can't be opened" %(filename)
130 return None
130 return None
131
131
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 systemHeaderObj = SystemHeader()
133 systemHeaderObj = SystemHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
135 processingHeaderObj = ProcessingHeader()
135 processingHeaderObj = ProcessingHeader()
136
136
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138
138
139 sts = firstBasicHeaderObj.read(fp)
139 sts = firstBasicHeaderObj.read(fp)
140
140
141 if not(sts):
141 if not(sts):
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 return None
143 return None
144
144
145 if not systemHeaderObj.read(fp):
145 if not systemHeaderObj.read(fp):
146 return None
146 return None
147
147
148 if not radarControllerHeaderObj.read(fp):
148 if not radarControllerHeaderObj.read(fp):
149 return None
149 return None
150
150
151 if not processingHeaderObj.read(fp):
151 if not processingHeaderObj.read(fp):
152 return None
152 return None
153
153
154 filesize = os.path.getsize(filename)
154 filesize = os.path.getsize(filename)
155
155
156 offset = processingHeaderObj.blockSize + 24 #header size
156 offset = processingHeaderObj.blockSize + 24 #header size
157
157
158 if filesize <= offset:
158 if filesize <= offset:
159 print "[Reading] %s: This file has not enough data" %filename
159 print "[Reading] %s: This file has not enough data" %filename
160 return None
160 return None
161
161
162 fp.seek(-offset, 2)
162 fp.seek(-offset, 2)
163
163
164 sts = lastBasicHeaderObj.read(fp)
164 sts = lastBasicHeaderObj.read(fp)
165
165
166 fp.close()
166 fp.close()
167
167
168 thisDatetime = lastBasicHeaderObj.datatime
168 thisDatetime = lastBasicHeaderObj.datatime
169 thisTime_last_block = thisDatetime.time()
169 thisTime_last_block = thisDatetime.time()
170
170
171 thisDatetime = firstBasicHeaderObj.datatime
171 thisDatetime = firstBasicHeaderObj.datatime
172 thisDate = thisDatetime.date()
172 thisDate = thisDatetime.date()
173 thisTime_first_block = thisDatetime.time()
173 thisTime_first_block = thisDatetime.time()
174
174
175 #General case
175 #General case
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 #-----------o----------------------------o-----------
177 #-----------o----------------------------o-----------
178 # startTime endTime
178 # startTime endTime
179
179
180 if endTime >= startTime:
180 if endTime >= startTime:
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 return None
182 return None
183
183
184 return thisDatetime
184 return thisDatetime
185
185
186 #If endTime < startTime then endTime belongs to the next day
186 #If endTime < startTime then endTime belongs to the next day
187
187
188
188
189 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 #-----------o----------------------------o-----------
190 #-----------o----------------------------o-----------
191 # endTime startTime
191 # endTime startTime
192
192
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 return None
194 return None
195
195
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 return None
197 return None
198
198
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 return None
200 return None
201
201
202 return thisDatetime
202 return thisDatetime
203
203
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 """
205 """
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207
207
208 Inputs:
208 Inputs:
209 folder : nombre completo del directorio.
209 folder : nombre completo del directorio.
210 Su formato deberia ser "/path_root/?YYYYDDD"
210 Su formato deberia ser "/path_root/?YYYYDDD"
211
211
212 siendo:
212 siendo:
213 YYYY : Anio (ejemplo 2015)
213 YYYY : Anio (ejemplo 2015)
214 DDD : Dia del anio (ejemplo 305)
214 DDD : Dia del anio (ejemplo 305)
215
215
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217
217
218 endDate : fecha final del rango seleccionado en formato datetime.date
218 endDate : fecha final del rango seleccionado en formato datetime.date
219
219
220 Return:
220 Return:
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 fecha especificado, de lo contrario retorna False.
222 fecha especificado, de lo contrario retorna False.
223 Excepciones:
223 Excepciones:
224 Si el directorio no tiene el formato adecuado
224 Si el directorio no tiene el formato adecuado
225 """
225 """
226
226
227 basename = os.path.basename(folder)
227 basename = os.path.basename(folder)
228
228
229 if not isRadarFolder(basename):
229 if not isRadarFolder(basename):
230 print "The folder %s has not the rigth format" %folder
230 print "The folder %s has not the rigth format" %folder
231 return 0
231 return 0
232
232
233 if startDate and endDate:
233 if startDate and endDate:
234 thisDate = getDateFromRadarFolder(basename)
234 thisDate = getDateFromRadarFolder(basename)
235
235
236 if thisDate < startDate:
236 if thisDate < startDate:
237 return 0
237 return 0
238
238
239 if thisDate > endDate:
239 if thisDate > endDate:
240 return 0
240 return 0
241
241
242 return 1
242 return 1
243
243
244 def isFileInDateRange(filename, startDate=None, endDate=None):
244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 """
245 """
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247
247
248 Inputs:
248 Inputs:
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250
250
251 Su formato deberia ser "?YYYYDDDsss"
251 Su formato deberia ser "?YYYYDDDsss"
252
252
253 siendo:
253 siendo:
254 YYYY : Anio (ejemplo 2015)
254 YYYY : Anio (ejemplo 2015)
255 DDD : Dia del anio (ejemplo 305)
255 DDD : Dia del anio (ejemplo 305)
256 sss : set
256 sss : set
257
257
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259
259
260 endDate : fecha final del rango seleccionado en formato datetime.date
260 endDate : fecha final del rango seleccionado en formato datetime.date
261
261
262 Return:
262 Return:
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 fecha especificado, de lo contrario retorna False.
264 fecha especificado, de lo contrario retorna False.
265 Excepciones:
265 Excepciones:
266 Si el archivo no tiene el formato adecuado
266 Si el archivo no tiene el formato adecuado
267 """
267 """
268
268
269 basename = os.path.basename(filename)
269 basename = os.path.basename(filename)
270
270
271 if not isRadarFile(basename):
271 if not isRadarFile(basename):
272 print "The filename %s has not the rigth format" %filename
272 print "The filename %s has not the rigth format" %filename
273 return 0
273 return 0
274
274
275 if startDate and endDate:
275 if startDate and endDate:
276 thisDate = getDateFromRadarFile(basename)
276 thisDate = getDateFromRadarFile(basename)
277
277
278 if thisDate < startDate:
278 if thisDate < startDate:
279 return 0
279 return 0
280
280
281 if thisDate > endDate:
281 if thisDate > endDate:
282 return 0
282 return 0
283
283
284 return 1
284 return 1
285
285
286 def getFileFromSet(path, ext, set):
286 def getFileFromSet(path, ext, set):
287 validFilelist = []
287 validFilelist = []
288 fileList = os.listdir(path)
288 fileList = os.listdir(path)
289
289
290 # 0 1234 567 89A BCDE
290 # 0 1234 567 89A BCDE
291 # H YYYY DDD SSS .ext
291 # H YYYY DDD SSS .ext
292
292
293 for thisFile in fileList:
293 for thisFile in fileList:
294 try:
294 try:
295 year = int(thisFile[1:5])
295 year = int(thisFile[1:5])
296 doy = int(thisFile[5:8])
296 doy = int(thisFile[5:8])
297 except:
297 except:
298 continue
298 continue
299
299
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 continue
301 continue
302
302
303 validFilelist.append(thisFile)
303 validFilelist.append(thisFile)
304
304
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306
306
307 if len(myfile)!= 0:
307 if len(myfile)!= 0:
308 return myfile[0]
308 return myfile[0]
309 else:
309 else:
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 print 'the filename %s does not exist'%filename
311 print 'the filename %s does not exist'%filename
312 print '...going to the last file: '
312 print '...going to the last file: '
313
313
314 if validFilelist:
314 if validFilelist:
315 validFilelist = sorted( validFilelist, key=str.lower )
315 validFilelist = sorted( validFilelist, key=str.lower )
316 return validFilelist[-1]
316 return validFilelist[-1]
317
317
318 return None
318 return None
319
319
320 def getlastFileFromPath(path, ext):
320 def getlastFileFromPath(path, ext):
321 """
321 """
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324
324
325 Input:
325 Input:
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 ext : extension de los files contenidos en una carpeta
327 ext : extension de los files contenidos en una carpeta
328
328
329 Return:
329 Return:
330 El ultimo file de una determinada carpeta, no se considera el path.
330 El ultimo file de una determinada carpeta, no se considera el path.
331 """
331 """
332 validFilelist = []
332 validFilelist = []
333 fileList = os.listdir(path)
333 fileList = os.listdir(path)
334
334
335 # 0 1234 567 89A BCDE
335 # 0 1234 567 89A BCDE
336 # H YYYY DDD SSS .ext
336 # H YYYY DDD SSS .ext
337
337
338 for thisFile in fileList:
338 for thisFile in fileList:
339
339
340 year = thisFile[1:5]
340 year = thisFile[1:5]
341 if not isNumber(year):
341 if not isNumber(year):
342 continue
342 continue
343
343
344 doy = thisFile[5:8]
344 doy = thisFile[5:8]
345 if not isNumber(doy):
345 if not isNumber(doy):
346 continue
346 continue
347
347
348 year = int(year)
348 year = int(year)
349 doy = int(doy)
349 doy = int(doy)
350
350
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 continue
352 continue
353
353
354 validFilelist.append(thisFile)
354 validFilelist.append(thisFile)
355
355
356 if validFilelist:
356 if validFilelist:
357 validFilelist = sorted( validFilelist, key=str.lower )
357 validFilelist = sorted( validFilelist, key=str.lower )
358 return validFilelist[-1]
358 return validFilelist[-1]
359
359
360 return None
360 return None
361
361
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 """
363 """
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 el path exacto de un determinado file.
366 el path exacto de un determinado file.
367
367
368 Example :
368 Example :
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370
370
371 Entonces la funcion prueba con las siguientes combinaciones
371 Entonces la funcion prueba con las siguientes combinaciones
372 .../.../y2009307367.ext
372 .../.../y2009307367.ext
373 .../.../Y2009307367.ext
373 .../.../Y2009307367.ext
374 .../.../x2009307/y2009307367.ext
374 .../.../x2009307/y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
376 .../.../X2009307/y2009307367.ext
376 .../.../X2009307/y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379
379
380 Return:
380 Return:
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 para el filename
383 para el filename
384 """
384 """
385 fullfilename = None
385 fullfilename = None
386 find_flag = False
386 find_flag = False
387 filename = None
387 filename = None
388
388
389 prefixDirList = [None,'d','D']
389 prefixDirList = [None,'d','D']
390 if ext.lower() == ".r": #voltage
390 if ext.lower() == ".r": #voltage
391 prefixFileList = ['d','D']
391 prefixFileList = ['d','D']
392 elif ext.lower() == ".pdata": #spectra
392 elif ext.lower() == ".pdata": #spectra
393 prefixFileList = ['p','P']
393 prefixFileList = ['p','P']
394 else:
394 else:
395 return None, filename
395 return None, filename
396
396
397 #barrido por las combinaciones posibles
397 #barrido por las combinaciones posibles
398 for prefixDir in prefixDirList:
398 for prefixDir in prefixDirList:
399 thispath = path
399 thispath = path
400 if prefixDir != None:
400 if prefixDir != None:
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 if foldercounter == 0:
402 if foldercounter == 0:
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 else:
404 else:
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409
409
410 if os.path.exists( fullfilename ): #verifico que exista
410 if os.path.exists( fullfilename ): #verifico que exista
411 find_flag = True
411 find_flag = True
412 break
412 break
413 if find_flag:
413 if find_flag:
414 break
414 break
415
415
416 if not(find_flag):
416 if not(find_flag):
417 return None, filename
417 return None, filename
418
418
419 return fullfilename, filename
419 return fullfilename, filename
420
420
421 def isRadarFolder(folder):
421 def isRadarFolder(folder):
422 try:
422 try:
423 year = int(folder[1:5])
423 year = int(folder[1:5])
424 doy = int(folder[5:8])
424 doy = int(folder[5:8])
425 except:
425 except:
426 return 0
426 return 0
427
427
428 return 1
428 return 1
429
429
430 def isRadarFile(file):
430 def isRadarFile(file):
431 try:
431 try:
432 year = int(file[1:5])
432 year = int(file[1:5])
433 doy = int(file[5:8])
433 doy = int(file[5:8])
434 set = int(file[8:11])
434 set = int(file[8:11])
435 except:
435 except:
436 return 0
436 return 0
437
437
438 return 1
438 return 1
439
439
440 def getDateFromRadarFile(file):
440 def getDateFromRadarFile(file):
441 try:
441 try:
442 year = int(file[1:5])
442 year = int(file[1:5])
443 doy = int(file[5:8])
443 doy = int(file[5:8])
444 set = int(file[8:11])
444 set = int(file[8:11])
445 except:
445 except:
446 return None
446 return None
447
447
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 return thisDate
449 return thisDate
450
450
451 def getDateFromRadarFolder(folder):
451 def getDateFromRadarFolder(folder):
452 try:
452 try:
453 year = int(folder[1:5])
453 year = int(folder[1:5])
454 doy = int(folder[5:8])
454 doy = int(folder[5:8])
455 except:
455 except:
456 return None
456 return None
457
457
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 return thisDate
459 return thisDate
460
460
461 class JRODataIO:
461 class JRODataIO:
462
462
463 c = 3E8
463 c = 3E8
464
464
465 isConfig = False
465 isConfig = False
466
466
467 basicHeaderObj = None
467 basicHeaderObj = None
468
468
469 systemHeaderObj = None
469 systemHeaderObj = None
470
470
471 radarControllerHeaderObj = None
471 radarControllerHeaderObj = None
472
472
473 processingHeaderObj = None
473 processingHeaderObj = None
474
474
475 dtype = None
475 dtype = None
476
476
477 pathList = []
477 pathList = []
478
478
479 filenameList = []
479 filenameList = []
480
480
481 filename = None
481 filename = None
482
482
483 ext = None
483 ext = None
484
484
485 flagIsNewFile = 1
485 flagIsNewFile = 1
486
486
487 flagDiscontinuousBlock = 0
487 flagDiscontinuousBlock = 0
488
488
489 flagIsNewBlock = 0
489 flagIsNewBlock = 0
490
490
491 fp = None
491 fp = None
492
492
493 firstHeaderSize = 0
493 firstHeaderSize = 0
494
494
495 basicHeaderSize = 24
495 basicHeaderSize = 24
496
496
497 versionFile = 1103
497 versionFile = 1103
498
498
499 fileSize = None
499 fileSize = None
500
500
501 # ippSeconds = None
501 # ippSeconds = None
502
502
503 fileSizeByHeader = None
503 fileSizeByHeader = None
504
504
505 fileIndex = None
505 fileIndex = None
506
506
507 profileIndex = None
507 profileIndex = None
508
508
509 blockIndex = None
509 blockIndex = None
510
510
511 nTotalBlocks = None
511 nTotalBlocks = None
512
512
513 maxTimeStep = 30
513 maxTimeStep = 30
514
514
515 lastUTTime = None
515 lastUTTime = None
516
516
517 datablock = None
517 datablock = None
518
518
519 dataOut = None
519 dataOut = None
520
520
521 blocksize = None
521 blocksize = None
522
522
523 getByBlock = False
523 getByBlock = False
524
524
525 def __init__(self):
525 def __init__(self):
526
526
527 raise NotImplementedError
527 raise NotImplementedError
528
528
529 def run(self):
529 def run(self):
530
530
531 raise NotImplementedError
531 raise NotImplementedError
532
532
533 def getDtypeWidth(self):
533 def getDtypeWidth(self):
534
534
535 dtype_index = get_dtype_index(self.dtype)
535 dtype_index = get_dtype_index(self.dtype)
536 dtype_width = get_dtype_width(dtype_index)
536 dtype_width = get_dtype_width(dtype_index)
537
537
538 return dtype_width
538 return dtype_width
539
539
540 def getAllowedArgs(self):
540 def getAllowedArgs(self):
541 return inspect.getargspec(self.run).args
541 return inspect.getargspec(self.run).args
542
542
543 class JRODataReader(JRODataIO):
543 class JRODataReader(JRODataIO):
544
544
545
545 firstTime = True
546 online = 0
546 online = 0
547
547
548 realtime = 0
548 realtime = 0
549
549
550 nReadBlocks = 0
550 nReadBlocks = 0
551
551
552 delay = 10 #number of seconds waiting a new file
552 delay = 10 #number of seconds waiting a new file
553
553
554 nTries = 3 #quantity tries
554 nTries = 3 #quantity tries
555
555
556 nFiles = 3 #number of files for searching
556 nFiles = 3 #number of files for searching
557
557
558 path = None
558 path = None
559
559
560 foldercounter = 0
560 foldercounter = 0
561
561
562 flagNoMoreFiles = 0
562 flagNoMoreFiles = 0
563
563
564 datetimeList = []
564 datetimeList = []
565
565
566 __isFirstTimeOnline = 1
566 __isFirstTimeOnline = 1
567
567
568 __printInfo = True
568 __printInfo = True
569
569
570 profileIndex = None
570 profileIndex = None
571
571
572 nTxs = 1
572 nTxs = 1
573
573
574 txIndex = None
574 txIndex = None
575
575
576 #Added--------------------
576 #Added--------------------
577
577
578 selBlocksize = None
578 selBlocksize = None
579
579
580 selBlocktime = None
580 selBlocktime = None
581
581
582
582 onlineWithDate = False
583 def __init__(self):
583 def __init__(self):
584
584
585 """
585 """
586 This class is used to find data files
586 This class is used to find data files
587
587
588 Example:
588 Example:
589 reader = JRODataReader()
589 reader = JRODataReader()
590 fileList = reader.findDataFiles()
590 fileList = reader.findDataFiles()
591
591
592 """
592 """
593 pass
593 pass
594
594
595
595
596 def createObjByDefault(self):
596 def createObjByDefault(self):
597 """
597 """
598
598
599 """
599 """
600 raise NotImplementedError
600 raise NotImplementedError
601
601
602 def getBlockDimension(self):
602 def getBlockDimension(self):
603
603
604 raise NotImplementedError
604 raise NotImplementedError
605
605
606 def __searchFilesOffLine(self,
606 def __searchFilesOffLine(self,
607 path,
607 path,
608 startDate=None,
608 startDate=None,
609 endDate=None,
609 endDate=None,
610 startTime=datetime.time(0,0,0),
610 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
611 endTime=datetime.time(23,59,59),
612 set=None,
612 set=None,
613 expLabel='',
613 expLabel='',
614 ext='.r',
614 ext='.r',
615 queue=None,
615 queue=None,
616 cursor=None,
616 cursor=None,
617 skip=None,
617 skip=None,
618 walk=True):
618 walk=True):
619
620 self.filenameList = []
619 self.filenameList = []
621 self.datetimeList = []
620 self.datetimeList = []
622
621
623 pathList = []
622 pathList = []
624
623
625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
626
625
627 if dateList == []:
626 if dateList == []:
628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
629 return None, None
628 return None, None
630
629
631 if len(dateList) > 1:
630 if len(dateList) > 1:
632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
633 else:
632 else:
634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
635
634
636 filenameList = []
635 filenameList = []
637 datetimeList = []
636 datetimeList = []
638
637
639 for thisPath in pathList:
638 for thisPath in pathList:
640 # thisPath = pathList[pathDict[file]]
639 # thisPath = pathList[pathDict[file]]
641
640
642 fileList = glob.glob1(thisPath, "*%s" %ext)
641 fileList = glob.glob1(thisPath, "*%s" %ext)
643 fileList.sort()
642 fileList.sort()
644
643
645 skippedFileList = []
644 skippedFileList = []
646
645
647 if cursor is not None and skip is not None:
646 if cursor is not None and skip is not None:
648 # if cursor*skip > len(fileList):
647 # if cursor*skip > len(fileList):
649 if skip == 0:
648 if skip == 0:
650 if queue is not None:
649 if queue is not None:
651 queue.put(len(fileList))
650 queue.put(len(fileList))
652 skippedFileList = []
651 skippedFileList = []
653 else:
652 else:
654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
655
654
656 else:
655 else:
657 skippedFileList = fileList
656 skippedFileList = fileList
658
657
659 for file in skippedFileList:
658 for file in skippedFileList:
660
659
661 filename = os.path.join(thisPath,file)
660 filename = os.path.join(thisPath,file)
662
661
663 if not isFileInDateRange(filename, startDate, endDate):
662 if not isFileInDateRange(filename, startDate, endDate):
664 continue
663 continue
665
664
666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
667
666
668 if not(thisDatetime):
667 if not(thisDatetime):
669 continue
668 continue
670
669
671 filenameList.append(filename)
670 filenameList.append(filename)
672 datetimeList.append(thisDatetime)
671 datetimeList.append(thisDatetime)
673
672
674 if not(filenameList):
673 if not(filenameList):
675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
676 return None, None
675 return None, None
677
676
678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
679 print
678 print
680
679
681 for i in range(len(filenameList)):
680 for i in range(len(filenameList)):
682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
683
682
684 self.filenameList = filenameList
683 self.filenameList = filenameList
685 self.datetimeList = datetimeList
684 self.datetimeList = datetimeList
686
687 return pathList, filenameList
685 return pathList, filenameList
688
686
689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
690
688
691 """
689 """
692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
693 devuelve el archivo encontrado ademas de otros datos.
691 devuelve el archivo encontrado ademas de otros datos.
694
692
695 Input:
693 Input:
696 path : carpeta donde estan contenidos los files que contiene data
694 path : carpeta donde estan contenidos los files que contiene data
697
695
698 expLabel : Nombre del subexperimento (subfolder)
696 expLabel : Nombre del subexperimento (subfolder)
699
697
700 ext : extension de los files
698 ext : extension de los files
701
699
702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
703
701
704 Return:
702 Return:
705 directory : eL directorio donde esta el file encontrado
703 directory : eL directorio donde esta el file encontrado
706 filename : el ultimo file de una determinada carpeta
704 filename : el ultimo file de una determinada carpeta
707 year : el anho
705 year : el anho
708 doy : el numero de dia del anho
706 doy : el numero de dia del anho
709 set : el set del archivo
707 set : el set del archivo
710
708
711
709
712 """
710 """
711 pathList = None
712 filenameList = None
713 if not os.path.isdir(path):
713 if not os.path.isdir(path):
714 return None, None, None, None, None, None
714 return None, None, None, None, None, None
715
715
716 dirList = []
716 dirList = []
717
717
718 if not walk:
718 if not walk:
719 fullpath = path
719 fullpath = path
720 foldercounter = 0
720 foldercounter = 0
721 else:
721 else:
722 #Filtra solo los directorios
722 # Filtra solo los directorios
723 for thisPath in os.listdir(path):
723 for thisPath in os.listdir(path):
724 if not os.path.isdir(os.path.join(path,thisPath)):
724 if not os.path.isdir(os.path.join(path,thisPath)):
725 continue
725 continue
726 if not isRadarFolder(thisPath):
726 if not isRadarFolder(thisPath):
727 continue
727 continue
728
728
729 dirList.append(thisPath)
729 dirList.append(thisPath)
730
730
731 if not(dirList):
731 if not(dirList):
732 return None, None, None, None, None, None
732 return None, None, None, None, None, None
733
733
734 dirList = sorted( dirList, key=str.lower )
734 dirList = sorted( dirList, key=str.lower )
735
735
736 doypath = dirList[-1]
736 doypath = dirList[-1]
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 fullpath = os.path.join(path, doypath, expLabel)
738 fullpath = os.path.join(path, doypath, expLabel)
739
739
740
740
741 print "[Reading] %s folder was found: " %(fullpath )
741 print "[Reading] %s folder was found: " %(fullpath )
742
742
743 if set == None:
743 if set == None:
744 filename = getlastFileFromPath(fullpath, ext)
744 filename = getlastFileFromPath(fullpath, ext)
745 else:
745 else:
746 filename = getFileFromSet(fullpath, ext, set)
746 filename = getFileFromSet(fullpath, ext, set)
747
747
748 if not(filename):
748 if not(filename):
749 return None, None, None, None, None, None
749 return None, None, None, None, None, None
750
750
751 print "[Reading] %s file was found" %(filename)
751 print "[Reading] %s file was found" %(filename)
752
752
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 return None, None, None, None, None, None
754 return None, None, None, None, None, None
755
755
756 year = int( filename[1:5] )
756 year = int( filename[1:5] )
757 doy = int( filename[5:8] )
757 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
758 set = int( filename[8:11] )
759
759
760 return fullpath, foldercounter, filename, year, doy, set
760 return fullpath, foldercounter, filename, year, doy, set
761
761
762 def __setNextFileOffline(self):
762 def __setNextFileOffline(self):
763
763
764 idFile = self.fileIndex
764 idFile = self.fileIndex
765
765
766 while (True):
766 while (True):
767 idFile += 1
767 idFile += 1
768 if not(idFile < len(self.filenameList)):
768 if not(idFile < len(self.filenameList)):
769 self.flagNoMoreFiles = 1
769 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
770 # print "[Reading] No more Files"
771 return 0
771 return 0
772
772
773 filename = self.filenameList[idFile]
773 filename = self.filenameList[idFile]
774
774
775 if not(self.__verifyFile(filename)):
775 if not(self.__verifyFile(filename)):
776 continue
776 continue
777
777
778 fileSize = os.path.getsize(filename)
778 fileSize = os.path.getsize(filename)
779 fp = open(filename,'rb')
779 fp = open(filename,'rb')
780 break
780 break
781
781
782 self.flagIsNewFile = 1
782 self.flagIsNewFile = 1
783 self.fileIndex = idFile
783 self.fileIndex = idFile
784 self.filename = filename
784 self.filename = filename
785 self.fileSize = fileSize
785 self.fileSize = fileSize
786 self.fp = fp
786 self.fp = fp
787
787
788 # print "[Reading] Setting the file: %s"%self.filename
788 #print "[Reading] Setting the file: %s"%self.filename
789
789
790 return 1
790 return 1
791
791
792 def __setNextFileOnline(self):
792 def __setNextFileOnline(self):
793 """
793 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
796 siguientes.
797
797
798 Affected:
798 Affected:
799 self.flagIsNewFile
799 self.flagIsNewFile
800 self.filename
800 self.filename
801 self.fileSize
801 self.fileSize
802 self.fp
802 self.fp
803 self.set
803 self.set
804 self.flagNoMoreFiles
804 self.flagNoMoreFiles
805
805
806 Return:
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
809
810 Excepciones:
810 Excepciones:
811 Si un determinado file no puede ser abierto
811 Si un determinado file no puede ser abierto
812 """
812 """
813
813 nFiles = 0
814 nFiles = 0
814 fileOk_flag = False
815 fileOk_flag = False
815 firstTime_flag = True
816 firstTime_flag = True
816
817
817 self.set += 1
818 self.set += 1
818
819
819 if self.set > 999:
820 if self.set > 999:
820 self.set = 0
821 self.set = 0
821 self.foldercounter += 1
822 self.foldercounter += 1
822
823
823 #busca el 1er file disponible
824 #busca el 1er file disponible
824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 if fullfilename:
826 if fullfilename:
826 if self.__verifyFile(fullfilename, False):
827 if self.__verifyFile(fullfilename, False):
827 fileOk_flag = True
828 fileOk_flag = True
828
829
829 #si no encuentra un file entonces espera y vuelve a buscar
830 #si no encuentra un file entonces espera y vuelve a buscar
830 if not(fileOk_flag):
831 if not(fileOk_flag):
831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832
833
833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 tries = self.nTries
835 tries = self.nTries
835 else:
836 else:
836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837
838
838 for nTries in range( tries ):
839 for nTries in range( tries ):
839 if firstTime_flag:
840 if firstTime_flag:
840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 sleep( self.delay )
842 sleep( self.delay )
842 else:
843 else:
843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844
845
845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 if fullfilename:
847 if fullfilename:
847 if self.__verifyFile(fullfilename):
848 if self.__verifyFile(fullfilename):
848 fileOk_flag = True
849 fileOk_flag = True
849 break
850 break
850
851
851 if fileOk_flag:
852 if fileOk_flag:
852 break
853 break
853
854
854 firstTime_flag = False
855 firstTime_flag = False
855
856
856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 self.set += 1
858 self.set += 1
858
859
859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 self.set = 0
861 self.set = 0
861 self.doy += 1
862 self.doy += 1
862 self.foldercounter = 0
863 self.foldercounter = 0
863
864
864 if fileOk_flag:
865 if fileOk_flag:
865 self.fileSize = os.path.getsize( fullfilename )
866 self.fileSize = os.path.getsize( fullfilename )
866 self.filename = fullfilename
867 self.filename = fullfilename
867 self.flagIsNewFile = 1
868 self.flagIsNewFile = 1
868 if self.fp != None: self.fp.close()
869 if self.fp != None: self.fp.close()
869 self.fp = open(fullfilename, 'rb')
870 self.fp = open(fullfilename, 'rb')
870 self.flagNoMoreFiles = 0
871 self.flagNoMoreFiles = 0
871 # print '[Reading] Setting the file: %s' % fullfilename
872 # print '[Reading] Setting the file: %s' % fullfilename
872 else:
873 else:
873 self.fileSize = 0
874 self.fileSize = 0
874 self.filename = None
875 self.filename = None
875 self.flagIsNewFile = 0
876 self.flagIsNewFile = 0
876 self.fp = None
877 self.fp = None
877 self.flagNoMoreFiles = 1
878 self.flagNoMoreFiles = 1
878 # print '[Reading] No more files to read'
879 # print '[Reading] No more files to read'
879
880
880 return fileOk_flag
881 return fileOk_flag
881
882
882 def setNextFile(self):
883 def setNextFile(self):
883 if self.fp != None:
884 if self.fp != None:
884 self.fp.close()
885 self.fp.close()
885
886 if self.online:
886 if self.online:
887 newFile = self.__setNextFileOnline()
887 newFile = self.__setNextFileOnline()
888 else:
888 else:
889 newFile = self.__setNextFileOffline()
889 newFile = self.__setNextFileOffline()
890
891 if not(newFile):
890 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
892 print '[Reading] No more files to read'
914 print '[Reading] No more files to read'
893 return 0
915 return 0
894
916
895 if self.verbose:
917 if self.verbose:
896 print '[Reading] Setting the file: %s' % self.filename
918 print '[Reading] Setting the file: %s' % self.filename
897
919
898 self.__readFirstHeader()
920 self.__readFirstHeader()
899 self.nReadBlocks = 0
921 self.nReadBlocks = 0
900 return 1
922 return 1
901
923
902 def __waitNewBlock(self):
924 def __waitNewBlock(self):
903 """
925 """
904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
905
927
906 Si el modo de lectura es OffLine siempre retorn 0
928 Si el modo de lectura es OffLine siempre retorn 0
907 """
929 """
908 if not self.online:
930 if not self.online:
909 return 0
931 return 0
910
932
911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
912 return 0
934 return 0
913
935
914 currentPointer = self.fp.tell()
936 currentPointer = self.fp.tell()
915
937
916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917
939
918 for nTries in range( self.nTries ):
940 for nTries in range( self.nTries ):
919
941
920 self.fp.close()
942 self.fp.close()
921 self.fp = open( self.filename, 'rb' )
943 self.fp = open( self.filename, 'rb' )
922 self.fp.seek( currentPointer )
944 self.fp.seek( currentPointer )
923
945
924 self.fileSize = os.path.getsize( self.filename )
946 self.fileSize = os.path.getsize( self.filename )
925 currentSize = self.fileSize - currentPointer
947 currentSize = self.fileSize - currentPointer
926
948
927 if ( currentSize >= neededSize ):
949 if ( currentSize >= neededSize ):
928 self.basicHeaderObj.read(self.fp)
950 self.basicHeaderObj.read(self.fp)
929 return 1
951 return 1
930
952
931 if self.fileSize == self.fileSizeByHeader:
953 if self.fileSize == self.fileSizeByHeader:
932 # self.flagEoF = True
954 # self.flagEoF = True
933 return 0
955 return 0
934
956
935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
936 sleep( self.delay )
958 sleep( self.delay )
937
959
938
960
939 return 0
961 return 0
940
962
941 def waitDataBlock(self,pointer_location):
963 def waitDataBlock(self,pointer_location):
942
964
943 currentPointer = pointer_location
965 currentPointer = pointer_location
944
966
945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
946
968
947 for nTries in range( self.nTries ):
969 for nTries in range( self.nTries ):
948 self.fp.close()
970 self.fp.close()
949 self.fp = open( self.filename, 'rb' )
971 self.fp = open( self.filename, 'rb' )
950 self.fp.seek( currentPointer )
972 self.fp.seek( currentPointer )
951
973
952 self.fileSize = os.path.getsize( self.filename )
974 self.fileSize = os.path.getsize( self.filename )
953 currentSize = self.fileSize - currentPointer
975 currentSize = self.fileSize - currentPointer
954
976
955 if ( currentSize >= neededSize ):
977 if ( currentSize >= neededSize ):
956 return 1
978 return 1
957
979
958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
959 sleep( self.delay )
981 sleep( self.delay )
960
982
961 return 0
983 return 0
962
984
963 def __jumpToLastBlock(self):
985 def __jumpToLastBlock(self):
964
986
965 if not(self.__isFirstTimeOnline):
987 if not(self.__isFirstTimeOnline):
966 return
988 return
967
989
968 csize = self.fileSize - self.fp.tell()
990 csize = self.fileSize - self.fp.tell()
969 blocksize = self.processingHeaderObj.blockSize
991 blocksize = self.processingHeaderObj.blockSize
970
992
971 #salta el primer bloque de datos
993 #salta el primer bloque de datos
972 if csize > self.processingHeaderObj.blockSize:
994 if csize > self.processingHeaderObj.blockSize:
973 self.fp.seek(self.fp.tell() + blocksize)
995 self.fp.seek(self.fp.tell() + blocksize)
974 else:
996 else:
975 return
997 return
976
998
977 csize = self.fileSize - self.fp.tell()
999 csize = self.fileSize - self.fp.tell()
978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
979 while True:
1001 while True:
980
1002
981 if self.fp.tell()<self.fileSize:
1003 if self.fp.tell()<self.fileSize:
982 self.fp.seek(self.fp.tell() + neededsize)
1004 self.fp.seek(self.fp.tell() + neededsize)
983 else:
1005 else:
984 self.fp.seek(self.fp.tell() - neededsize)
1006 self.fp.seek(self.fp.tell() - neededsize)
985 break
1007 break
986
1008
987 # csize = self.fileSize - self.fp.tell()
1009 # csize = self.fileSize - self.fp.tell()
988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
989 # factor = int(csize/neededsize)
1011 # factor = int(csize/neededsize)
990 # if factor > 0:
1012 # if factor > 0:
991 # self.fp.seek(self.fp.tell() + factor*neededsize)
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
992
1014
993 self.flagIsNewFile = 0
1015 self.flagIsNewFile = 0
994 self.__isFirstTimeOnline = 0
1016 self.__isFirstTimeOnline = 0
995
1017
996 def __setNewBlock(self):
1018 def __setNewBlock(self):
997 #if self.server is None:
1019 #if self.server is None:
998 if self.fp == None:
1020 if self.fp == None:
999 return 0
1021 return 0
1000
1022
1001 # if self.online:
1023 # if self.online:
1002 # self.__jumpToLastBlock()
1024 # self.__jumpToLastBlock()
1003
1025
1004 if self.flagIsNewFile:
1026 if self.flagIsNewFile:
1005 self.lastUTTime = self.basicHeaderObj.utc
1027 self.lastUTTime = self.basicHeaderObj.utc
1006 return 1
1028 return 1
1007
1029
1008 if self.realtime:
1030 if self.realtime:
1009 self.flagDiscontinuousBlock = 1
1031 self.flagDiscontinuousBlock = 1
1010 if not(self.setNextFile()):
1032 if not(self.setNextFile()):
1011 return 0
1033 return 0
1012 else:
1034 else:
1013 return 1
1035 return 1
1014 #if self.server is None:
1036 #if self.server is None:
1015 currentSize = self.fileSize - self.fp.tell()
1037 currentSize = self.fileSize - self.fp.tell()
1016 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1017 if (currentSize >= neededSize):
1039 if (currentSize >= neededSize):
1018 self.basicHeaderObj.read(self.fp)
1040 self.basicHeaderObj.read(self.fp)
1019 self.lastUTTime = self.basicHeaderObj.utc
1041 self.lastUTTime = self.basicHeaderObj.utc
1020 return 1
1042 return 1
1021 # else:
1043 # else:
1022 # self.basicHeaderObj.read(self.zHeader)
1044 # self.basicHeaderObj.read(self.zHeader)
1023 # self.lastUTTime = self.basicHeaderObj.utc
1045 # self.lastUTTime = self.basicHeaderObj.utc
1024 # return 1
1046 # return 1
1025 if self.__waitNewBlock():
1047 if self.__waitNewBlock():
1026 self.lastUTTime = self.basicHeaderObj.utc
1048 self.lastUTTime = self.basicHeaderObj.utc
1027 return 1
1049 return 1
1028 #if self.server is None:
1050 #if self.server is None:
1029 if not(self.setNextFile()):
1051 if not(self.setNextFile()):
1030 return 0
1052 return 0
1031
1053
1032 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1033 self.lastUTTime = self.basicHeaderObj.utc
1055 self.lastUTTime = self.basicHeaderObj.utc
1034
1056
1035 self.flagDiscontinuousBlock = 0
1057 self.flagDiscontinuousBlock = 0
1036
1058
1037 if deltaTime > self.maxTimeStep:
1059 if deltaTime > self.maxTimeStep:
1038 self.flagDiscontinuousBlock = 1
1060 self.flagDiscontinuousBlock = 1
1039
1061
1040 return 1
1062 return 1
1041
1063
1042 def readNextBlock(self):
1064 def readNextBlock(self):
1043
1065
1044 #Skip block out of startTime and endTime
1066 #Skip block out of startTime and endTime
1045 while True:
1067 while True:
1046 if not(self.__setNewBlock()):
1068 if not(self.__setNewBlock()):
1047 print 'returning'
1069 print 'returning'
1048 return 0
1070 return 0
1049
1050 if not(self.readBlock()):
1071 if not(self.readBlock()):
1051 return 0
1072 return 0
1052
1053 self.getBasicHeader()
1073 self.getBasicHeader()
1054
1055 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1056
1075
1057 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1058 self.processingHeaderObj.dataBlocksPerFile,
1077 self.processingHeaderObj.dataBlocksPerFile,
1059 self.dataOut.datatime.ctime())
1078 self.dataOut.datatime.ctime())
1060 continue
1079 continue
1061
1080
1062 break
1081 break
1063
1082
1064 if self.verbose:
1083 if self.verbose:
1065 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1066 self.processingHeaderObj.dataBlocksPerFile,
1085 self.processingHeaderObj.dataBlocksPerFile,
1067 self.dataOut.datatime.ctime())
1086 self.dataOut.datatime.ctime())
1068 return 1
1087 return 1
1069
1088
1070 def __readFirstHeader(self):
1089 def __readFirstHeader(self):
1071
1090
1072 self.basicHeaderObj.read(self.fp)
1091 self.basicHeaderObj.read(self.fp)
1073 self.systemHeaderObj.read(self.fp)
1092 self.systemHeaderObj.read(self.fp)
1074 self.radarControllerHeaderObj.read(self.fp)
1093 self.radarControllerHeaderObj.read(self.fp)
1075 self.processingHeaderObj.read(self.fp)
1094 self.processingHeaderObj.read(self.fp)
1076
1095
1077 self.firstHeaderSize = self.basicHeaderObj.size
1096 self.firstHeaderSize = self.basicHeaderObj.size
1078
1097
1079 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1080 if datatype == 0:
1099 if datatype == 0:
1081 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1082 elif datatype == 1:
1101 elif datatype == 1:
1083 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1084 elif datatype == 2:
1103 elif datatype == 2:
1085 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1086 elif datatype == 3:
1105 elif datatype == 3:
1087 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1088 elif datatype == 4:
1107 elif datatype == 4:
1089 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1090 elif datatype == 5:
1109 elif datatype == 5:
1091 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1092 else:
1111 else:
1093 raise ValueError, 'Data type was not defined'
1112 raise ValueError, 'Data type was not defined'
1094
1113
1095 self.dtype = datatype_str
1114 self.dtype = datatype_str
1096 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1097 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1098 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1099 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1100 self.getBlockDimension()
1119 self.getBlockDimension()
1101
1120
1102 def __verifyFile(self, filename, msgFlag=True):
1121 def __verifyFile(self, filename, msgFlag=True):
1103
1122
1104 msg = None
1123 msg = None
1105
1124
1106 try:
1125 try:
1107 fp = open(filename, 'rb')
1126 fp = open(filename, 'rb')
1108 except IOError:
1127 except IOError:
1109
1128
1110 if msgFlag:
1129 if msgFlag:
1111 print "[Reading] File %s can't be opened" % (filename)
1130 print "[Reading] File %s can't be opened" % (filename)
1112
1131
1113 return False
1132 return False
1114
1133
1115 currentPosition = fp.tell()
1134 currentPosition = fp.tell()
1116 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1117
1136
1118 if neededSize == 0:
1137 if neededSize == 0:
1119 basicHeaderObj = BasicHeader(LOCALTIME)
1138 basicHeaderObj = BasicHeader(LOCALTIME)
1120 systemHeaderObj = SystemHeader()
1139 systemHeaderObj = SystemHeader()
1121 radarControllerHeaderObj = RadarControllerHeader()
1140 radarControllerHeaderObj = RadarControllerHeader()
1122 processingHeaderObj = ProcessingHeader()
1141 processingHeaderObj = ProcessingHeader()
1123
1142
1124 if not( basicHeaderObj.read(fp) ):
1143 if not( basicHeaderObj.read(fp) ):
1125 fp.close()
1144 fp.close()
1126 return False
1145 return False
1127
1146
1128 if not( systemHeaderObj.read(fp) ):
1147 if not( systemHeaderObj.read(fp) ):
1129 fp.close()
1148 fp.close()
1130 return False
1149 return False
1131
1150
1132 if not( radarControllerHeaderObj.read(fp) ):
1151 if not( radarControllerHeaderObj.read(fp) ):
1133 fp.close()
1152 fp.close()
1134 return False
1153 return False
1135
1154
1136 if not( processingHeaderObj.read(fp) ):
1155 if not( processingHeaderObj.read(fp) ):
1137 fp.close()
1156 fp.close()
1138 return False
1157 return False
1139
1158
1140 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1141 else:
1160 else:
1142 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1143
1162
1144 fp.close()
1163 fp.close()
1145
1164
1146 fileSize = os.path.getsize(filename)
1165 fileSize = os.path.getsize(filename)
1147 currentSize = fileSize - currentPosition
1166 currentSize = fileSize - currentPosition
1148
1167
1149 if currentSize < neededSize:
1168 if currentSize < neededSize:
1150 if msgFlag and (msg != None):
1169 if msgFlag and (msg != None):
1151 print msg
1170 print msg
1152 return False
1171 return False
1153
1172
1154 return True
1173 return True
1155
1174
1156 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1157
1176
1158 path_empty = True
1177 path_empty = True
1159
1178
1160 dateList = []
1179 dateList = []
1161 pathList = []
1180 pathList = []
1162
1181
1163 multi_path = path.split(',')
1182 multi_path = path.split(',')
1164
1183
1165 if not walk:
1184 if not walk:
1166
1185
1167 for single_path in multi_path:
1186 for single_path in multi_path:
1168
1187
1169 if not os.path.isdir(single_path):
1188 if not os.path.isdir(single_path):
1170 continue
1189 continue
1171
1190
1172 fileList = glob.glob1(single_path, "*"+ext)
1191 fileList = glob.glob1(single_path, "*"+ext)
1173
1192
1174 if not fileList:
1193 if not fileList:
1175 continue
1194 continue
1176
1195
1177 path_empty = False
1196 path_empty = False
1178
1197
1179 fileList.sort()
1198 fileList.sort()
1180
1199
1181 for thisFile in fileList:
1200 for thisFile in fileList:
1182
1201
1183 if not os.path.isfile(os.path.join(single_path, thisFile)):
1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1184 continue
1203 continue
1185
1204
1186 if not isRadarFile(thisFile):
1205 if not isRadarFile(thisFile):
1187 continue
1206 continue
1188
1207
1189 if not isFileInDateRange(thisFile, startDate, endDate):
1208 if not isFileInDateRange(thisFile, startDate, endDate):
1190 continue
1209 continue
1191
1210
1192 thisDate = getDateFromRadarFile(thisFile)
1211 thisDate = getDateFromRadarFile(thisFile)
1193
1212
1194 if thisDate in dateList:
1213 if thisDate in dateList:
1195 continue
1214 continue
1196
1215
1197 dateList.append(thisDate)
1216 dateList.append(thisDate)
1198 pathList.append(single_path)
1217 pathList.append(single_path)
1199
1218
1200 else:
1219 else:
1201 for single_path in multi_path:
1220 for single_path in multi_path:
1202
1221
1203 if not os.path.isdir(single_path):
1222 if not os.path.isdir(single_path):
1204 continue
1223 continue
1205
1224
1206 dirList = []
1225 dirList = []
1207
1226
1208 for thisPath in os.listdir(single_path):
1227 for thisPath in os.listdir(single_path):
1209
1228
1210 if not os.path.isdir(os.path.join(single_path,thisPath)):
1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1211 continue
1230 continue
1212
1231
1213 if not isRadarFolder(thisPath):
1232 if not isRadarFolder(thisPath):
1214 continue
1233 continue
1215
1234
1216 if not isFolderInDateRange(thisPath, startDate, endDate):
1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1217 continue
1236 continue
1218
1237
1219 dirList.append(thisPath)
1238 dirList.append(thisPath)
1220
1239
1221 if not dirList:
1240 if not dirList:
1222 continue
1241 continue
1223
1242
1224 dirList.sort()
1243 dirList.sort()
1225
1244
1226 for thisDir in dirList:
1245 for thisDir in dirList:
1227
1246
1228 datapath = os.path.join(single_path, thisDir, expLabel)
1247 datapath = os.path.join(single_path, thisDir, expLabel)
1229 fileList = glob.glob1(datapath, "*"+ext)
1248 fileList = glob.glob1(datapath, "*"+ext)
1230
1249
1231 if not fileList:
1250 if not fileList:
1232 continue
1251 continue
1233
1252
1234 path_empty = False
1253 path_empty = False
1235
1254
1236 thisDate = getDateFromRadarFolder(thisDir)
1255 thisDate = getDateFromRadarFolder(thisDir)
1237
1256
1238 pathList.append(datapath)
1257 pathList.append(datapath)
1239 dateList.append(thisDate)
1258 dateList.append(thisDate)
1240
1259
1241 dateList.sort()
1260 dateList.sort()
1242
1261
1243 if walk:
1262 if walk:
1244 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1245 else:
1264 else:
1246 pattern_path = multi_path[0]
1265 pattern_path = multi_path[0]
1247
1266
1248 if path_empty:
1267 if path_empty:
1249 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1250 else:
1269 else:
1251 if not dateList:
1270 if not dateList:
1252 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1253
1272
1254 if include_path:
1273 if include_path:
1255 return dateList, pathList
1274 return dateList, pathList
1256
1275
1257 return dateList
1276 return dateList
1258
1277
1259 def setup(self,
1278 def setup(self,
1260 path=None,
1279 path=None,
1261 startDate=None,
1280 startDate=None,
1262 endDate=None,
1281 endDate=None,
1263 startTime=datetime.time(0,0,0),
1282 startTime=datetime.time(0,0,0),
1264 endTime=datetime.time(23,59,59),
1283 endTime=datetime.time(23,59,59),
1265 set=None,
1284 set=None,
1266 expLabel = "",
1285 expLabel = "",
1267 ext = None,
1286 ext = None,
1268 online = False,
1287 online = False,
1269 delay = 60,
1288 delay = 60,
1270 walk = True,
1289 walk = True,
1271 getblock = False,
1290 getblock = False,
1272 nTxs = 1,
1291 nTxs = 1,
1273 realtime=False,
1292 realtime=False,
1274 blocksize=None,
1293 blocksize=None,
1275 blocktime=None,
1294 blocktime=None,
1276 queue=None,
1295 queue=None,
1277 skip=None,
1296 skip=None,
1278 cursor=None,
1297 cursor=None,
1279 warnings=True,
1298 warnings=True,
1280 verbose=True,
1299 verbose=True,
1281 server=None):
1300 server=None):
1282 if server is not None:
1301 if server is not None:
1283 if 'tcp://' in server:
1302 if 'tcp://' in server:
1284 address = server
1303 address = server
1285 else:
1304 else:
1286 address = 'ipc:///tmp/%s' % server
1305 address = 'ipc:///tmp/%s' % server
1287 self.server = address
1306 self.server = address
1288 self.context = zmq.Context()
1307 self.context = zmq.Context()
1289 self.receiver = self.context.socket(zmq.PULL)
1308 self.receiver = self.context.socket(zmq.PULL)
1290 self.receiver.connect(self.server)
1309 self.receiver.connect(self.server)
1291 time.sleep(0.5)
1310 time.sleep(0.5)
1292 print '[Starting] ReceiverData from {}'.format(self.server)
1311 print '[Starting] ReceiverData from {}'.format(self.server)
1293 else:
1312 else:
1294 self.server = None
1313 self.server = None
1295 if path == None:
1314 if path == None:
1296 raise ValueError, "[Reading] The path is not valid"
1315 raise ValueError, "[Reading] The path is not valid"
1297
1316
1298 if ext == None:
1317 if ext == None:
1299 ext = self.ext
1318 ext = self.ext
1300
1319
1301 if online:
1320 if online:
1302 print "[Reading] Searching files in online mode..."
1321 print "[Reading] Searching files in online mode..."
1303
1322
1304 for nTries in range( self.nTries ):
1323 for nTries in range( self.nTries ):
1305 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1324 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1306
1325
1307 if fullpath:
1326 if fullpath:
1308 break
1327 break
1309
1328
1310 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1329 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1311 sleep( self.delay )
1330 sleep( self.delay )
1312
1331
1313 if not(fullpath):
1332 if not(fullpath):
1314 print "[Reading] There 'isn't any valid file in %s" % path
1333 print "[Reading] There 'isn't any valid file in %s" % path
1315 return
1334 return
1316
1335
1317 self.year = year
1336 self.year = year
1318 self.doy = doy
1337 self.doy = doy
1319 self.set = set - 1
1338 self.set = set - 1
1320 self.path = path
1339 self.path = path
1321 self.foldercounter = foldercounter
1340 self.foldercounter = foldercounter
1322 last_set = None
1341 last_set = None
1323 else:
1342 else:
1324 print "[Reading] Searching files in offline mode ..."
1343 print "[Reading] Searching files in offline mode ..."
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1344 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 startTime=startTime, endTime=endTime,
1345 startTime=startTime, endTime=endTime,
1327 set=set, expLabel=expLabel, ext=ext,
1346 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk, cursor=cursor,
1347 walk=walk, cursor=cursor,
1329 skip=skip, queue=queue)
1348 skip=skip, queue=queue)
1330
1349
1350 <<<<<<< HEAD
1331 if not(pathList):
1351 if not(pathList):
1332 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1352 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1333 # datetime.datetime.combine(startDate,startTime).ctime(),
1353 # datetime.datetime.combine(startDate,startTime).ctime(),
1334 # datetime.datetime.combine(endDate,endTime).ctime())
1354 # datetime.datetime.combine(endDate,endTime).ctime())
1335
1355
1336 # sys.exit(-1)
1356 # sys.exit(-1)
1337
1357
1338 self.fileIndex = -1
1358 self.fileIndex = -1
1339 self.pathList = []
1359 self.pathList = []
1340 self.filenameList = []
1360 self.filenameList = []
1341 return
1361 return
1342
1362
1363 =======
1364 if path == None:
1365 raise ValueError, "[Reading] The path is not valid"
1366
1367
1368 if ext == None:
1369 ext = self.ext
1370
1371 self.path = path
1372 self.startDate = startDate
1373 self.endDate = endDate
1374 self.startTime = startTime
1375 self.endTime = endTime
1376 self.set = set
1377 self.expLabel = expLabel
1378 self.ext = ext
1379 self.online = online
1380 self.delay = delay
1381 self.walk = walk
1382 self.getblock = getblock
1383 self.nTxs = nTxs
1384 self.realtime = realtime
1385 self.blocksize = blocksize
1386 self.blocktime = blocktime
1387
1388
1389 if self.firstTime is True:
1390 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1391 startTime=startTime, endTime=endTime,
1392 set=set, expLabel=expLabel, ext=ext,
1393 walk=walk)
1394 filenameList = filenameList[:-1]
1395
1396 if pathList is not None and filenameList is not None and online:
1397 self.onlineWithDate = True
1398 online = False
1399 self.fileIndex = -1
1400 self.pathList = pathList
1401 self.filenameList = filenameList
1402 file_name = os.path.basename(filenameList[-1])
1403 basename, ext = os.path.splitext(file_name)
1404 last_set = int(basename[-3:])
1405
1406 if online:
1407 print "[Reading] Searching files in online mode..."
1408
1409 for nTries in range(self.nTries):
1410 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1411 expLabel=expLabel,
1412 ext=ext,
1413 walk=walk,
1414 startDate=startDate,
1415 startTime=startTime,
1416 set=set)
1417
1418 if fullpath:
1419 break
1420 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1421 sleep( self.delay )
1422
1423 if not(fullpath):
1424 print "[Reading] There 'isn't any valid file in %s" % path
1425 return
1426
1427 self.year = year
1428 self.doy = doy
1429 self.set = set - 1
1430 self.path = path
1431 self.foldercounter = foldercounter
1432 last_set = None
1433
1434 else:
1435 print "[Reading] Searching files in offline mode ..."
1436 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1437 startTime=startTime, endTime=endTime,
1438 set=set, expLabel=expLabel, ext=ext,
1439 walk=walk)
1440
1441 if not(pathList):
1442 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1443 # datetime.datetime.combine(startDate,startTime).ctime(),
1444 # datetime.datetime.combine(endDate,endTime).ctime())
1445
1446 # sys.exit(-1)
1447
1448 self.fileIndex = -1
1449 self.pathList = []
1450 self.filenameList = []
1451 return
1452
1453 self.fileIndex = -1
1454 self.pathList = pathList
1455 self.filenameList = filenameList
1456 file_name = os.path.basename(filenameList[-1])
1457 basename, ext = os.path.splitext(file_name)
1458 last_set = int(basename[-3:])
1459
1460
1461 self.online = online
1462 self.realtime = realtime
1463 self.delay = delay
1464 ext = ext.lower()
1465 self.ext = ext
1466 self.getByBlock = getblock
1467 self.nTxs = nTxs
1468 self.startTime = startTime
1469 self.endTime = endTime
1470
1471
1472 #Added-----------------
1473 self.selBlocksize = blocksize
1474 self.selBlocktime = blocktime
1475
1476
1477 if not(self.setNextFile()):
1478 if (startDate!=None) and (endDate!=None):
1479 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1480 elif startDate != None:
1481 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1482 else:
1483 print "[Reading] No files"
1484
1485 >>>>>>> online_data_hour
1343 self.fileIndex = -1
1486 self.fileIndex = -1
1344 self.pathList = pathList
1487 self.pathList = pathList
1345 self.filenameList = filenameList
1488 self.filenameList = filenameList
1346 file_name = os.path.basename(filenameList[-1])
1489 file_name = os.path.basename(filenameList[-1])
1347 basename, ext = os.path.splitext(file_name)
1490 basename, ext = os.path.splitext(file_name)
1348 last_set = int(basename[-3:])
1491 last_set = int(basename[-3:])
1349
1492
1350 self.online = online
1493 self.online = online
1351 self.realtime = realtime
1494 self.realtime = realtime
1352 self.delay = delay
1495 self.delay = delay
1353 ext = ext.lower()
1496 ext = ext.lower()
1354 self.ext = ext
1497 self.ext = ext
1355 self.getByBlock = getblock
1498 self.getByBlock = getblock
1356 self.nTxs = nTxs
1499 self.nTxs = nTxs
1357 self.startTime = startTime
1500 self.startTime = startTime
1358 self.endTime = endTime
1501 self.endTime = endTime
1359
1502
1360 #Added-----------------
1503 #Added-----------------
1361 self.selBlocksize = blocksize
1504 self.selBlocksize = blocksize
1362 self.selBlocktime = blocktime
1505 self.selBlocktime = blocktime
1363
1506
1364 # Verbose-----------
1507 # Verbose-----------
1365 self.verbose = verbose
1508 self.verbose = verbose
1366 self.warnings = warnings
1509 self.warnings = warnings
1367
1510
1511 <<<<<<< HEAD
1368 if not(self.setNextFile()):
1512 if not(self.setNextFile()):
1369 if (startDate!=None) and (endDate!=None):
1513 if (startDate!=None) and (endDate!=None):
1370 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1514 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1371 elif startDate != None:
1515 elif startDate != None:
1372 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1516 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1373 else:
1517 else:
1374 print "[Reading] No files"
1518 print "[Reading] No files"
1375
1519
1376 self.fileIndex = -1
1520 self.fileIndex = -1
1377 self.pathList = []
1521 self.pathList = []
1378 self.filenameList = []
1522 self.filenameList = []
1379 return
1523 return
1380
1524
1381 # self.getBasicHeader()
1525 # self.getBasicHeader()
1382
1526
1383 if last_set != None:
1527 if last_set != None:
1384 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1528 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1385 return
1529 return
1530 =======
1531 # self.getBasicHeader()
1532
1533 if last_set != None:
1534 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1535 return
1536 >>>>>>> online_data_hour
1386
1537
1387 def getBasicHeader(self):
1538 def getBasicHeader(self):
1388
1539
1389 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1540 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1390
1541
1391 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1542 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1392
1543
1393 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1544 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1394
1545
1395 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1546 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1396
1547
1397 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1548 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1398
1549
1399 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1550 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1400
1551
1401 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1552 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1402
1553
1403 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1554 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1404
1555
1405
1556
1406 def getFirstHeader(self):
1557 def getFirstHeader(self):
1407
1558
1408 raise NotImplementedError
1559 raise NotImplementedError
1409
1560
1410 def getData(self):
1561 def getData(self):
1411
1562
1412 raise NotImplementedError
1563 raise NotImplementedError
1413
1564
1414 def hasNotDataInBuffer(self):
1565 def hasNotDataInBuffer(self):
1415
1566
1416 raise NotImplementedError
1567 raise NotImplementedError
1417
1568
1418 def readBlock(self):
1569 def readBlock(self):
1419
1570
1420 raise NotImplementedError
1571 raise NotImplementedError
1421
1572
1422 def isEndProcess(self):
1573 def isEndProcess(self):
1423
1574
1424 return self.flagNoMoreFiles
1575 return self.flagNoMoreFiles
1425
1576
1426 def printReadBlocks(self):
1577 def printReadBlocks(self):
1427
1578
1428 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1579 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1429
1580
1430 def printTotalBlocks(self):
1581 def printTotalBlocks(self):
1431
1582
1432 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1583 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1433
1584
1434 def printNumberOfBlock(self):
1585 def printNumberOfBlock(self):
1435
1586
1436 if self.flagIsNewBlock:
1587 if self.flagIsNewBlock:
1437 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1588 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1438 self.processingHeaderObj.dataBlocksPerFile,
1589 self.processingHeaderObj.dataBlocksPerFile,
1439 self.dataOut.datatime.ctime())
1590 self.dataOut.datatime.ctime())
1440
1591
1441 def printInfo(self):
1592 def printInfo(self):
1442
1593
1443 if self.__printInfo == False:
1594 if self.__printInfo == False:
1444 return
1595 return
1445
1596
1446 self.basicHeaderObj.printInfo()
1597 self.basicHeaderObj.printInfo()
1447 self.systemHeaderObj.printInfo()
1598 self.systemHeaderObj.printInfo()
1448 self.radarControllerHeaderObj.printInfo()
1599 self.radarControllerHeaderObj.printInfo()
1449 self.processingHeaderObj.printInfo()
1600 self.processingHeaderObj.printInfo()
1450
1601
1451 self.__printInfo = False
1602 self.__printInfo = False
1452
1603
1453
1604
1454 def run(self,
1605 def run(self,
1455 path=None,
1606 path=None,
1456 startDate=None,
1607 startDate=None,
1457 endDate=None,
1608 endDate=None,
1458 startTime=datetime.time(0,0,0),
1609 startTime=datetime.time(0,0,0),
1459 endTime=datetime.time(23,59,59),
1610 endTime=datetime.time(23,59,59),
1460 set=None,
1611 set=None,
1461 expLabel = "",
1612 expLabel = "",
1462 ext = None,
1613 ext = None,
1463 online = False,
1614 online = False,
1464 delay = 60,
1615 delay = 60,
1465 walk = True,
1616 walk = True,
1466 getblock = False,
1617 getblock = False,
1467 nTxs = 1,
1618 nTxs = 1,
1468 realtime=False,
1619 realtime=False,
1469 blocksize=None,
1620 blocksize=None,
1470 blocktime=None,
1621 blocktime=None,
1471 queue=None,
1622 queue=None,
1472 skip=None,
1623 skip=None,
1473 cursor=None,
1624 cursor=None,
1474 warnings=True,
1625 warnings=True,
1475 server=None,
1626 server=None,
1476 verbose=True, **kwargs):
1627 verbose=True, **kwargs):
1477
1628
1478 if not(self.isConfig):
1629 if not(self.isConfig):
1630 <<<<<<< HEAD
1479 # self.dataOut = dataOut
1631 # self.dataOut = dataOut
1480 self.setup( path=path,
1632 self.setup( path=path,
1481 startDate=startDate,
1633 startDate=startDate,
1482 endDate=endDate,
1634 endDate=endDate,
1483 startTime=startTime,
1635 startTime=startTime,
1484 endTime=endTime,
1636 endTime=endTime,
1485 set=set,
1637 set=set,
1486 expLabel=expLabel,
1638 expLabel=expLabel,
1487 ext=ext,
1639 ext=ext,
1488 online=online,
1640 online=online,
1489 delay=delay,
1641 delay=delay,
1490 walk=walk,
1642 walk=walk,
1491 getblock=getblock,
1643 getblock=getblock,
1492 nTxs=nTxs,
1644 nTxs=nTxs,
1493 realtime=realtime,
1645 realtime=realtime,
1494 blocksize=blocksize,
1646 blocksize=blocksize,
1495 blocktime=blocktime,
1647 blocktime=blocktime,
1496 queue=queue,
1648 queue=queue,
1497 skip=skip,
1649 skip=skip,
1498 cursor=cursor,
1650 cursor=cursor,
1499 warnings=warnings,
1651 warnings=warnings,
1500 server=server,
1652 server=server,
1501 verbose=verbose)
1653 verbose=verbose)
1654 =======
1655
1656 # self.dataOut = dataOut
1657 self.setup(**kwargs)
1658 >>>>>>> online_data_hour
1502 self.isConfig = True
1659 self.isConfig = True
1503 if server is None:
1660 if server is None:
1504 self.getData()
1661 self.getData()
1505 else:
1662 else:
1506 self.getFromServer()
1663 self.getFromServer()
1507
1664
1508 class JRODataWriter(JRODataIO):
1665 class JRODataWriter(JRODataIO):
1509
1666
1510 """
1667 """
1511 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1668 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1512 de los datos siempre se realiza por bloques.
1669 de los datos siempre se realiza por bloques.
1513 """
1670 """
1514
1671
1515 blockIndex = 0
1672 blockIndex = 0
1516
1673
1517 path = None
1674 path = None
1518
1675
1519 setFile = None
1676 setFile = None
1520
1677
1521 profilesPerBlock = None
1678 profilesPerBlock = None
1522
1679
1523 blocksPerFile = None
1680 blocksPerFile = None
1524
1681
1525 nWriteBlocks = 0
1682 nWriteBlocks = 0
1526
1683
1527 fileDate = None
1684 fileDate = None
1528
1685
1529 def __init__(self, dataOut=None):
1686 def __init__(self, dataOut=None):
1530 raise NotImplementedError
1687 raise NotImplementedError
1531
1688
1532
1689
1533 def hasAllDataInBuffer(self):
1690 def hasAllDataInBuffer(self):
1534 raise NotImplementedError
1691 raise NotImplementedError
1535
1692
1536
1693
1537 def setBlockDimension(self):
1694 def setBlockDimension(self):
1538 raise NotImplementedError
1695 raise NotImplementedError
1539
1696
1540
1697
1541 def writeBlock(self):
1698 def writeBlock(self):
1542 raise NotImplementedError
1699 raise NotImplementedError
1543
1700
1544
1701
1545 def putData(self):
1702 def putData(self):
1546 raise NotImplementedError
1703 raise NotImplementedError
1547
1704
1548
1705
1549 def getProcessFlags(self):
1706 def getProcessFlags(self):
1550
1707
1551 processFlags = 0
1708 processFlags = 0
1552
1709
1553 dtype_index = get_dtype_index(self.dtype)
1710 dtype_index = get_dtype_index(self.dtype)
1554 procflag_dtype = get_procflag_dtype(dtype_index)
1711 procflag_dtype = get_procflag_dtype(dtype_index)
1555
1712
1556 processFlags += procflag_dtype
1713 processFlags += procflag_dtype
1557
1714
1558 if self.dataOut.flagDecodeData:
1715 if self.dataOut.flagDecodeData:
1559 processFlags += PROCFLAG.DECODE_DATA
1716 processFlags += PROCFLAG.DECODE_DATA
1560
1717
1561 if self.dataOut.flagDeflipData:
1718 if self.dataOut.flagDeflipData:
1562 processFlags += PROCFLAG.DEFLIP_DATA
1719 processFlags += PROCFLAG.DEFLIP_DATA
1563
1720
1564 if self.dataOut.code is not None:
1721 if self.dataOut.code is not None:
1565 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1722 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1566
1723
1567 if self.dataOut.nCohInt > 1:
1724 if self.dataOut.nCohInt > 1:
1568 processFlags += PROCFLAG.COHERENT_INTEGRATION
1725 processFlags += PROCFLAG.COHERENT_INTEGRATION
1569
1726
1570 if self.dataOut.type == "Spectra":
1727 if self.dataOut.type == "Spectra":
1571 if self.dataOut.nIncohInt > 1:
1728 if self.dataOut.nIncohInt > 1:
1572 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1729 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1573
1730
1574 if self.dataOut.data_dc is not None:
1731 if self.dataOut.data_dc is not None:
1575 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1732 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1576
1733
1577 if self.dataOut.flagShiftFFT:
1734 if self.dataOut.flagShiftFFT:
1578 processFlags += PROCFLAG.SHIFT_FFT_DATA
1735 processFlags += PROCFLAG.SHIFT_FFT_DATA
1579
1736
1580 return processFlags
1737 return processFlags
1581
1738
1582 def setBasicHeader(self):
1739 def setBasicHeader(self):
1583
1740
1584 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1741 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1585 self.basicHeaderObj.version = self.versionFile
1742 self.basicHeaderObj.version = self.versionFile
1586 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1743 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1587
1744
1588 utc = numpy.floor(self.dataOut.utctime)
1745 utc = numpy.floor(self.dataOut.utctime)
1589 milisecond = (self.dataOut.utctime - utc)* 1000.0
1746 milisecond = (self.dataOut.utctime - utc)* 1000.0
1590
1747
1591 self.basicHeaderObj.utc = utc
1748 self.basicHeaderObj.utc = utc
1592 self.basicHeaderObj.miliSecond = milisecond
1749 self.basicHeaderObj.miliSecond = milisecond
1593 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1750 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1594 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1751 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1595 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1752 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1596
1753
1597 def setFirstHeader(self):
1754 def setFirstHeader(self):
1598 """
1755 """
1599 Obtiene una copia del First Header
1756 Obtiene una copia del First Header
1600
1757
1601 Affected:
1758 Affected:
1602
1759
1603 self.basicHeaderObj
1760 self.basicHeaderObj
1604 self.systemHeaderObj
1761 self.systemHeaderObj
1605 self.radarControllerHeaderObj
1762 self.radarControllerHeaderObj
1606 self.processingHeaderObj self.
1763 self.processingHeaderObj self.
1607
1764
1608 Return:
1765 Return:
1609 None
1766 None
1610 """
1767 """
1611
1768
1612 raise NotImplementedError
1769 raise NotImplementedError
1613
1770
1614 def __writeFirstHeader(self):
1771 def __writeFirstHeader(self):
1615 """
1772 """
1616 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1773 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1617
1774
1618 Affected:
1775 Affected:
1619 __dataType
1776 __dataType
1620
1777
1621 Return:
1778 Return:
1622 None
1779 None
1623 """
1780 """
1781 <<<<<<< HEAD
1624
1782
1625 # CALCULAR PARAMETROS
1783 # CALCULAR PARAMETROS
1626
1784
1785 =======
1786
1787 # CALCULAR PARAMETROS
1788
1789 >>>>>>> online_data_hour
1627 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1790 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1628 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1791 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1629
1792
1630 self.basicHeaderObj.write(self.fp)
1793 self.basicHeaderObj.write(self.fp)
1631 self.systemHeaderObj.write(self.fp)
1794 self.systemHeaderObj.write(self.fp)
1632 self.radarControllerHeaderObj.write(self.fp)
1795 self.radarControllerHeaderObj.write(self.fp)
1633 self.processingHeaderObj.write(self.fp)
1796 self.processingHeaderObj.write(self.fp)
1634
1797
1635 def __setNewBlock(self):
1798 def __setNewBlock(self):
1636 """
1799 """
1637 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1800 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1638
1801
1639 Return:
1802 Return:
1640 0 : si no pudo escribir nada
1803 0 : si no pudo escribir nada
1641 1 : Si escribio el Basic el First Header
1804 1 : Si escribio el Basic el First Header
1642 """
1805 """
1643 if self.fp == None:
1806 if self.fp == None:
1644 self.setNextFile()
1807 self.setNextFile()
1645
1808
1646 if self.flagIsNewFile:
1809 if self.flagIsNewFile:
1647 return 1
1810 return 1
1648
1811
1649 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1812 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1650 self.basicHeaderObj.write(self.fp)
1813 self.basicHeaderObj.write(self.fp)
1651 return 1
1814 return 1
1652
1815
1653 if not( self.setNextFile() ):
1816 if not( self.setNextFile() ):
1654 return 0
1817 return 0
1655
1818
1656 return 1
1819 return 1
1657
1820
1658
1821
1659 def writeNextBlock(self):
1822 def writeNextBlock(self):
1660 """
1823 """
1661 Selecciona el bloque siguiente de datos y los escribe en un file
1824 Selecciona el bloque siguiente de datos y los escribe en un file
1662
1825
1663 Return:
1826 Return:
1664 0 : Si no hizo pudo escribir el bloque de datos
1827 0 : Si no hizo pudo escribir el bloque de datos
1665 1 : Si no pudo escribir el bloque de datos
1828 1 : Si no pudo escribir el bloque de datos
1666 """
1829 """
1667 if not( self.__setNewBlock() ):
1830 if not( self.__setNewBlock() ):
1668 return 0
1831 return 0
1669
1832
1670 self.writeBlock()
1833 self.writeBlock()
1671
1834
1672 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1835 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1673 self.processingHeaderObj.dataBlocksPerFile)
1836 self.processingHeaderObj.dataBlocksPerFile)
1674
1837
1675 return 1
1838 return 1
1676
1839
1677 def setNextFile(self):
1840 def setNextFile(self):
1678 """
1841 """
1679 Determina el siguiente file que sera escrito
1842 Determina el siguiente file que sera escrito
1680
1843
1681 Affected:
1844 Affected:
1682 self.filename
1845 self.filename
1683 self.subfolder
1846 self.subfolder
1684 self.fp
1847 self.fp
1685 self.setFile
1848 self.setFile
1686 self.flagIsNewFile
1849 self.flagIsNewFile
1687
1850
1688 Return:
1851 Return:
1689 0 : Si el archivo no puede ser escrito
1852 0 : Si el archivo no puede ser escrito
1690 1 : Si el archivo esta listo para ser escrito
1853 1 : Si el archivo esta listo para ser escrito
1691 """
1854 """
1692 ext = self.ext
1855 ext = self.ext
1693 path = self.path
1856 path = self.path
1694
1857
1695 if self.fp != None:
1858 if self.fp != None:
1696 self.fp.close()
1859 self.fp.close()
1697
1860
1698 timeTuple = time.localtime( self.dataOut.utctime)
1861 timeTuple = time.localtime( self.dataOut.utctime)
1699 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1862 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1700
1863
1701 fullpath = os.path.join( path, subfolder )
1864 fullpath = os.path.join( path, subfolder )
1702 setFile = self.setFile
1865 setFile = self.setFile
1703
1866
1704 if not( os.path.exists(fullpath) ):
1867 if not( os.path.exists(fullpath) ):
1705 os.mkdir(fullpath)
1868 os.mkdir(fullpath)
1706 setFile = -1 #inicializo mi contador de seteo
1869 setFile = -1 #inicializo mi contador de seteo
1707 else:
1870 else:
1708 filesList = os.listdir( fullpath )
1871 filesList = os.listdir( fullpath )
1709 if len( filesList ) > 0:
1872 if len( filesList ) > 0:
1710 filesList = sorted( filesList, key=str.lower )
1873 filesList = sorted( filesList, key=str.lower )
1711 filen = filesList[-1]
1874 filen = filesList[-1]
1712 # el filename debera tener el siguiente formato
1875 # el filename debera tener el siguiente formato
1713 # 0 1234 567 89A BCDE (hex)
1876 # 0 1234 567 89A BCDE (hex)
1714 # x YYYY DDD SSS .ext
1877 # x YYYY DDD SSS .ext
1715 if isNumber( filen[8:11] ):
1878 if isNumber( filen[8:11] ):
1716 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1879 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1717 else:
1880 else:
1718 setFile = -1
1881 setFile = -1
1719 else:
1882 else:
1720 setFile = -1 #inicializo mi contador de seteo
1883 setFile = -1 #inicializo mi contador de seteo
1721
1884
1722 setFile += 1
1885 setFile += 1
1723
1886
1724 #If this is a new day it resets some values
1887 #If this is a new day it resets some values
1725 if self.dataOut.datatime.date() > self.fileDate:
1888 if self.dataOut.datatime.date() > self.fileDate:
1726 setFile = 0
1889 setFile = 0
1727 self.nTotalBlocks = 0
1890 self.nTotalBlocks = 0
1728
1891
1729 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1892 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1730
1893
1731 filename = os.path.join( path, subfolder, filen )
1894 filename = os.path.join( path, subfolder, filen )
1732
1895
1733 fp = open( filename,'wb' )
1896 fp = open( filename,'wb' )
1734
1897
1735 self.blockIndex = 0
1898 self.blockIndex = 0
1736
1899
1737 #guardando atributos
1900 #guardando atributos
1738 self.filename = filename
1901 self.filename = filename
1739 self.subfolder = subfolder
1902 self.subfolder = subfolder
1740 self.fp = fp
1903 self.fp = fp
1741 self.setFile = setFile
1904 self.setFile = setFile
1742 self.flagIsNewFile = 1
1905 self.flagIsNewFile = 1
1743 self.fileDate = self.dataOut.datatime.date()
1906 self.fileDate = self.dataOut.datatime.date()
1744
1907
1745 self.setFirstHeader()
1908 self.setFirstHeader()
1746
1909
1747 print '[Writing] Opening file: %s'%self.filename
1910 print '[Writing] Opening file: %s'%self.filename
1748
1911
1749 self.__writeFirstHeader()
1912 self.__writeFirstHeader()
1750
1913
1751 return 1
1914 return 1
1752
1915
1753 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1916 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1754 """
1917 """
1755 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1918 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1756
1919
1757 Inputs:
1920 Inputs:
1758 path : directory where data will be saved
1921 path : directory where data will be saved
1759 profilesPerBlock : number of profiles per block
1922 profilesPerBlock : number of profiles per block
1760 set : initial file set
1923 set : initial file set
1761 datatype : An integer number that defines data type:
1924 datatype : An integer number that defines data type:
1762 0 : int8 (1 byte)
1925 0 : int8 (1 byte)
1763 1 : int16 (2 bytes)
1926 1 : int16 (2 bytes)
1764 2 : int32 (4 bytes)
1927 2 : int32 (4 bytes)
1765 3 : int64 (8 bytes)
1928 3 : int64 (8 bytes)
1766 4 : float32 (4 bytes)
1929 4 : float32 (4 bytes)
1767 5 : double64 (8 bytes)
1930 5 : double64 (8 bytes)
1768
1931
1769 Return:
1932 Return:
1770 0 : Si no realizo un buen seteo
1933 0 : Si no realizo un buen seteo
1771 1 : Si realizo un buen seteo
1934 1 : Si realizo un buen seteo
1772 """
1935 """
1773
1936
1774 if ext == None:
1937 if ext == None:
1775 ext = self.ext
1938 ext = self.ext
1776
1939
1777 self.ext = ext.lower()
1940 self.ext = ext.lower()
1778
1941
1779 self.path = path
1942 self.path = path
1780
1943
1781 if set is None:
1944 if set is None:
1782 self.setFile = -1
1945 self.setFile = -1
1783 else:
1946 else:
1784 self.setFile = set - 1
1947 self.setFile = set - 1
1785
1948
1786 self.blocksPerFile = blocksPerFile
1949 self.blocksPerFile = blocksPerFile
1787
1950
1788 self.profilesPerBlock = profilesPerBlock
1951 self.profilesPerBlock = profilesPerBlock
1789
1952
1790 self.dataOut = dataOut
1953 self.dataOut = dataOut
1791 self.fileDate = self.dataOut.datatime.date()
1954 self.fileDate = self.dataOut.datatime.date()
1792 #By default
1955 #By default
1793 self.dtype = self.dataOut.dtype
1956 self.dtype = self.dataOut.dtype
1794
1957
1795 if datatype is not None:
1958 if datatype is not None:
1796 self.dtype = get_numpy_dtype(datatype)
1959 self.dtype = get_numpy_dtype(datatype)
1797
1960
1798 if not(self.setNextFile()):
1961 if not(self.setNextFile()):
1799 print "[Writing] There isn't a next file"
1962 print "[Writing] There isn't a next file"
1800 return 0
1963 return 0
1801
1964
1802 self.setBlockDimension()
1965 self.setBlockDimension()
1803
1966
1804 return 1
1967 return 1
1805
1968
1806 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1969 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1807
1970
1808 if not(self.isConfig):
1971 if not(self.isConfig):
1809
1972
1810 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1973 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1811 self.isConfig = True
1974 self.isConfig = True
1812
1975
1813 self.putData()
1976 self.putData()
Show file before | Show file after | Hide comments
@@ -1,737 +1,824
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import numpy
7 import numpy
8
8
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13 import zmq
13 import zmq
14 import tempfile
14 import tempfile
15 from StringIO import StringIO
15 from StringIO import StringIO
16 # from _sha import blocksize
16 # from _sha import blocksize
17
17
18 class VoltageReader(JRODataReader, ProcessingUnit):
18 class VoltageReader(JRODataReader, ProcessingUnit):
19 """
19 """
20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
21 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
22 perfiles*alturas*canales) son almacenados en la variable "buffer".
22 perfiles*alturas*canales) son almacenados en la variable "buffer".
23
23
24 perfiles * alturas * canales
24 perfiles * alturas * canales
25
25
26 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
26 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
27 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
27 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
28 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
28 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
29 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
29 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
30
30
31 Example:
31 Example:
32
32
33 dpath = "/home/myuser/data"
33 dpath = "/home/myuser/data"
34
34
35 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
35 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
36
36
37 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
37 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
38
38
39 readerObj = VoltageReader()
39 readerObj = VoltageReader()
40
40
41 readerObj.setup(dpath, startTime, endTime)
41 readerObj.setup(dpath, startTime, endTime)
42
42
43 while(True):
43 while(True):
44
44
45 #to get one profile
45 #to get one profile
46 profile = readerObj.getData()
46 profile = readerObj.getData()
47
47
48 #print the profile
48 #print the profile
49 print profile
49 print profile
50
50
51 #If you want to see all datablock
51 #If you want to see all datablock
52 print readerObj.datablock
52 print readerObj.datablock
53
53
54 if readerObj.flagNoMoreFiles:
54 if readerObj.flagNoMoreFiles:
55 break
55 break
56
56
57 """
57 """
58
58
59 ext = ".r"
59 ext = ".r"
60
60
61 optchar = "D"
61 optchar = "D"
62 dataOut = None
62 dataOut = None
63
63
64 def __init__(self, **kwargs):
64 def __init__(self, **kwargs):
65 """
65 """
66 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
66 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
67
67
68 Input:
68 Input:
69 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
69 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
70 almacenar un perfil de datos cada vez que se haga un requerimiento
70 almacenar un perfil de datos cada vez que se haga un requerimiento
71 (getData). El perfil sera obtenido a partir del buffer de datos,
71 (getData). El perfil sera obtenido a partir del buffer de datos,
72 si el buffer esta vacio se hara un nuevo proceso de lectura de un
72 si el buffer esta vacio se hara un nuevo proceso de lectura de un
73 bloque de datos.
73 bloque de datos.
74 Si este parametro no es pasado se creara uno internamente.
74 Si este parametro no es pasado se creara uno internamente.
75
75
76 Variables afectadas:
76 Variables afectadas:
77 self.dataOut
77 self.dataOut
78
78
79 Return:
79 Return:
80 None
80 None
81 """
81 """
82
82
83 ProcessingUnit.__init__(self, **kwargs)
83 ProcessingUnit.__init__(self, **kwargs)
84
84
85 self.isConfig = False
85 self.isConfig = False
86
86
87 self.datablock = None
87 self.datablock = None
88
88
89 self.utc = 0
89 self.utc = 0
90
90
91 self.ext = ".r"
91 self.ext = ".r"
92
92
93 self.optchar = "D"
93 self.optchar = "D"
94
94
95 self.basicHeaderObj = BasicHeader(LOCALTIME)
95 self.basicHeaderObj = BasicHeader(LOCALTIME)
96
96
97 self.systemHeaderObj = SystemHeader()
97 self.systemHeaderObj = SystemHeader()
98
98
99 self.radarControllerHeaderObj = RadarControllerHeader()
99 self.radarControllerHeaderObj = RadarControllerHeader()
100
100
101 self.processingHeaderObj = ProcessingHeader()
101 self.processingHeaderObj = ProcessingHeader()
102
102
103 self.online = 0
103 self.online = 0
104
104
105 self.fp = None
105 self.fp = None
106
106
107 self.idFile = None
107 self.idFile = None
108
108
109 self.dtype = None
109 self.dtype = None
110
110
111 self.fileSizeByHeader = None
111 self.fileSizeByHeader = None
112
112
113 self.filenameList = []
113 self.filenameList = []
114
114
115 self.filename = None
115 self.filename = None
116
116
117 self.fileSize = None
117 self.fileSize = None
118
118
119 self.firstHeaderSize = 0
119 self.firstHeaderSize = 0
120
120
121 self.basicHeaderSize = 24
121 self.basicHeaderSize = 24
122
122
123 self.pathList = []
123 self.pathList = []
124
124
125 self.filenameList = []
125 self.filenameList = []
126
126
127 self.lastUTTime = 0
127 self.lastUTTime = 0
128
128
129 self.maxTimeStep = 30
129 self.maxTimeStep = 30
130
130
131 self.flagNoMoreFiles = 0
131 self.flagNoMoreFiles = 0
132
132
133 self.set = 0
133 self.set = 0
134
134
135 self.path = None
135 self.path = None
136
136
137 self.profileIndex = 2**32-1
137 self.profileIndex = 2**32-1
138
138
139 self.delay = 3 #seconds
139 self.delay = 3 #seconds
140
140
141 self.nTries = 3 #quantity tries
141 self.nTries = 3 #quantity tries
142
142
143 self.nFiles = 3 #number of files for searching
143 self.nFiles = 3 #number of files for searching
144
144
145 self.nReadBlocks = 0
145 self.nReadBlocks = 0
146
146
147 self.flagIsNewFile = 1
147 self.flagIsNewFile = 1
148
148
149 self.__isFirstTimeOnline = 1
149 self.__isFirstTimeOnline = 1
150
150
151 # self.ippSeconds = 0
151 # self.ippSeconds = 0
152
152
153 self.flagDiscontinuousBlock = 0
153 self.flagDiscontinuousBlock = 0
154
154
155 self.flagIsNewBlock = 0
155 self.flagIsNewBlock = 0
156
156
157 self.nTotalBlocks = 0
157 self.nTotalBlocks = 0
158
158
159 self.blocksize = 0
159 self.blocksize = 0
160
160
161 self.dataOut = self.createObjByDefault()
161 self.dataOut = self.createObjByDefault()
162
162
163 self.nTxs = 1
163 self.nTxs = 1
164
164
165 self.txIndex = 0
165 self.txIndex = 0
166
166
167 def createObjByDefault(self):
167 def createObjByDefault(self):
168
168
169 dataObj = Voltage()
169 dataObj = Voltage()
170
170
171 return dataObj
171 return dataObj
172
172
173 def __hasNotDataInBuffer(self):
173 def __hasNotDataInBuffer(self):
174
174
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
176 return 1
176 return 1
177
177
178 return 0
178 return 0
179
179
180
180
181 def getBlockDimension(self):
181 def getBlockDimension(self):
182 """
182 """
183 <<<<<<< HEAD
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184 Obtiene la cantidad de puntos a leer por cada bloque de datos
184
185
185 Affected:
186 Affected:
186 self.blocksize
187 self.blocksize
187
188 =======
188 Return:
189 Obtiene la cantidad de puntos a leer por cada bloque de datos
189 None
190
191 Affected:
192 self.blocksize
193 >>>>>>> online_data_hour
194
195 Return:
196 None
190 """
197 """
191 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
198 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
192 self.blocksize = pts2read
199 self.blocksize = pts2read
193
200
194
201
195
202
196 def readBlock(self):
203 def readBlock(self):
197 """
204 """
205 <<<<<<< HEAD
198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
206 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
199 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
207 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
200 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
208 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
201 es seteado a 0
209 es seteado a 0
202
210
203 Inputs:
211 Inputs:
204 None
212 None
205
213
206 Return:
214 Return:
207 None
215 None
208
216
209 Affected:
217 Affected:
210 self.profileIndex
218 self.profileIndex
211 self.datablock
219 self.datablock
212 self.flagIsNewFile
220 self.flagIsNewFile
213 self.flagIsNewBlock
221 self.flagIsNewBlock
214 self.nTotalBlocks
222 self.nTotalBlocks
215
223
216 Exceptions:
224 Exceptions:
217 Si un bloque leido no es un bloque valido
225 Si un bloque leido no es un bloque valido
226 =======
227 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
228 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
229 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
230 es seteado a 0
231
232 Inputs:
233 None
234
235 Return:
236 None
237
238 Affected:
239 self.profileIndex
240 self.datablock
241 self.flagIsNewFile
242 self.flagIsNewBlock
243 self.nTotalBlocks
244
245 Exceptions:
246 Si un bloque leido no es un bloque valido
247 >>>>>>> online_data_hour
218 """
248 """
219
249
220 # if self.server is not None:
250 # if self.server is not None:
221 # self.zBlock = self.receiver.recv()
251 # self.zBlock = self.receiver.recv()
222 # self.zHeader = self.zBlock[:24]
252 # self.zHeader = self.zBlock[:24]
223 # self.zDataBlock = self.zBlock[24:]
253 # self.zDataBlock = self.zBlock[24:]
224 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
254 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
225 # self.processingHeaderObj.profilesPerBlock = 240
255 # self.processingHeaderObj.profilesPerBlock = 240
226 # self.processingHeaderObj.nHeights = 248
256 # self.processingHeaderObj.nHeights = 248
227 # self.systemHeaderObj.nChannels
257 # self.systemHeaderObj.nChannels
228 # else:
258 # else:
229 current_pointer_location = self.fp.tell()
259 current_pointer_location = self.fp.tell()
230 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
260 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
231
261
232 try:
262 try:
233 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
263 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
234 except:
264 except:
235 #print "The read block (%3d) has not enough data" %self.nReadBlocks
265 #print "The read block (%3d) has not enough data" %self.nReadBlocks
236
266
237 if self.waitDataBlock(pointer_location=current_pointer_location):
267 if self.waitDataBlock(pointer_location=current_pointer_location):
238 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
268 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
239 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
269 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
240 # return 0
270 # return 0
241
271
242 #Dimensions : nChannels, nProfiles, nSamples
272 #Dimensions : nChannels, nProfiles, nSamples
243
273
244 junk = numpy.transpose(junk, (2,0,1))
274 junk = numpy.transpose(junk, (2,0,1))
245 self.datablock = junk['real'] + junk['imag']*1j
275 self.datablock = junk['real'] + junk['imag']*1j
246
276
247 self.profileIndex = 0
277 self.profileIndex = 0
248
278
249 self.flagIsNewFile = 0
279 self.flagIsNewFile = 0
250 self.flagIsNewBlock = 1
280 self.flagIsNewBlock = 1
251
281
252 self.nTotalBlocks += 1
282 self.nTotalBlocks += 1
253 self.nReadBlocks += 1
283 self.nReadBlocks += 1
254
284
255 return 1
285 return 1
256
286
257 def getFirstHeader(self):
287 def getFirstHeader(self):
258
288
259 self.getBasicHeader()
289 self.getBasicHeader()
260
290
261 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
291 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
262
292
263 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
293 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
264
294
265 if self.nTxs > 1:
295 if self.nTxs > 1:
266 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
296 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
267
297
268 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
298 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
269
299
270 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
300 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
271 #
301 #
272 # if self.radarControllerHeaderObj.code is not None:
302 # if self.radarControllerHeaderObj.code is not None:
273 #
303 #
274 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
304 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
275 #
305 #
276 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
306 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
277 #
307 #
278 # self.dataOut.code = self.radarControllerHeaderObj.code
308 # self.dataOut.code = self.radarControllerHeaderObj.code
279
309
280 self.dataOut.dtype = self.dtype
310 self.dataOut.dtype = self.dtype
281
311
282 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
312 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
283
313
284 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
314 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
285
315
286 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
316 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
287
317
288 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
318 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
289
319
290 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
320 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
291
321
292 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
322 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
293
323
294 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
324 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
295
325
296 def reshapeData(self):
326 def reshapeData(self):
297
327
298 if self.nTxs < 0:
328 if self.nTxs < 0:
299 return
329 return
300
330
301 if self.nTxs == 1:
331 if self.nTxs == 1:
302 return
332 return
303
333
304 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
334 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
305 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
335 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
306
336
307 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
337 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
308 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
338 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
309
339
310 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
340 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
311
341
312 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
342 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
313 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
343 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
314 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
344 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
315
345
316 return
346 return
317
347
318 def readFirstHeaderFromServer(self):
348 def readFirstHeaderFromServer(self):
319
349
320 self.getFirstHeader()
350 self.getFirstHeader()
321
351
322 self.firstHeaderSize = self.basicHeaderObj.size
352 self.firstHeaderSize = self.basicHeaderObj.size
323
353
324 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
354 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
325 if datatype == 0:
355 if datatype == 0:
326 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
356 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
327 elif datatype == 1:
357 elif datatype == 1:
328 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
358 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
329 elif datatype == 2:
359 elif datatype == 2:
330 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
360 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
331 elif datatype == 3:
361 elif datatype == 3:
332 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
362 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
333 elif datatype == 4:
363 elif datatype == 4:
334 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
364 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
335 elif datatype == 5:
365 elif datatype == 5:
336 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
366 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
337 else:
367 else:
338 raise ValueError, 'Data type was not defined'
368 raise ValueError, 'Data type was not defined'
339
369
340 self.dtype = datatype_str
370 self.dtype = datatype_str
341 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
371 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
342 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
372 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
343 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
373 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
344 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
374 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
345 self.getBlockDimension()
375 self.getBlockDimension()
346
376
347
377
348 def getFromServer(self):
378 def getFromServer(self):
349 self.flagDiscontinuousBlock = 0
379 self.flagDiscontinuousBlock = 0
350 self.profileIndex = 0
380 self.profileIndex = 0
351 self.flagIsNewBlock = 1
381 self.flagIsNewBlock = 1
352 self.dataOut.flagNoData = False
382 self.dataOut.flagNoData = False
353 self.nTotalBlocks += 1
383 self.nTotalBlocks += 1
354 self.nReadBlocks += 1
384 self.nReadBlocks += 1
355 self.blockPointer = 0
385 self.blockPointer = 0
356
386
357 block = self.receiver.recv()
387 block = self.receiver.recv()
358
388
359 self.basicHeaderObj.read(block[self.blockPointer:])
389 self.basicHeaderObj.read(block[self.blockPointer:])
360 self.blockPointer += self.basicHeaderObj.length
390 self.blockPointer += self.basicHeaderObj.length
361 self.systemHeaderObj.read(block[self.blockPointer:])
391 self.systemHeaderObj.read(block[self.blockPointer:])
362 self.blockPointer += self.systemHeaderObj.length
392 self.blockPointer += self.systemHeaderObj.length
363 self.radarControllerHeaderObj.read(block[self.blockPointer:])
393 self.radarControllerHeaderObj.read(block[self.blockPointer:])
364 self.blockPointer += self.radarControllerHeaderObj.length
394 self.blockPointer += self.radarControllerHeaderObj.length
365 self.processingHeaderObj.read(block[self.blockPointer:])
395 self.processingHeaderObj.read(block[self.blockPointer:])
366 self.blockPointer += self.processingHeaderObj.length
396 self.blockPointer += self.processingHeaderObj.length
367 self.readFirstHeaderFromServer()
397 self.readFirstHeaderFromServer()
368
398
369 timestamp = self.basicHeaderObj.get_datatime()
399 timestamp = self.basicHeaderObj.get_datatime()
370 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
400 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
371 current_pointer_location = self.blockPointer
401 current_pointer_location = self.blockPointer
372 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
402 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
373
403
374 try:
404 try:
375 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
405 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
376 except:
406 except:
377 #print "The read block (%3d) has not enough data" %self.nReadBlocks
407 #print "The read block (%3d) has not enough data" %self.nReadBlocks
378 if self.waitDataBlock(pointer_location=current_pointer_location):
408 if self.waitDataBlock(pointer_location=current_pointer_location):
379 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
409 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
380 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
410 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
381 # return 0
411 # return 0
382
412
383 #Dimensions : nChannels, nProfiles, nSamples
413 #Dimensions : nChannels, nProfiles, nSamples
384
414
385 junk = numpy.transpose(junk, (2,0,1))
415 junk = numpy.transpose(junk, (2,0,1))
386 self.datablock = junk['real'] + junk['imag'] * 1j
416 self.datablock = junk['real'] + junk['imag'] * 1j
387 self.profileIndex = 0
417 self.profileIndex = 0
388 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
418 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
389 if self.selBlocktime != None:
419 if self.selBlocktime != None:
390 if self.dataOut.nCohInt is not None:
420 if self.dataOut.nCohInt is not None:
391 nCohInt = self.dataOut.nCohInt
421 nCohInt = self.dataOut.nCohInt
392 else:
422 else:
393 nCohInt = 1
423 nCohInt = 1
394 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
424 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
395 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
425 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
396 datasize = self.dataOut.data.shape[1]
426 datasize = self.dataOut.data.shape[1]
397 if datasize < self.selBlocksize:
427 if datasize < self.selBlocksize:
398 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
428 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
399 buffer[:,:datasize,:] = self.dataOut.data
429 buffer[:,:datasize,:] = self.dataOut.data
400 self.dataOut.data = buffer
430 self.dataOut.data = buffer
401 self.profileIndex = blockIndex
431 self.profileIndex = blockIndex
402
432
403 self.dataOut.flagDataAsBlock = True
433 self.dataOut.flagDataAsBlock = True
404 self.flagIsNewBlock = 1
434 self.flagIsNewBlock = 1
405 self.dataOut.realtime = self.online
435 self.dataOut.realtime = self.online
406
436
407 return self.dataOut.data
437 return self.dataOut.data
408
438
409 def getData(self):
439 def getData(self):
410 """
440 """
441 <<<<<<< HEAD
411 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
442 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
412 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
443 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
413 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
444 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
414 "readNextBlock"
445 "readNextBlock"
415
446
416 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
447 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
417
448
418 Return:
449 Return:
419
450
420 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
451 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
421 es igual al total de perfiles leidos desde el archivo.
452 es igual al total de perfiles leidos desde el archivo.
422
453
423 Si self.getByBlock == False:
454 Si self.getByBlock == False:
424
455
425 self.dataOut.data = buffer[:, thisProfile, :]
456 self.dataOut.data = buffer[:, thisProfile, :]
426
457
427 shape = [nChannels, nHeis]
458 shape = [nChannels, nHeis]
428
459
429 Si self.getByBlock == True:
460 Si self.getByBlock == True:
430
461
431 self.dataOut.data = buffer[:, :, :]
462 self.dataOut.data = buffer[:, :, :]
432
463
433 shape = [nChannels, nProfiles, nHeis]
464 shape = [nChannels, nProfiles, nHeis]
434
465
435 Variables afectadas:
466 Variables afectadas:
436 self.dataOut
467 self.dataOut
437 self.profileIndex
468 self.profileIndex
438
469
439 Affected:
470 Affected:
440 self.dataOut
471 self.dataOut
441 self.profileIndex
472 self.profileIndex
442 self.flagDiscontinuousBlock
473 self.flagDiscontinuousBlock
443 self.flagIsNewBlock
474 self.flagIsNewBlock
475 =======
476 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
477 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
478 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
479 "readNextBlock"
480
481 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
482
483 Return:
484
485 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
486 es igual al total de perfiles leidos desde el archivo.
487
488 Si self.getByBlock == False:
489
490 self.dataOut.data = buffer[:, thisProfile, :]
491
492 shape = [nChannels, nHeis]
493
494 Si self.getByBlock == True:
495
496 self.dataOut.data = buffer[:, :, :]
497
498 shape = [nChannels, nProfiles, nHeis]
499
500 Variables afectadas:
501 self.dataOut
502 self.profileIndex
503
504 Affected:
505 self.dataOut
506 self.profileIndex
507 self.flagDiscontinuousBlock
508 self.flagIsNewBlock
509 >>>>>>> online_data_hour
444 """
510 """
445 if self.flagNoMoreFiles:
511 if self.flagNoMoreFiles:
446 self.dataOut.flagNoData = True
512 self.dataOut.flagNoData = True
447 print 'Process finished'
513 print 'Process finished'
448 return 0
514 return 0
449 self.flagDiscontinuousBlock = 0
515 self.flagDiscontinuousBlock = 0
450 self.flagIsNewBlock = 0
516 self.flagIsNewBlock = 0
451 if self.__hasNotDataInBuffer():
517 if self.__hasNotDataInBuffer():
452 if not( self.readNextBlock() ):
518 if not( self.readNextBlock() ):
453 return 0
519 return 0
454
520
455 self.getFirstHeader()
521 self.getFirstHeader()
456
522
457 self.reshapeData()
523 self.reshapeData()
458 if self.datablock is None:
524 if self.datablock is None:
459 self.dataOut.flagNoData = True
525 self.dataOut.flagNoData = True
460 return 0
526 return 0
461
527
462 if not self.getByBlock:
528 if not self.getByBlock:
463
529
464 """
530 """
531 <<<<<<< HEAD
465 Return profile by profile
532 Return profile by profile
466
533
467 If nTxs > 1 then one profile is divided by nTxs and number of total
534 If nTxs > 1 then one profile is divided by nTxs and number of total
468 blocks is increased by nTxs (nProfiles *= nTxs)
535 blocks is increased by nTxs (nProfiles *= nTxs)
536 =======
537 Return profile by profile
538
539 If nTxs > 1 then one profile is divided by nTxs and number of total
540 blocks is increased by nTxs (nProfiles *= nTxs)
541 >>>>>>> online_data_hour
469 """
542 """
470 self.dataOut.flagDataAsBlock = False
543 self.dataOut.flagDataAsBlock = False
471 self.dataOut.data = self.datablock[:,self.profileIndex,:]
544 self.dataOut.data = self.datablock[:,self.profileIndex,:]
472 self.dataOut.profileIndex = self.profileIndex
545 self.dataOut.profileIndex = self.profileIndex
473
546
474 self.profileIndex += 1
547 self.profileIndex += 1
548 <<<<<<< HEAD
475
549
476 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
550 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
477 # """
551 # """
478 # Return all block
552 # Return all block
479 # """
553 # """
480 # self.dataOut.flagDataAsBlock = True
554 # self.dataOut.flagDataAsBlock = True
481 # self.dataOut.data = self.datablock
555 # self.dataOut.data = self.datablock
482 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
556 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
483 #
557 #
484 # self.profileIndex = self.dataOut.nProfiles
558 # self.profileIndex = self.dataOut.nProfiles
485
559
560 =======
561
562 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
563 # """
564 # Return all block
565 # """
566 # self.dataOut.flagDataAsBlock = True
567 # self.dataOut.data = self.datablock
568 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
569 #
570 # self.profileIndex = self.dataOut.nProfiles
571
572 >>>>>>> online_data_hour
486 else:
573 else:
487 """
574 """
488 Return a block
575 Return a block
489 """
576 """
490 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
577 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
491 if self.selBlocktime != None:
578 if self.selBlocktime != None:
492 if self.dataOut.nCohInt is not None:
579 if self.dataOut.nCohInt is not None:
493 nCohInt = self.dataOut.nCohInt
580 nCohInt = self.dataOut.nCohInt
494 else:
581 else:
495 nCohInt = 1
582 nCohInt = 1
496 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
583 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
497
584
498 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
585 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
499 self.profileIndex += self.selBlocksize
586 self.profileIndex += self.selBlocksize
500 datasize = self.dataOut.data.shape[1]
587 datasize = self.dataOut.data.shape[1]
501
588
502 if datasize < self.selBlocksize:
589 if datasize < self.selBlocksize:
503 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
590 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
504 buffer[:,:datasize,:] = self.dataOut.data
591 buffer[:,:datasize,:] = self.dataOut.data
505
592
506 while datasize < self.selBlocksize: #Not enough profiles to fill the block
593 while datasize < self.selBlocksize: #Not enough profiles to fill the block
507 if not( self.readNextBlock() ):
594 if not( self.readNextBlock() ):
508 return 0
595 return 0
509 self.getFirstHeader()
596 self.getFirstHeader()
510 self.reshapeData()
597 self.reshapeData()
511 if self.datablock is None:
598 if self.datablock is None:
512 self.dataOut.flagNoData = True
599 self.dataOut.flagNoData = True
513 return 0
600 return 0
514 #stack data
601 #stack data
515 blockIndex = self.selBlocksize - datasize
602 blockIndex = self.selBlocksize - datasize
516 datablock1 = self.datablock[:,:blockIndex,:]
603 datablock1 = self.datablock[:,:blockIndex,:]
517
604
518 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
605 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
519 datasize += datablock1.shape[1]
606 datasize += datablock1.shape[1]
520
607
521 self.dataOut.data = buffer
608 self.dataOut.data = buffer
522 self.profileIndex = blockIndex
609 self.profileIndex = blockIndex
523
610
524 self.dataOut.flagDataAsBlock = True
611 self.dataOut.flagDataAsBlock = True
525 self.dataOut.nProfiles = self.dataOut.data.shape[1]
612 self.dataOut.nProfiles = self.dataOut.data.shape[1]
526
613
527 self.dataOut.flagNoData = False
614 self.dataOut.flagNoData = False
528
615
529 self.getBasicHeader()
616 self.getBasicHeader()
530
617
531 self.dataOut.realtime = self.online
618 self.dataOut.realtime = self.online
532
619
533 return self.dataOut.data
620 return self.dataOut.data
534
621
535 class VoltageWriter(JRODataWriter, Operation):
622 class VoltageWriter(JRODataWriter, Operation):
536 """
623 """
537 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
624 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
538 de los datos siempre se realiza por bloques.
625 de los datos siempre se realiza por bloques.
539 """
626 """
540
627
541 ext = ".r"
628 ext = ".r"
542
629
543 optchar = "D"
630 optchar = "D"
544
631
545 shapeBuffer = None
632 shapeBuffer = None
546
633
547
634
548 def __init__(self, **kwargs):
635 def __init__(self, **kwargs):
549 """
636 """
550 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
637 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
551
638
552 Affected:
639 Affected:
553 self.dataOut
640 self.dataOut
554
641
555 Return: None
642 Return: None
556 """
643 """
557 Operation.__init__(self, **kwargs)
644 Operation.__init__(self, **kwargs)
558
645
559 self.nTotalBlocks = 0
646 self.nTotalBlocks = 0
560
647
561 self.profileIndex = 0
648 self.profileIndex = 0
562
649
563 self.isConfig = False
650 self.isConfig = False
564
651
565 self.fp = None
652 self.fp = None
566
653
567 self.flagIsNewFile = 1
654 self.flagIsNewFile = 1
568
655
569 self.blockIndex = 0
656 self.blockIndex = 0
570
657
571 self.flagIsNewBlock = 0
658 self.flagIsNewBlock = 0
572
659
573 self.setFile = None
660 self.setFile = None
574
661
575 self.dtype = None
662 self.dtype = None
576
663
577 self.path = None
664 self.path = None
578
665
579 self.filename = None
666 self.filename = None
580
667
581 self.basicHeaderObj = BasicHeader(LOCALTIME)
668 self.basicHeaderObj = BasicHeader(LOCALTIME)
582
669
583 self.systemHeaderObj = SystemHeader()
670 self.systemHeaderObj = SystemHeader()
584
671
585 self.radarControllerHeaderObj = RadarControllerHeader()
672 self.radarControllerHeaderObj = RadarControllerHeader()
586
673
587 self.processingHeaderObj = ProcessingHeader()
674 self.processingHeaderObj = ProcessingHeader()
588
675
589 def hasAllDataInBuffer(self):
676 def hasAllDataInBuffer(self):
590 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
677 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
591 return 1
678 return 1
592 return 0
679 return 0
593
680
594
681
595 def setBlockDimension(self):
682 def setBlockDimension(self):
596 """
683 """
597 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
684 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
598
685
599 Affected:
686 Affected:
600 self.shape_spc_Buffer
687 self.shape_spc_Buffer
601 self.shape_cspc_Buffer
688 self.shape_cspc_Buffer
602 self.shape_dc_Buffer
689 self.shape_dc_Buffer
603
690
604 Return: None
691 Return: None
605 """
692 """
606 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
693 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
607 self.processingHeaderObj.nHeights,
694 self.processingHeaderObj.nHeights,
608 self.systemHeaderObj.nChannels)
695 self.systemHeaderObj.nChannels)
609
696
610 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
697 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
611 self.processingHeaderObj.profilesPerBlock,
698 self.processingHeaderObj.profilesPerBlock,
612 self.processingHeaderObj.nHeights),
699 self.processingHeaderObj.nHeights),
613 dtype=numpy.dtype('complex64'))
700 dtype=numpy.dtype('complex64'))
614
701
615 def writeBlock(self):
702 def writeBlock(self):
616 """
703 """
617 Escribe el buffer en el file designado
704 Escribe el buffer en el file designado
618
705
619 Affected:
706 Affected:
620 self.profileIndex
707 self.profileIndex
621 self.flagIsNewFile
708 self.flagIsNewFile
622 self.flagIsNewBlock
709 self.flagIsNewBlock
623 self.nTotalBlocks
710 self.nTotalBlocks
624 self.blockIndex
711 self.blockIndex
625
712
626 Return: None
713 Return: None
627 """
714 """
628 data = numpy.zeros( self.shapeBuffer, self.dtype )
715 data = numpy.zeros( self.shapeBuffer, self.dtype )
629
716
630 junk = numpy.transpose(self.datablock, (1,2,0))
717 junk = numpy.transpose(self.datablock, (1,2,0))
631
718
632 data['real'] = junk.real
719 data['real'] = junk.real
633 data['imag'] = junk.imag
720 data['imag'] = junk.imag
634
721
635 data = data.reshape( (-1) )
722 data = data.reshape( (-1) )
636
723
637 data.tofile( self.fp )
724 data.tofile( self.fp )
638
725
639 self.datablock.fill(0)
726 self.datablock.fill(0)
640
727
641 self.profileIndex = 0
728 self.profileIndex = 0
642 self.flagIsNewFile = 0
729 self.flagIsNewFile = 0
643 self.flagIsNewBlock = 1
730 self.flagIsNewBlock = 1
644
731
645 self.blockIndex += 1
732 self.blockIndex += 1
646 self.nTotalBlocks += 1
733 self.nTotalBlocks += 1
647
734
648 # print "[Writing] Block = %04d" %self.blockIndex
735 # print "[Writing] Block = %04d" %self.blockIndex
649
736
650 def putData(self):
737 def putData(self):
651 """
738 """
652 Setea un bloque de datos y luego los escribe en un file
739 Setea un bloque de datos y luego los escribe en un file
653
740
654 Affected:
741 Affected:
655 self.flagIsNewBlock
742 self.flagIsNewBlock
656 self.profileIndex
743 self.profileIndex
657
744
658 Return:
745 Return:
659 0 : Si no hay data o no hay mas files que puedan escribirse
746 0 : Si no hay data o no hay mas files que puedan escribirse
660 1 : Si se escribio la data de un bloque en un file
747 1 : Si se escribio la data de un bloque en un file
661 """
748 """
662 if self.dataOut.flagNoData:
749 if self.dataOut.flagNoData:
663 return 0
750 return 0
664
751
665 self.flagIsNewBlock = 0
752 self.flagIsNewBlock = 0
666
753
667 if self.dataOut.flagDiscontinuousBlock:
754 if self.dataOut.flagDiscontinuousBlock:
668 self.datablock.fill(0)
755 self.datablock.fill(0)
669 self.profileIndex = 0
756 self.profileIndex = 0
670 self.setNextFile()
757 self.setNextFile()
671
758
672 if self.profileIndex == 0:
759 if self.profileIndex == 0:
673 self.setBasicHeader()
760 self.setBasicHeader()
674
761
675 self.datablock[:,self.profileIndex,:] = self.dataOut.data
762 self.datablock[:,self.profileIndex,:] = self.dataOut.data
676
763
677 self.profileIndex += 1
764 self.profileIndex += 1
678
765
679 if self.hasAllDataInBuffer():
766 if self.hasAllDataInBuffer():
680 #if self.flagIsNewFile:
767 #if self.flagIsNewFile:
681 self.writeNextBlock()
768 self.writeNextBlock()
682 # self.setFirstHeader()
769 # self.setFirstHeader()
683
770
684 return 1
771 return 1
685
772
686 def __getBlockSize(self):
773 def __getBlockSize(self):
687 '''
774 '''
688 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
775 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
689 '''
776 '''
690
777
691 dtype_width = self.getDtypeWidth()
778 dtype_width = self.getDtypeWidth()
692
779
693 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
780 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
694
781
695 return blocksize
782 return blocksize
696
783
697 def setFirstHeader(self):
784 def setFirstHeader(self):
698
785
699 """
786 """
700 Obtiene una copia del First Header
787 Obtiene una copia del First Header
701
788
702 Affected:
789 Affected:
703 self.systemHeaderObj
790 self.systemHeaderObj
704 self.radarControllerHeaderObj
791 self.radarControllerHeaderObj
705 self.dtype
792 self.dtype
706
793
707 Return:
794 Return:
708 None
795 None
709 """
796 """
710
797
711 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
798 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
712 self.systemHeaderObj.nChannels = self.dataOut.nChannels
799 self.systemHeaderObj.nChannels = self.dataOut.nChannels
713 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
800 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
714
801
715 self.processingHeaderObj.dtype = 0 # Voltage
802 self.processingHeaderObj.dtype = 0 # Voltage
716 self.processingHeaderObj.blockSize = self.__getBlockSize()
803 self.processingHeaderObj.blockSize = self.__getBlockSize()
717 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
804 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
718 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
805 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
719 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
806 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
720 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
807 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
721 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
808 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
722 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
809 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
723
810
724 if self.dataOut.code is not None:
811 if self.dataOut.code is not None:
725 self.processingHeaderObj.code = self.dataOut.code
812 self.processingHeaderObj.code = self.dataOut.code
726 self.processingHeaderObj.nCode = self.dataOut.nCode
813 self.processingHeaderObj.nCode = self.dataOut.nCode
727 self.processingHeaderObj.nBaud = self.dataOut.nBaud
814 self.processingHeaderObj.nBaud = self.dataOut.nBaud
728
815
729 if self.processingHeaderObj.nWindows != 0:
816 if self.processingHeaderObj.nWindows != 0:
730 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
817 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
731 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
818 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
732 self.processingHeaderObj.nHeights = self.dataOut.nHeights
819 self.processingHeaderObj.nHeights = self.dataOut.nHeights
733 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
820 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
734
821
735 self.processingHeaderObj.processFlags = self.getProcessFlags()
822 self.processingHeaderObj.processFlags = self.getProcessFlags()
736
823
737 self.setBasicHeader()
824 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -1,1007 +1,1021
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13
13
14 import time
14 import time
15
15
16 import threading
16 import threading
17 Thread = threading.Thread
17 Thread = threading.Thread
18
18
19 # try:
19 # try:
20 # from gevent import sleep
20 # from gevent import sleep
21 # except:
21 # except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25
25
26 class Remote(Thread):
26 class Remote(Thread):
27 """
27 """
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 used to upload or download files remotely.
29 used to upload or download files remotely.
30
30
31 Non-standard Python modules used:
31 Non-standard Python modules used:
32 None
32 None
33
33
34 Written by:
34 Written by:
35 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
35 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 """
36 """
37
37
38 server = None
38 server = None
39 username = None
39 username = None
40 password = None
40 password = None
41 remotefolder = None
41 remotefolder = None
42
42
43 period = 60
43 period = 60
44 fileList = []
44 fileList = []
45 bussy = False
45 bussy = False
46
46
47 def __init__(self, server, username, password, remotefolder, period=60):
47 def __init__(self, server, username, password, remotefolder, period=60):
48
48
49 Thread.__init__(self)
49 Thread.__init__(self)
50
50
51 self.setDaemon(True)
51 self.setDaemon(True)
52
52
53 self.status = 0
53 self.status = 0
54
54
55 self.__server = server
55 self.__server = server
56 self.__username = username
56 self.__username = username
57 self.__password = password
57 self.__password = password
58 self.__remotefolder = remotefolder
58 self.__remotefolder = remotefolder
59
59
60 self.period = period
60 self.period = period
61
61
62 self.fileList = []
62 self.fileList = []
63 self.bussy = False
63 self.bussy = False
64
64
65 self.stopFlag = False
65 self.stopFlag = False
66
66
67 print "[Remote Server] Opening server: %s" %self.__server
67 print "[Remote Server] Opening server: %s" %self.__server
68 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
68 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
69 print "[Remote Server] %s server was opened successfully" %self.__server
69 print "[Remote Server] %s server was opened successfully" %self.__server
70
70
71 self.close()
71 self.close()
72
72
73 self.mutex = threading.Lock()
73 self.mutex = threading.Lock()
74
74
75 def stop(self):
75 def stop(self):
76
76
77 self.stopFlag = True
77 self.stopFlag = True
78 self.join(10)
78 self.join(10)
79
79
80 def open(self):
80 def open(self):
81 """
81 """
82 Connect to server and create a connection class (FTP or SSH) to remote server.
82 Connect to server and create a connection class (FTP or SSH) to remote server.
83 """
83 """
84 raise NotImplementedError, "Implement this method in child class"
84 raise NotImplementedError, "Implement this method in child class"
85
85
86 def close(self):
86 def close(self):
87 """
87 """
88 Close connection to server
88 Close connection to server
89 """
89 """
90 raise NotImplementedError, "Implement this method in child class"
90 raise NotImplementedError, "Implement this method in child class"
91
91
92 def mkdir(self, remotefolder):
92 def mkdir(self, remotefolder):
93 """
93 """
94 Create a folder remotely
94 Create a folder remotely
95 """
95 """
96 raise NotImplementedError, "Implement this method in child class"
96 raise NotImplementedError, "Implement this method in child class"
97
97
98 def cd(self, remotefolder):
98 def cd(self, remotefolder):
99 """
99 """
100 Change working directory in remote server
100 Change working directory in remote server
101 """
101 """
102 raise NotImplementedError, "Implement this method in child class"
102 raise NotImplementedError, "Implement this method in child class"
103
103
104 def download(self, filename, localfolder=None):
104 def download(self, filename, localfolder=None):
105 """
105 """
106 Download a file from server to local host
106 Download a file from server to local host
107 """
107 """
108 raise NotImplementedError, "Implement this method in child class"
108 raise NotImplementedError, "Implement this method in child class"
109
109
110 def sendFile(self, fullfilename):
110 def sendFile(self, fullfilename):
111 """
111 """
112 sendFile method is used to upload a local file to the current directory in remote server
112 sendFile method is used to upload a local file to the current directory in remote server
113
113
114 Inputs:
114 Inputs:
115 fullfilename - full path name of local file to store in remote directory
115 fullfilename - full path name of local file to store in remote directory
116
116
117 Returns:
117 Returns:
118 0 in error case else 1
118 0 in error case else 1
119 """
119 """
120 raise NotImplementedError, "Implement this method in child class"
120 raise NotImplementedError, "Implement this method in child class"
121
121
122 def upload(self, fullfilename, remotefolder=None):
122 def upload(self, fullfilename, remotefolder=None):
123 """
123 """
124 upload method is used to upload a local file to remote directory. This method changes
124 upload method is used to upload a local file to remote directory. This method changes
125 working directory before sending a file.
125 working directory before sending a file.
126
126
127 Inputs:
127 Inputs:
128 fullfilename - full path name of local file to store in remote directory
128 fullfilename - full path name of local file to store in remote directory
129
129
130 remotefolder - remote directory
130 remotefolder - remote directory
131
131
132 Returns:
132 Returns:
133 0 in error case else 1
133 0 in error case else 1
134 """
134 """
135 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
135 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
136
136
137 if not self.status:
137 if not self.status:
138 return 0
138 return 0
139
139
140 if remotefolder == None:
140 if remotefolder == None:
141 remotefolder = self.remotefolder
141 remotefolder = self.remotefolder
142
142
143 if not self.cd(remotefolder):
143 if not self.cd(remotefolder):
144 return 0
144 return 0
145
145
146 if not self.sendFile(fullfilename):
146 if not self.sendFile(fullfilename):
147 print "[Remote Server] Error uploading file %s" %fullfilename
147 print "[Remote Server] Error uploading file %s" %fullfilename
148 return 0
148 return 0
149
149
150 print "[Remote Server] upload finished successfully"
150 print "[Remote Server] upload finished successfully"
151
151
152 return 1
152 return 1
153
153
154 def delete(self, filename):
154 def delete(self, filename):
155 """
155 """
156 Remove a file from remote server
156 Remove a file from remote server
157 """
157 """
158 pass
158 pass
159
159
160 def updateFileList(self, fileList):
160 def updateFileList(self, fileList):
161 """
161 """
162 Remove a file from remote server
162 Remove a file from remote server
163 """
163 """
164
164
165 if fileList == self.fileList:
165 if fileList == self.fileList:
166 return 0
166 return 0
167
167
168 self.mutex.acquire()
168 self.mutex.acquire()
169 <<<<<<< HEAD
169 # init = time.time()
170 # init = time.time()
170 #
171 #
171 # while(self.bussy):
172 # while(self.bussy):
172 # sleep(0.1)
173 # sleep(0.1)
173 # if time.time() - init > 2*self.period:
174 # if time.time() - init > 2*self.period:
174 # return 0
175 # return 0
176 =======
177 # init = time.time()
178 #
179 # while(self.bussy):
180 # sleep(0.1)
181 # if time.time() - init > 2*self.period:
182 # return 0
183 >>>>>>> online_data_hour
175
184
176 self.fileList = fileList
185 self.fileList = fileList
177 self.mutex.release()
186 self.mutex.release()
178 return 1
187 return 1
179
188
180 def run(self):
189 def run(self):
181
190
182 if not self.status:
191 if not self.status:
183 print "Finishing FTP service"
192 print "Finishing FTP service"
184 return
193 return
185
194
186 if not self.cd(self.remotefolder):
195 if not self.cd(self.remotefolder):
187 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
196 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
188
197
189 while True:
198 while True:
190
199
191 for i in range(self.period):
200 for i in range(self.period):
192 if self.stopFlag:
201 if self.stopFlag:
193 break
202 break
194 sleep(1)
203 sleep(1)
195
204
196 if self.stopFlag:
205 if self.stopFlag:
197 break
206 break
198
207
208 <<<<<<< HEAD
199 # self.bussy = True
209 # self.bussy = True
210 =======
211 # self.bussy = True
212 >>>>>>> online_data_hour
200 self.mutex.acquire()
213 self.mutex.acquire()
201
214
202 print "[Remote Server] Opening %s" %self.__server
215 print "[Remote Server] Opening %s" %self.__server
203 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
216 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
204 self.mutex.release()
217 self.mutex.release()
205 continue
218 continue
206
219
207 for thisFile in self.fileList:
220 for thisFile in self.fileList:
208 self.upload(thisFile, self.remotefolder)
221 self.upload(thisFile, self.remotefolder)
209
222
210 print "[Remote Server] Closing %s" %self.__server
223 print "[Remote Server] Closing %s" %self.__server
211 self.close()
224 self.close()
212
225
213 self.mutex.release()
226 self.mutex.release()
214 # self.bussy = False
227 # self.bussy = False
215
228
216 print "[Remote Server] Thread stopped successfully"
229 print "[Remote Server] Thread stopped successfully"
217
230
218 class FTPClient(Remote):
231 class FTPClient(Remote):
219
232
220 __ftpClientObj = None
233 __ftpClientObj = None
221
234
222 def __init__(self, server, username, password, remotefolder, period=60):
235 def __init__(self, server, username, password, remotefolder, period=60):
223 """
236 """
224 """
237 """
225 Remote.__init__(self, server, username, password, remotefolder, period)
238 Remote.__init__(self, server, username, password, remotefolder, period)
226
239
227 def open(self, server, username, password, remotefolder):
240 def open(self, server, username, password, remotefolder):
228
241
229 """
242 """
230 This method is used to set FTP parameters and establish a connection to remote server
243 This method is used to set FTP parameters and establish a connection to remote server
231
244
232 Inputs:
245 Inputs:
233 server - remote server IP Address
246 server - remote server IP Address
234
247
235 username - remote server Username
248 username - remote server Username
236
249
237 password - remote server password
250 password - remote server password
238
251
239 remotefolder - remote server current working directory
252 remotefolder - remote server current working directory
240
253
241 Return:
254 Return:
242 Boolean - Returns 1 if a connection has been established, 0 otherwise
255 Boolean - Returns 1 if a connection has been established, 0 otherwise
243
256
244 Affects:
257 Affects:
245 self.status - in case of error or fail connection this parameter is set to 0 else 1
258 self.status - in case of error or fail connection this parameter is set to 0 else 1
246
259
247 """
260 """
248
261
249 if server == None:
262 if server == None:
250 raise ValueError, "FTP server should be defined"
263 raise ValueError, "FTP server should be defined"
251
264
252 if username == None:
265 if username == None:
253 raise ValueError, "FTP username should be defined"
266 raise ValueError, "FTP username should be defined"
254
267
255 if password == None:
268 if password == None:
256 raise ValueError, "FTP password should be defined"
269 raise ValueError, "FTP password should be defined"
257
270
258 if remotefolder == None:
271 if remotefolder == None:
259 raise ValueError, "FTP remote folder should be defined"
272 raise ValueError, "FTP remote folder should be defined"
260
273
261 try:
274 try:
262 ftpClientObj = ftplib.FTP(server)
275 ftpClientObj = ftplib.FTP(server)
263 except ftplib.all_errors, e:
276 except ftplib.all_errors, e:
264 print "[FTP Server]: FTP server connection fail: %s" %server
277 print "[FTP Server]: FTP server connection fail: %s" %server
265 print "[FTP Server]:", e
278 print "[FTP Server]:", e
266 self.status = 0
279 self.status = 0
267 return 0
280 return 0
268
281
269 try:
282 try:
270 ftpClientObj.login(username, password)
283 ftpClientObj.login(username, password)
271 except ftplib.all_errors:
284 except ftplib.all_errors:
272 print "[FTP Server]: FTP username or password are incorrect"
285 print "[FTP Server]: FTP username or password are incorrect"
273 self.status = 0
286 self.status = 0
274 return 0
287 return 0
275
288
276 if remotefolder == None:
289 if remotefolder == None:
277 remotefolder = ftpClientObj.pwd()
290 remotefolder = ftpClientObj.pwd()
278 else:
291 else:
279 try:
292 try:
280 ftpClientObj.cwd(remotefolder)
293 ftpClientObj.cwd(remotefolder)
281 except ftplib.all_errors:
294 except ftplib.all_errors:
282 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
295 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
283 remotefolder = ftpClientObj.pwd()
296 remotefolder = ftpClientObj.pwd()
284
297
285 self.server = server
298 self.server = server
286 self.username = username
299 self.username = username
287 self.password = password
300 self.password = password
288 self.remotefolder = remotefolder
301 self.remotefolder = remotefolder
289 self.__ftpClientObj = ftpClientObj
302 self.__ftpClientObj = ftpClientObj
290 self.status = 1
303 self.status = 1
291
304
292 return 1
305 return 1
293
306
294 def close(self):
307 def close(self):
295 """
308 """
296 Close connection to remote server
309 Close connection to remote server
297 """
310 """
298 if not self.status:
311 if not self.status:
299 return 0
312 return 0
300
313
301 self.__ftpClientObj.close()
314 self.__ftpClientObj.close()
302
315
303 def mkdir(self, remotefolder):
316 def mkdir(self, remotefolder):
304 """
317 """
305 mkdir is used to make a new directory in remote server
318 mkdir is used to make a new directory in remote server
306
319
307 Input:
320 Input:
308 remotefolder - directory name
321 remotefolder - directory name
309
322
310 Return:
323 Return:
311 0 in error case else 1
324 0 in error case else 1
312 """
325 """
313 if not self.status:
326 if not self.status:
314 return 0
327 return 0
315
328
316 try:
329 try:
317 self.__ftpClientObj.mkd(dirname)
330 self.__ftpClientObj.mkd(dirname)
318 except ftplib.all_errors:
331 except ftplib.all_errors:
319 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
332 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
320 return 0
333 return 0
321
334
322 return 1
335 return 1
323
336
324 def cd(self, remotefolder):
337 def cd(self, remotefolder):
325 """
338 """
326 cd is used to change remote working directory on server
339 cd is used to change remote working directory on server
327
340
328 Input:
341 Input:
329 remotefolder - current working directory
342 remotefolder - current working directory
330
343
331 Affects:
344 Affects:
332 self.remotefolder
345 self.remotefolder
333
346
334 Return:
347 Return:
335 0 in case of error else 1
348 0 in case of error else 1
336 """
349 """
337 if not self.status:
350 if not self.status:
338 return 0
351 return 0
339
352
340 if remotefolder == self.remotefolder:
353 if remotefolder == self.remotefolder:
341 return 1
354 return 1
342
355
343 try:
356 try:
344 self.__ftpClientObj.cwd(remotefolder)
357 self.__ftpClientObj.cwd(remotefolder)
345 except ftplib.all_errors:
358 except ftplib.all_errors:
346 print '[FTP Server]: Error changing to %s' %remotefolder
359 print '[FTP Server]: Error changing to %s' %remotefolder
347 print '[FTP Server]: Trying to create remote folder'
360 print '[FTP Server]: Trying to create remote folder'
348
361
349 if not self.mkdir(remotefolder):
362 if not self.mkdir(remotefolder):
350 print '[FTP Server]: Remote folder could not be created'
363 print '[FTP Server]: Remote folder could not be created'
351 return 0
364 return 0
352
365
353 try:
366 try:
354 self.__ftpClientObj.cwd(remotefolder)
367 self.__ftpClientObj.cwd(remotefolder)
355 except ftplib.all_errors:
368 except ftplib.all_errors:
356 return 0
369 return 0
357
370
358 self.remotefolder = remotefolder
371 self.remotefolder = remotefolder
359
372
360 return 1
373 return 1
361
374
362 def sendFile(self, fullfilename):
375 def sendFile(self, fullfilename):
363
376
364 if not self.status:
377 if not self.status:
365 return 0
378 return 0
366
379
367 fp = open(fullfilename, 'rb')
380 fp = open(fullfilename, 'rb')
368
381
369 filename = os.path.basename(fullfilename)
382 filename = os.path.basename(fullfilename)
370
383
371 command = "STOR %s" %filename
384 command = "STOR %s" %filename
372
385
373 try:
386 try:
374 self.__ftpClientObj.storbinary(command, fp)
387 self.__ftpClientObj.storbinary(command, fp)
375 except ftplib.all_errors, e:
388 except ftplib.all_errors, e:
376 print "[FTP Server]:", e
389 print "[FTP Server]:", e
377 return 0
390 return 0
378
391
379 try:
392 try:
380 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
393 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
381 except ftplib.all_errors, e:
394 except ftplib.all_errors, e:
382 print "[FTP Server]:", e
395 print "[FTP Server]:", e
383
396
384 fp.close()
397 fp.close()
385
398
386 return 1
399 return 1
387
400
388 class SSHClient(Remote):
401 class SSHClient(Remote):
389
402
390 __sshClientObj = None
403 __sshClientObj = None
391 __scpClientObj = None
404 __scpClientObj = None
392
405
393 def __init__(self, server, username, password, remotefolder, period=60):
406 def __init__(self, server, username, password, remotefolder, period=60):
394 """
407 """
395 """
408 """
396 Remote.__init__(self, server, username, password, remotefolder, period)
409 Remote.__init__(self, server, username, password, remotefolder, period)
397
410
398 def open(self, server, username, password, remotefolder, port=22):
411 def open(self, server, username, password, remotefolder, port=22):
399
412
400 """
413 """
401 This method is used to set SSH parameters and establish a connection to a remote server
414 This method is used to set SSH parameters and establish a connection to a remote server
402
415
403 Inputs:
416 Inputs:
404 server - remote server IP Address
417 server - remote server IP Address
405
418
406 username - remote server Username
419 username - remote server Username
407
420
408 password - remote server password
421 password - remote server password
409
422
410 remotefolder - remote server current working directory
423 remotefolder - remote server current working directory
411
424
412 Return: void
425 Return: void
413
426
414 Affects:
427 Affects:
415 self.status - in case of error or fail connection this parameter is set to 0 else 1
428 self.status - in case of error or fail connection this parameter is set to 0 else 1
416
429
417 """
430 """
418 import socket
431 import socket
419
432
420 if server == None:
433 if server == None:
421 raise ValueError, "SSH server should be defined"
434 raise ValueError, "SSH server should be defined"
422
435
423 if username == None:
436 if username == None:
424 raise ValueError, "SSH username should be defined"
437 raise ValueError, "SSH username should be defined"
425
438
426 if password == None:
439 if password == None:
427 raise ValueError, "SSH password should be defined"
440 raise ValueError, "SSH password should be defined"
428
441
429 if remotefolder == None:
442 if remotefolder == None:
430 raise ValueError, "SSH remote folder should be defined"
443 raise ValueError, "SSH remote folder should be defined"
431
444
432 sshClientObj = paramiko.SSHClient()
445 sshClientObj = paramiko.SSHClient()
433
446
434 sshClientObj.load_system_host_keys()
447 sshClientObj.load_system_host_keys()
435 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
448 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
436
449
437 self.status = 0
450 self.status = 0
438 try:
451 try:
439 sshClientObj.connect(server, username=username, password=password, port=port)
452 sshClientObj.connect(server, username=username, password=password, port=port)
440 except paramiko.AuthenticationException, e:
453 except paramiko.AuthenticationException, e:
441 # print "SSH username or password are incorrect: %s"
454 # print "SSH username or password are incorrect: %s"
442 print "[SSH Server]:", e
455 print "[SSH Server]:", e
443 return 0
456 return 0
444 except SSHException, e:
457 except SSHException, e:
445 print "[SSH Server]:", e
458 print "[SSH Server]:", e
446 return 0
459 return 0
447 except socket.error:
460 except socket.error:
448 self.status = 0
461 self.status = 0
449 print "[SSH Server]:", e
462 print "[SSH Server]:", e
450 return 0
463 return 0
451
464
452 self.status = 1
465 self.status = 1
453 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
466 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
454
467
455 if remotefolder == None:
468 if remotefolder == None:
456 remotefolder = self.pwd()
469 remotefolder = self.pwd()
457
470
458 self.server = server
471 self.server = server
459 self.username = username
472 self.username = username
460 self.password = password
473 self.password = password
461 self.__sshClientObj = sshClientObj
474 self.__sshClientObj = sshClientObj
462 self.__scpClientObj = scpClientObj
475 self.__scpClientObj = scpClientObj
463 self.status = 1
476 self.status = 1
464
477
465 if not self.cd(remotefolder):
478 if not self.cd(remotefolder):
466 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
479 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
467 return 0
480 return 0
468
481
469 self.remotefolder = remotefolder
482 self.remotefolder = remotefolder
470
483
471 return 1
484 return 1
472
485
473 def close(self):
486 def close(self):
474 """
487 """
475 Close connection to remote server
488 Close connection to remote server
476 """
489 """
477 if not self.status:
490 if not self.status:
478 return 0
491 return 0
479
492
480 self.__scpClientObj.close()
493 self.__scpClientObj.close()
481 self.__sshClientObj.close()
494 self.__sshClientObj.close()
482
495
483 def __execute(self, command):
496 def __execute(self, command):
484 """
497 """
485 __execute a command on remote server
498 __execute a command on remote server
486
499
487 Input:
500 Input:
488 command - Exmaple 'ls -l'
501 command - Exmaple 'ls -l'
489
502
490 Return:
503 Return:
491 0 in error case else 1
504 0 in error case else 1
492 """
505 """
493 if not self.status:
506 if not self.status:
494 return 0
507 return 0
495
508
496 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
509 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
497
510
498 result = stderr.readlines()
511 result = stderr.readlines()
499 if len(result) > 1:
512 if len(result) > 1:
500 return 0
513 return 0
501
514
502 result = stdout.readlines()
515 result = stdout.readlines()
503 if len(result) > 1:
516 if len(result) > 1:
504 return result[0][:-1]
517 return result[0][:-1]
505
518
506 return 1
519 return 1
507
520
508 def mkdir(self, remotefolder):
521 def mkdir(self, remotefolder):
509 """
522 """
510 mkdir is used to make a new directory in remote server
523 mkdir is used to make a new directory in remote server
511
524
512 Input:
525 Input:
513 remotefolder - directory name
526 remotefolder - directory name
514
527
515 Return:
528 Return:
516 0 in error case else 1
529 0 in error case else 1
517 """
530 """
518
531
519 command = 'mkdir %s' %remotefolder
532 command = 'mkdir %s' %remotefolder
520
533
521 return self.__execute(command)
534 return self.__execute(command)
522
535
523 def pwd(self):
536 def pwd(self):
524
537
525 command = 'pwd'
538 command = 'pwd'
526
539
527 return self.__execute(command)
540 return self.__execute(command)
528
541
529 def cd(self, remotefolder):
542 def cd(self, remotefolder):
530 """
543 """
531 cd is used to change remote working directory on server
544 cd is used to change remote working directory on server
532
545
533 Input:
546 Input:
534 remotefolder - current working directory
547 remotefolder - current working directory
535
548
536 Affects:
549 Affects:
537 self.remotefolder
550 self.remotefolder
538
551
539 Return:
552 Return:
540 0 in case of error else 1
553 0 in case of error else 1
541 """
554 """
542 if not self.status:
555 if not self.status:
543 return 0
556 return 0
544
557
545 if remotefolder == self.remotefolder:
558 if remotefolder == self.remotefolder:
546 return 1
559 return 1
547
560
548 chk_command = "cd %s; pwd" %remotefolder
561 chk_command = "cd %s; pwd" %remotefolder
549 mkdir_command = "mkdir %s" %remotefolder
562 mkdir_command = "mkdir %s" %remotefolder
550
563
551 if not self.__execute(chk_command):
564 if not self.__execute(chk_command):
552 if not self.__execute(mkdir_command):
565 if not self.__execute(mkdir_command):
553 self.remotefolder = None
566 self.remotefolder = None
554 return 0
567 return 0
555
568
556 self.remotefolder = remotefolder
569 self.remotefolder = remotefolder
557
570
558 return 1
571 return 1
559
572
560 def sendFile(self, fullfilename):
573 def sendFile(self, fullfilename):
561
574
562 if not self.status:
575 if not self.status:
563 return 0
576 return 0
564
577
565 try:
578 try:
566 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
579 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
567 except scp.ScpError, e:
580 except scp.ScpError, e:
568 print "[SSH Server]", str(e)
581 print "[SSH Server]", str(e)
569 return 0
582 return 0
570
583
571 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
584 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
572 command = 'chmod 775 %s' %remotefile
585 command = 'chmod 775 %s' %remotefile
573
586
574 return self.__execute(command)
587 return self.__execute(command)
575
588
576 class SendToServer(ProcessingUnit):
589 class SendToServer(ProcessingUnit):
577
590
578 def __init__(self, **kwargs):
591 def __init__(self, **kwargs):
579
592
580 ProcessingUnit.__init__(self, **kwargs)
593 ProcessingUnit.__init__(self, **kwargs)
581
594
582 self.isConfig = False
595 self.isConfig = False
583 self.clientObj = None
596 self.clientObj = None
584
597
585 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
598 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
586
599
587 self.clientObj = None
600 self.clientObj = None
588 self.localfolder = localfolder
601 self.localfolder = localfolder
589 self.ext = ext
602 self.ext = ext
590 self.period = period
603 self.period = period
591
604
592 if str.lower(protocol) == 'ftp':
605 if str.lower(protocol) == 'ftp':
593 self.clientObj = FTPClient(server, username, password, remotefolder, period)
606 self.clientObj = FTPClient(server, username, password, remotefolder, period)
594
607
595 if str.lower(protocol) == 'ssh':
608 if str.lower(protocol) == 'ssh':
596 self.clientObj = SSHClient(server, username, password, remotefolder, period)
609 self.clientObj = SSHClient(server, username, password, remotefolder, period)
597
610
598 if not self.clientObj:
611 if not self.clientObj:
599 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
612 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
600
613
601 self.clientObj.start()
614 self.clientObj.start()
602
615
603 def findFiles(self):
616 def findFiles(self):
604
617
605 if not type(self.localfolder) == list:
618 if not type(self.localfolder) == list:
606 folderList = [self.localfolder]
619 folderList = [self.localfolder]
607 else:
620 else:
608 folderList = self.localfolder
621 folderList = self.localfolder
609
622
610 #Remove duplicate items
623 #Remove duplicate items
611 folderList = list(set(folderList))
624 folderList = list(set(folderList))
612
625
613 fullfilenameList = []
626 fullfilenameList = []
614
627
615 for thisFolder in folderList:
628 for thisFolder in folderList:
616
629
617 print "[Remote Server]: Searching files on %s" %thisFolder
630 print "[Remote Server]: Searching files on %s" %thisFolder
618
631
619 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
632 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
620
633
621 if len(filenameList) < 1:
634 if len(filenameList) < 1:
635
622 continue
636 continue
623
637
624 for thisFile in filenameList:
638 for thisFile in filenameList:
625 fullfilename = os.path.join(thisFolder, thisFile)
639 fullfilename = os.path.join(thisFolder, thisFile)
626
640
627 if fullfilename in fullfilenameList:
641 if fullfilename in fullfilenameList:
628 continue
642 continue
629
643
630 #Only files modified in the last 30 minutes are considered
644 #Only files modified in the last 30 minutes are considered
631 if os.path.getmtime(fullfilename) < time.time() - 30*60:
645 if os.path.getmtime(fullfilename) < time.time() - 30*60:
632 continue
646 continue
633
647
634 fullfilenameList.append(fullfilename)
648 fullfilenameList.append(fullfilename)
635
649
636 return fullfilenameList
650 return fullfilenameList
637
651
638 def run(self, **kwargs):
652 def run(self, **kwargs):
639 if not self.isConfig:
653 if not self.isConfig:
640 self.init = time.time()
654 self.init = time.time()
641 self.setup(**kwargs)
655 self.setup(**kwargs)
642 self.isConfig = True
656 self.isConfig = True
643
657
644 if not self.clientObj.is_alive():
658 if not self.clientObj.is_alive():
645 print "[Remote Server]: Restarting connection "
659 print "[Remote Server]: Restarting connection "
646 self.setup(**kwargs)
660 self.setup(**kwargs)
647
661
648 if time.time() - self.init >= self.period:
662 if time.time() - self.init >= self.period:
649 fullfilenameList = self.findFiles()
663 fullfilenameList = self.findFiles()
650
664
651 if self.clientObj.updateFileList(fullfilenameList):
665 if self.clientObj.updateFileList(fullfilenameList):
652 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
666 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
653 self.init = time.time()
667 self.init = time.time()
654
668
655 def close(self):
669 def close(self):
656 print "[Remote Server] Stopping thread"
670 print "[Remote Server] Stopping thread"
657 self.clientObj.stop()
671 self.clientObj.stop()
658
672
659
673
660 class FTP(object):
674 class FTP(object):
661 """
675 """
662 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
676 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
663
677
664 Non-standard Python modules used: None
678 Non-standard Python modules used: None
665
679
666 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
680 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
667 """
681 """
668
682
669 def __init__(self,server = None, username=None, password=None, remotefolder=None):
683 def __init__(self,server = None, username=None, password=None, remotefolder=None):
670 """
684 """
671 This method is used to setting parameters for FTP and establishing connection to remote server
685 This method is used to setting parameters for FTP and establishing connection to remote server
672
686
673 Inputs:
687 Inputs:
674 server - remote server IP Address
688 server - remote server IP Address
675
689
676 username - remote server Username
690 username - remote server Username
677
691
678 password - remote server password
692 password - remote server password
679
693
680 remotefolder - remote server current working directory
694 remotefolder - remote server current working directory
681
695
682 Return: void
696 Return: void
683
697
684 Affects:
698 Affects:
685 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
699 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
686
700
687 self.folderList - sub-folder list of remote folder
701 self.folderList - sub-folder list of remote folder
688
702
689 self.fileList - file list of remote folder
703 self.fileList - file list of remote folder
690
704
691
705
692 """
706 """
693
707
694 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
708 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
695 server, username, password, remotefolder = self.parmsByDefault()
709 server, username, password, remotefolder = self.parmsByDefault()
696
710
697 self.server = server
711 self.server = server
698 self.username = username
712 self.username = username
699 self.password = password
713 self.password = password
700 self.remotefolder = remotefolder
714 self.remotefolder = remotefolder
701 self.file = None
715 self.file = None
702 self.ftp = None
716 self.ftp = None
703 self.status = 0
717 self.status = 0
704
718
705 try:
719 try:
706 self.ftp = ftplib.FTP(self.server)
720 self.ftp = ftplib.FTP(self.server)
707 self.ftp.login(self.username,self.password)
721 self.ftp.login(self.username,self.password)
708 self.ftp.cwd(self.remotefolder)
722 self.ftp.cwd(self.remotefolder)
709 # print 'Connect to FTP Server: Successfully'
723 # print 'Connect to FTP Server: Successfully'
710
724
711 except ftplib.all_errors:
725 except ftplib.all_errors:
712 print 'Error FTP Service'
726 print 'Error FTP Service'
713 self.status = 1
727 self.status = 1
714 return
728 return
715
729
716
730
717
731
718 self.dirList = []
732 self.dirList = []
719
733
720 try:
734 try:
721 self.dirList = self.ftp.nlst()
735 self.dirList = self.ftp.nlst()
722
736
723 except ftplib.error_perm, resp:
737 except ftplib.error_perm, resp:
724 if str(resp) == "550 No files found":
738 if str(resp) == "550 No files found":
725 print "no files in this directory"
739 print "no files in this directory"
726 self.status = 1
740 self.status = 1
727 return
741 return
728
742
729 except ftplib.all_errors:
743 except ftplib.all_errors:
730 print 'Error Displaying Dir-Files'
744 print 'Error Displaying Dir-Files'
731 self.status = 1
745 self.status = 1
732 return
746 return
733
747
734 self.fileList = []
748 self.fileList = []
735 self.folderList = []
749 self.folderList = []
736 #only for test
750 #only for test
737 for f in self.dirList:
751 for f in self.dirList:
738 name, ext = os.path.splitext(f)
752 name, ext = os.path.splitext(f)
739 if ext != '':
753 if ext != '':
740 self.fileList.append(f)
754 self.fileList.append(f)
741 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
755 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
742
756
743 def parmsByDefault(self):
757 def parmsByDefault(self):
744 server = 'jro-app.igp.gob.pe'
758 server = 'jro-app.igp.gob.pe'
745 username = 'wmaster'
759 username = 'wmaster'
746 password = 'mst2010vhf'
760 password = 'mst2010vhf'
747 remotefolder = '/home/wmaster/graficos'
761 remotefolder = '/home/wmaster/graficos'
748
762
749 return server, username, password, remotefolder
763 return server, username, password, remotefolder
750
764
751
765
752 def mkd(self,dirname):
766 def mkd(self,dirname):
753 """
767 """
754 mkd is used to make directory in remote server
768 mkd is used to make directory in remote server
755
769
756 Input:
770 Input:
757 dirname - directory name
771 dirname - directory name
758
772
759 Return:
773 Return:
760 1 in error case else 0
774 1 in error case else 0
761 """
775 """
762 try:
776 try:
763 self.ftp.mkd(dirname)
777 self.ftp.mkd(dirname)
764 except:
778 except:
765 print 'Error creating remote folder:%s'%dirname
779 print 'Error creating remote folder:%s'%dirname
766 return 1
780 return 1
767
781
768 return 0
782 return 0
769
783
770
784
771 def delete(self,filename):
785 def delete(self,filename):
772 """
786 """
773 delete is used to delete file in current working directory of remote server
787 delete is used to delete file in current working directory of remote server
774
788
775 Input:
789 Input:
776 filename - filename to delete in remote folder
790 filename - filename to delete in remote folder
777
791
778 Return:
792 Return:
779 1 in error case else 0
793 1 in error case else 0
780 """
794 """
781
795
782 try:
796 try:
783 self.ftp.delete(filename)
797 self.ftp.delete(filename)
784 except:
798 except:
785 print 'Error deleting remote file:%s'%filename
799 print 'Error deleting remote file:%s'%filename
786 return 1
800 return 1
787
801
788 return 0
802 return 0
789
803
790 def download(self,filename,localfolder):
804 def download(self,filename,localfolder):
791 """
805 """
792 download is used to downloading file from remote folder into local folder
806 download is used to downloading file from remote folder into local folder
793
807
794 Inputs:
808 Inputs:
795 filename - filename to donwload
809 filename - filename to donwload
796
810
797 localfolder - directory local to store filename
811 localfolder - directory local to store filename
798
812
799 Returns:
813 Returns:
800 self.status - 1 in error case else 0
814 self.status - 1 in error case else 0
801 """
815 """
802
816
803 self.status = 0
817 self.status = 0
804
818
805
819
806 if not(filename in self.fileList):
820 if not(filename in self.fileList):
807 print 'filename:%s not exists'%filename
821 print 'filename:%s not exists'%filename
808 self.status = 1
822 self.status = 1
809 return self.status
823 return self.status
810
824
811 newfilename = os.path.join(localfolder,filename)
825 newfilename = os.path.join(localfolder,filename)
812
826
813 self.file = open(newfilename, 'wb')
827 self.file = open(newfilename, 'wb')
814
828
815 try:
829 try:
816 print 'Download: ' + filename
830 print 'Download: ' + filename
817 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
831 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
818 print 'Download Complete'
832 print 'Download Complete'
819 except ftplib.all_errors:
833 except ftplib.all_errors:
820 print 'Error Downloading ' + filename
834 print 'Error Downloading ' + filename
821 self.status = 1
835 self.status = 1
822 return self.status
836 return self.status
823
837
824 self.file.close()
838 self.file.close()
825
839
826 return self.status
840 return self.status
827
841
828
842
829 def __handleDownload(self,block):
843 def __handleDownload(self,block):
830 """
844 """
831 __handleDownload is used to handle writing file
845 __handleDownload is used to handle writing file
832 """
846 """
833 self.file.write(block)
847 self.file.write(block)
834
848
835
849
836 def upload(self,filename,remotefolder=None):
850 def upload(self,filename,remotefolder=None):
837 """
851 """
838 upload is used to uploading local file to remote directory
852 upload is used to uploading local file to remote directory
839
853
840 Inputs:
854 Inputs:
841 filename - full path name of local file to store in remote directory
855 filename - full path name of local file to store in remote directory
842
856
843 remotefolder - remote directory
857 remotefolder - remote directory
844
858
845 Returns:
859 Returns:
846 self.status - 1 in error case else 0
860 self.status - 1 in error case else 0
847 """
861 """
848
862
849 if remotefolder == None:
863 if remotefolder == None:
850 remotefolder = self.remotefolder
864 remotefolder = self.remotefolder
851
865
852 self.status = 0
866 self.status = 0
853
867
854 try:
868 try:
855 self.ftp.cwd(remotefolder)
869 self.ftp.cwd(remotefolder)
856
870
857 self.file = open(filename, 'rb')
871 self.file = open(filename, 'rb')
858
872
859 (head, tail) = os.path.split(filename)
873 (head, tail) = os.path.split(filename)
860
874
861 command = "STOR " + tail
875 command = "STOR " + tail
862
876
863 print 'Uploading: ' + tail
877 print 'Uploading: ' + tail
864 self.ftp.storbinary(command, self.file)
878 self.ftp.storbinary(command, self.file)
865 print 'Upload Completed'
879 print 'Upload Completed'
866
880
867 except ftplib.all_errors:
881 except ftplib.all_errors:
868 print 'Error Uploading ' + tail
882 print 'Error Uploading ' + tail
869 self.status = 1
883 self.status = 1
870 return self.status
884 return self.status
871
885
872 self.file.close()
886 self.file.close()
873
887
874 #back to initial directory in __init__()
888 #back to initial directory in __init__()
875 self.ftp.cwd(self.remotefolder)
889 self.ftp.cwd(self.remotefolder)
876
890
877 return self.status
891 return self.status
878
892
879
893
880 def dir(self,remotefolder):
894 def dir(self,remotefolder):
881 """
895 """
882 dir is used to change working directory of remote server and get folder and file list
896 dir is used to change working directory of remote server and get folder and file list
883
897
884 Input:
898 Input:
885 remotefolder - current working directory
899 remotefolder - current working directory
886
900
887 Affects:
901 Affects:
888 self.fileList - file list of working directory
902 self.fileList - file list of working directory
889
903
890 Return:
904 Return:
891 infoList - list with filenames and size of file in bytes
905 infoList - list with filenames and size of file in bytes
892
906
893 self.folderList - folder list
907 self.folderList - folder list
894 """
908 """
895
909
896 self.remotefolder = remotefolder
910 self.remotefolder = remotefolder
897 print 'Change to ' + self.remotefolder
911 print 'Change to ' + self.remotefolder
898 try:
912 try:
899 self.ftp.cwd(remotefolder)
913 self.ftp.cwd(remotefolder)
900 except ftplib.all_errors:
914 except ftplib.all_errors:
901 print 'Error Change to ' + self.remotefolder
915 print 'Error Change to ' + self.remotefolder
902 infoList = None
916 infoList = None
903 self.folderList = None
917 self.folderList = None
904 return infoList,self.folderList
918 return infoList,self.folderList
905
919
906 self.dirList = []
920 self.dirList = []
907
921
908 try:
922 try:
909 self.dirList = self.ftp.nlst()
923 self.dirList = self.ftp.nlst()
910
924
911 except ftplib.error_perm, resp:
925 except ftplib.error_perm, resp:
912 if str(resp) == "550 No files found":
926 if str(resp) == "550 No files found":
913 print "no files in this directory"
927 print "no files in this directory"
914 infoList = None
928 infoList = None
915 self.folderList = None
929 self.folderList = None
916 return infoList,self.folderList
930 return infoList,self.folderList
917 except ftplib.all_errors:
931 except ftplib.all_errors:
918 print 'Error Displaying Dir-Files'
932 print 'Error Displaying Dir-Files'
919 infoList = None
933 infoList = None
920 self.folderList = None
934 self.folderList = None
921 return infoList,self.folderList
935 return infoList,self.folderList
922
936
923 infoList = []
937 infoList = []
924 self.fileList = []
938 self.fileList = []
925 self.folderList = []
939 self.folderList = []
926 for f in self.dirList:
940 for f in self.dirList:
927 name,ext = os.path.splitext(f)
941 name,ext = os.path.splitext(f)
928 if ext != '':
942 if ext != '':
929 self.fileList.append(f)
943 self.fileList.append(f)
930 value = (f,self.ftp.size(f))
944 value = (f,self.ftp.size(f))
931 infoList.append(value)
945 infoList.append(value)
932
946
933 if ext == '':
947 if ext == '':
934 self.folderList.append(f)
948 self.folderList.append(f)
935
949
936 return infoList,self.folderList
950 return infoList,self.folderList
937
951
938
952
939 def close(self):
953 def close(self):
940 """
954 """
941 close is used to close and end FTP connection
955 close is used to close and end FTP connection
942
956
943 Inputs: None
957 Inputs: None
944
958
945 Return: void
959 Return: void
946
960
947 """
961 """
948 self.ftp.close()
962 self.ftp.close()
949
963
950 class SendByFTP(Operation):
964 class SendByFTP(Operation):
951
965
952 def __init__(self, **kwargs):
966 def __init__(self, **kwargs):
953 Operation.__init__(self, **kwargs)
967 Operation.__init__(self, **kwargs)
954 self.status = 1
968 self.status = 1
955 self.counter = 0
969 self.counter = 0
956
970
957 def error_print(self, ValueError):
971 def error_print(self, ValueError):
958
972
959 print ValueError, 'Error FTP'
973 print ValueError, 'Error FTP'
960 print "don't worry the program is running..."
974 print "don't worry the program is running..."
961
975
962 def worker_ftp(self, server, username, password, remotefolder, filenameList):
976 def worker_ftp(self, server, username, password, remotefolder, filenameList):
963
977
964 self.ftpClientObj = FTP(server, username, password, remotefolder)
978 self.ftpClientObj = FTP(server, username, password, remotefolder)
965 for filename in filenameList:
979 for filename in filenameList:
966 self.ftpClientObj.upload(filename)
980 self.ftpClientObj.upload(filename)
967 self.ftpClientObj.close()
981 self.ftpClientObj.close()
968
982
969 def ftp_thread(self, server, username, password, remotefolder):
983 def ftp_thread(self, server, username, password, remotefolder):
970 if not(self.status):
984 if not(self.status):
971 return
985 return
972
986
973 import multiprocessing
987 import multiprocessing
974
988
975 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
989 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
976 p.start()
990 p.start()
977
991
978 p.join(3)
992 p.join(3)
979
993
980 if p.is_alive():
994 if p.is_alive():
981 p.terminate()
995 p.terminate()
982 p.join()
996 p.join()
983 print 'killing ftp process...'
997 print 'killing ftp process...'
984 self.status = 0
998 self.status = 0
985 return
999 return
986
1000
987 self.status = 1
1001 self.status = 1
988 return
1002 return
989
1003
990 def filterByExt(self, ext, localfolder):
1004 def filterByExt(self, ext, localfolder):
991 fnameList = glob.glob1(localfolder,ext)
1005 fnameList = glob.glob1(localfolder,ext)
992 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
1006 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
993
1007
994 if len(self.filenameList) == 0:
1008 if len(self.filenameList) == 0:
995 self.status = 0
1009 self.status = 0
996
1010
997 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
1011 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
998
1012
999 self.counter += 1
1013 self.counter += 1
1000 if self.counter >= period:
1014 if self.counter >= period:
1001 self.filterByExt(ext, localfolder)
1015 self.filterByExt(ext, localfolder)
1002
1016
1003 self.ftp_thread(server, username, password, remotefolder)
1017 self.ftp_thread(server, username, password, remotefolder)
1004
1018
1005 self.counter = 0
1019 self.counter = 0
1006
1020
1007 self.status = 1
1021 self.status = 1
Show file before | Show file after | Hide comments
@@ -1,85 +1,114
1
1
2 import os, sys
2 import os, sys
3
3
4 from schainpy.controller import Project
4 from schainpy.controller import Project
5
5
6 controllerObj = Project()
6 controllerObj = Project()
7 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
7 controllerObj.setup(id = '002', name='script02', description="JASMET Meteor Detection")
8
8
9 #-------------------------------------- Setup -----------------------------------------
9 #-------------------------------------- Setup -----------------------------------------
10 #Verificar estas variables
10 #Verificar estas variables
11
11
12 #Path para los archivos
12 #Path para los archivos
13 # path = '/mnt/jars/2016_08/NOCHE'
13 # path = '/mnt/jars/2016_08/NOCHE'
14 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
14 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/DIA'
15 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
15 # path = '/media/joscanoa/DATA_JASMET/JASMET/2016_08/NOCHE'
16 path = '/home/nanosat/data/jasmet'
16 path = '/home/nanosat/data/jasmet'
17
17
18 #Path para los graficos
18 #Path para los graficos
19 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
19 pathfig = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/graphics')
20
20
21 #Path para los archivos HDF5 de meteoros
21 #Path para los archivos HDF5 de meteoros
22 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
22 pathfile = os.path.join(os.environ['HOME'],'Pictures/JASMET30/201608/meteor')
23
23
24 #Fechas para busqueda de archivos
24 #Fechas para busqueda de archivos
25 startDate = '2010/08/29'
25 startDate = '2010/08/29'
26 endDate = '2017/09/11'
26 endDate = '2017/09/11'
27 #Horas para busqueda de archivos
27 #Horas para busqueda de archivos
28 startTime = '00:00:00'
28 startTime = '00:00:00'
29 endTime = '23:59:59'
29 endTime = '23:59:59'
30
30
31
31
32 #------------------------------ Voltage Reading Unit ----------------------------------
32 #------------------------------ Voltage Reading Unit ----------------------------------
33
33
34 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
34 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
35 path=path,
35 path=path,
36 startDate=startDate,
36 startDate=startDate,
37 endDate=endDate,
37 endDate=endDate,
38 startTime=startTime,
38 startTime=startTime,
39 endTime=endTime,
39 endTime=endTime,
40 online=0,
40 online=0,
41 delay=30,
41 delay=30,
42 walk=1,
42 walk=1,
43 getblock=1,
43 getblock=1,
44 blocktime=100)
44 blocktime=100)
45
45
46 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
46 opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
47
47
48 #-------------------------- Voltage Processing Unit ------------------------------------
48 #-------------------------- Voltage Processing Unit ------------------------------------
49
49
50 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
50 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
51
51
52 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
52 opObj00 = procUnitConfObj0.addOperation(name='selectChannels')
53 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
53 opObj00.addParameter(name='channelList', value='0,1,2,3,4', format='intlist')
54
54
55 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
55 opObj01 = procUnitConfObj0.addOperation(name='setRadarFrequency')
56 opObj01.addParameter(name='frequency', value='30.e6', format='float')
56 opObj01.addParameter(name='frequency', value='30.e6', format='float')
57
57
58 opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
58 # opObj01 = procUnitConfObj0.addOperation(name='interpolateHeights')
59 opObj01.addParameter(name='topLim', value='73', format='int')
59 # opObj01.addParameter(name='topLim', value='73', format='int')
60 opObj01.addParameter(name='botLim', value='71', format='int')
60 # opObj01.addParameter(name='botLim', value='71', format='int')
61
61
62 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
62 opObj02 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
63
63
64 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
64 opObj03 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
65 opObj03.addParameter(name='n', value='2', format='int')
65 opObj03.addParameter(name='n', value='2', format='int')
66
66
67 procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId())
68 opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
69 opObj11.addParameter(name='id', value='237', format='int')
70 opObj11.addParameter(name='xmin', value='9.0', format='float')
71 opObj11.addParameter(name='xmax', value='16.0', format='float')
72 opObj11.addParameter(name='zmin', value='15.0', format='float')
73 opObj11.addParameter(name='zmax', value='50.0', format='float')
74
67 #--------------------------- Parameters Processing Unit ------------------------------------
75 #--------------------------- Parameters Processing Unit ------------------------------------
68
76
77 <<<<<<< HEAD
69 procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
78 procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
70 #
79 #
71 opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
80 opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
72 opObj10.addParameter(name='azimuth', value='45', format='float')
81 opObj10.addParameter(name='azimuth', value='45', format='float')
73 opObj10.addParameter(name='hmin', value='60', format='float')
82 opObj10.addParameter(name='hmin', value='60', format='float')
74 opObj10.addParameter(name='hmax', value='120', format='float')
83 opObj10.addParameter(name='hmax', value='120', format='float')
75
84
76 opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
85 opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
77 opObj12.addParameter(name='path', value=pathfile)
86 opObj12.addParameter(name='path', value=pathfile)
78 opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
87 opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
79 opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
88 opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
80 opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
89 opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
81 opObj12.addParameter(name='mode',value='2',format='int')
90 opObj12.addParameter(name='mode',value='2',format='int')
82
91
83 #--------------------------------------------------------------------------------------------------
92 #--------------------------------------------------------------------------------------------------
84
93
85 controllerObj.start()
94 controllerObj.start()
95 =======
96 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj0.getId())
97 # #
98 # opObj10 = procUnitConfObj1.addOperation(name='SMDetection', optype='other')
99 # opObj10.addParameter(name='azimuth', value='45', format='float')
100 # opObj10.addParameter(name='hmin', value='60', format='float')
101 # opObj10.addParameter(name='hmax', value='120', format='float')
102
103 # opObj12 = procUnitConfObj1.addOperation(name='ParamWriter', optype='other')
104 # opObj12.addParameter(name='path', value=pathfile)
105 # opObj12.addParameter(name='blocksPerFile', value='1000', format='int')
106 # opObj12.addParameter(name='metadataList',value='type,heightList,paramInterval,timeZone',format='list')
107 # opObj12.addParameter(name='dataList',value='data_param,utctime',format='list')
108 # opObj12.addParameter(name='mode',value='2',format='int')
109
110 #--------------------------------------------------------------------------------------------------
111 controllerObj.createObjects()
112 controllerObj.connectObjects()
113 controllerObj.run()
114 >>>>>>> online_data_hour
General Comments 0
You need to be logged in to leave comments. Login now