##// END OF EJS Templates
schain
RSS
arreglado varios dias con mp
José Chávez -
r1079:77593f0c5ecb
parent child
Show More
Side by Side Unified
Context file:
r1079:77593f0c5ecb Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,86 +1,88
1 basic = '''from schainpy.controller import Project
1 basic = '''from schainpy.controller import Project
2
2
3 desc = "{desc}"
3 desc = "{desc}"
4 project = Project()
4 project = Project()
5 project.setup(id='200', name="{name}", description=desc)
5 project.setup(id='200', name="{name}", description=desc)
6
6
7 voltage_reader = project.addReadUnit(datatype='VoltageReader',
7 voltage_reader = project.addReadUnit(datatype='VoltageReader',
8 path="{path}",
8 path="{path}",
9 startDate="{startDate}",
9 startDate="{startDate}",
10 endDate="{endDate}",
10 endDate="{endDate}",
11 startTime="{startHour}",
11 startTime="{startHour}",
12 endTime="{endHour}",
12 endTime="{endHour}",
13 online=0,
13 online=0,
14 verbose=1,
14 verbose=1,
15 walk=1,
15 walk=1,
16 )
16 )
17
17
18 voltage_proc = project.addProcUnit(datatype='VoltageProc', inputId=voltage_reader.getId())
18 voltage_proc = project.addProcUnit(datatype='VoltageProc', inputId=voltage_reader.getId())
19
19
20 profile = voltage_proc.addOperation(name='ProfileSelector', optype='other')
20 profile = voltage_proc.addOperation(name='ProfileSelector', optype='other')
21 profile.addParameter(name='profileRangeList', value='120,183', format='intlist')
21 profile.addParameter(name='profileRangeList', value='120,183', format='intlist')
22
22
23 rti = voltage_proc.addOperation(name='RTIPlot', optype='other')
23 rti = voltage_proc.addOperation(name='RTIPlot', optype='other')
24 rti.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
24 rti.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
25 rti.addParameter(name='showprofile', value='0', format='int')
25 rti.addParameter(name='showprofile', value='0', format='int')
26 rti.addParameter(name='xmin', value='0', format='int')
26 rti.addParameter(name='xmin', value='0', format='int')
27 rti.addParameter(name='xmax', value='24', format='int')
27 rti.addParameter(name='xmax', value='24', format='int')
28 rti.addParameter(name='figpath', value="{figpath}", format='str')
28 rti.addParameter(name='figpath', value="{figpath}", format='str')
29 rti.addParameter(name='wr_period', value='5', format='int')
29 rti.addParameter(name='wr_period', value='5', format='int')
30 rti.addParameter(name='exp_code', value='22', format='int')
30 rti.addParameter(name='exp_code', value='22', format='int')
31
31
32
32
33 controller.start()
33 controller.start()
34 '''
34 '''
35
35
36 multiprocess = '''from schainpy.controller import Project, MPProject
36 multiprocess = '''from schainpy.controller import Project, MPProject
37
37 from time import sleep
38 desc = "{desc}"
38 desc = "{desc}"
39
39
40 ####################
40 ####################
41 # PLOTTER RECEIVER #
41 # PLOTTER RECEIVER #
42 ####################
42 ####################
43 plotter = Project()
43 plotter = Project()
44 plotter.setup(id='100', name='receiver', description=desc)
44 plotter.setup(id='100', name='receiver', description=desc)
45
45
46 receiver_proc = plotter.addProcUnit(name='PlotterReceiver')
46 receiver_proc = plotter.addProcUnit(name='PlotterReceiver')
47 receiver_proc.addParameter(name='throttle', value=20, format='int')
47 receiver_proc.addParameter(name='throttle', value=20, format='int')
48
48
49 rti = receiver_proc.addOperation(name='PlotRTIData', optype='other')
49 rti = receiver_proc.addOperation(name='PlotRTIData', optype='other')
50 rti.addParameter(name='zmin', value='-40.0', format='float')
50 rti.addParameter(name='zmin', value='-40.0', format='float')
51 rti.addParameter(name='zmax', value='100.0', format='float')
51 rti.addParameter(name='zmax', value='100.0', format='float')
52 rti.addParameter(name='xmin', value='0.0', format='int')
52 rti.addParameter(name='xmin', value='0.0', format='int')
53 rti.addParameter(name='colormap', value='jet', format='str')
53 rti.addParameter(name='colormap', value='jet', format='str')
54
54
55 plotter.start()
55 plotter.start()
56
56
57 sleep(2)
58
57 ################
59 ################
58 # DATA EMITTER #
60 # DATA EMITTER #
59 ################
61 ################
60 project = Project()
62 project = Project()
61 project.setup(id='200', name="{name}", description=desc)
63 project.setup(id='200', name="{name}", description=desc)
62
64
63 spectra_reader = project.addReadUnit(datatype='SpectraReader',
65 spectra_reader = project.addReadUnit(datatype='SpectraReader',
64 path="{path}",
66 path="{path}",
65 startDate={startDate},
67 startDate={startDate},
66 endDate={endDate},
68 endDate={endDate},
67 startTime="{startHour}",
69 startTime="{startHour}",
68 endTime="{endHour}",
70 endTime="{endHour}",
69 online=0,
71 online=0,
70 verbose=1,
72 verbose=1,
71 walk=1,
73 walk=1,
72 )
74 )
73
75
74 spectra_proc = project.addProcUnit(datatype='Spectra', inputId=spectra_reader.getId())
76 spectra_proc = project.addProcUnit(datatype='Spectra', inputId=spectra_reader.getId())
75
77
76 parameters_proc = project.addProcUnit(datatype='ParametersProc', inputId=spectra_proc.getId())
78 parameters_proc = project.addProcUnit(datatype='ParametersProc', inputId=spectra_proc.getId())
77 moments = parameters_proc.addOperation(name='SpectralMoments', optype='other')
79 moments = parameters_proc.addOperation(name='SpectralMoments', optype='other')
78
80
79 publish = parameters_proc.addOperation(name='PublishData', optype='other')
81 publish = parameters_proc.addOperation(name='PublishData', optype='other')
80 publish.addParameter(name='zeromq', value=1, format='int')
82 publish.addParameter(name='zeromq', value=1, format='int')
81 publish.addParameter(name='verbose', value=0, format='bool')
83 publish.addParameter(name='verbose', value=0, format='bool')
82
84
83 MPProject(project, 16)
85 MPProject(project, 16)
84
86
85
87
86 '''
88 '''
Show file before | Show file after | Hide comments
@@ -1,1834 +1,1830
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time
13 import time
14 import datetime
14 import datetime
15 import traceback
15 import traceback
16 import zmq
16 import zmq
17
17
18 try:
18 try:
19 from gevent import sleep
19 from gevent import sleep
20 except:
20 except:
21 from time import sleep
21 from time import sleep
22
22
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25
25
26 LOCALTIME = True
26 LOCALTIME = True
27
27
28
28
29 def isNumber(cad):
29 def isNumber(cad):
30 """
30 """
31 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
31 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
32
32
33 Excepciones:
33 Excepciones:
34 Si un determinado string no puede ser convertido a numero
34 Si un determinado string no puede ser convertido a numero
35 Input:
35 Input:
36 str, string al cual se le analiza para determinar si convertible a un numero o no
36 str, string al cual se le analiza para determinar si convertible a un numero o no
37
37
38 Return:
38 Return:
39 True : si el string es uno numerico
39 True : si el string es uno numerico
40 False : no es un string numerico
40 False : no es un string numerico
41 """
41 """
42 try:
42 try:
43 float(cad)
43 float(cad)
44 return True
44 return True
45 except:
45 except:
46 return False
46 return False
47
47
48
48
49 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
49 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
50 """
50 """
51 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
51 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
52
52
53 Inputs:
53 Inputs:
54 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
54 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
55
55
56 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
56 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
57 segundos contados desde 01/01/1970.
57 segundos contados desde 01/01/1970.
58 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
58 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
59 segundos contados desde 01/01/1970.
59 segundos contados desde 01/01/1970.
60
60
61 Return:
61 Return:
62 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
62 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
63 fecha especificado, de lo contrario retorna False.
63 fecha especificado, de lo contrario retorna False.
64
64
65 Excepciones:
65 Excepciones:
66 Si el archivo no existe o no puede ser abierto
66 Si el archivo no existe o no puede ser abierto
67 Si la cabecera no puede ser leida.
67 Si la cabecera no puede ser leida.
68
68
69 """
69 """
70 basicHeaderObj = BasicHeader(LOCALTIME)
70 basicHeaderObj = BasicHeader(LOCALTIME)
71
71
72 try:
72 try:
73 fp = open(filename, 'rb')
73 fp = open(filename, 'rb')
74 except IOError:
74 except IOError:
75 print "The file %s can't be opened" % (filename)
75 print "The file %s can't be opened" % (filename)
76 return 0
76 return 0
77
77
78 sts = basicHeaderObj.read(fp)
78 sts = basicHeaderObj.read(fp)
79 fp.close()
79 fp.close()
80
80
81 if not(sts):
81 if not(sts):
82 print "Skipping the file %s because it has not a valid header" % (filename)
82 print "Skipping the file %s because it has not a valid header" % (filename)
83 return 0
83 return 0
84
84
85 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
85 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
86 return 0
86 return 0
87
87
88 return 1
88 return 1
89
89
90
90
91 def isTimeInRange(thisTime, startTime, endTime):
91 def isTimeInRange(thisTime, startTime, endTime):
92
93 if endTime >= startTime:
92 if endTime >= startTime:
94 if (thisTime < startTime) or (thisTime > endTime):
93 if (thisTime < startTime) or (thisTime > endTime):
95 return 0
94 return 0
96
97 return 1
95 return 1
98 else:
96 else:
99 if (thisTime < startTime) and (thisTime > endTime):
97 if (thisTime < startTime) and (thisTime > endTime):
100 return 0
98 return 0
101
102 return 1
99 return 1
103
100
104
101
105 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
102 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
106 """
103 """
107 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
104 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
108
105
109 Inputs:
106 Inputs:
110 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
107 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
111
108
112 startDate : fecha inicial del rango seleccionado en formato datetime.date
109 startDate : fecha inicial del rango seleccionado en formato datetime.date
113
110
114 endDate : fecha final del rango seleccionado en formato datetime.date
111 endDate : fecha final del rango seleccionado en formato datetime.date
115
112
116 startTime : tiempo inicial del rango seleccionado en formato datetime.time
113 startTime : tiempo inicial del rango seleccionado en formato datetime.time
117
114
118 endTime : tiempo final del rango seleccionado en formato datetime.time
115 endTime : tiempo final del rango seleccionado en formato datetime.time
119
116
120 Return:
117 Return:
121 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
118 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
122 fecha especificado, de lo contrario retorna False.
119 fecha especificado, de lo contrario retorna False.
123
120
124 Excepciones:
121 Excepciones:
125 Si el archivo no existe o no puede ser abierto
122 Si el archivo no existe o no puede ser abierto
126 Si la cabecera no puede ser leida.
123 Si la cabecera no puede ser leida.
127
124
128 """
125 """
129
126
130 try:
127 try:
131 fp = open(filename, 'rb')
128 fp = open(filename, 'rb')
132 except IOError:
129 except IOError:
133 print "The file %s can't be opened" % (filename)
130 print "The file %s can't be opened" % (filename)
134 return None
131 return None
135
132
136 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 firstBasicHeaderObj = BasicHeader(LOCALTIME)
137 systemHeaderObj = SystemHeader()
134 systemHeaderObj = SystemHeader()
138 radarControllerHeaderObj = RadarControllerHeader()
135 radarControllerHeaderObj = RadarControllerHeader()
139 processingHeaderObj = ProcessingHeader()
136 processingHeaderObj = ProcessingHeader()
140
137
141 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 lastBasicHeaderObj = BasicHeader(LOCALTIME)
142
139
143 sts = firstBasicHeaderObj.read(fp)
140 sts = firstBasicHeaderObj.read(fp)
144
141
145 if not(sts):
142 if not(sts):
146 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
143 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
147 return None
144 return None
148
145
149 if not systemHeaderObj.read(fp):
146 if not systemHeaderObj.read(fp):
150 return None
147 return None
151
148
152 if not radarControllerHeaderObj.read(fp):
149 if not radarControllerHeaderObj.read(fp):
153 return None
150 return None
154
151
155 if not processingHeaderObj.read(fp):
152 if not processingHeaderObj.read(fp):
156 return None
153 return None
157
154
158 filesize = os.path.getsize(filename)
155 filesize = os.path.getsize(filename)
159
156
160 offset = processingHeaderObj.blockSize + 24 # header size
157 offset = processingHeaderObj.blockSize + 24 # header size
161
158
162 if filesize <= offset:
159 if filesize <= offset:
163 print "[Reading] %s: This file has not enough data" % filename
160 print "[Reading] %s: This file has not enough data" % filename
164 return None
161 return None
165
162
166 fp.seek(-offset, 2)
163 fp.seek(-offset, 2)
167
164
168 sts = lastBasicHeaderObj.read(fp)
165 sts = lastBasicHeaderObj.read(fp)
169
166
170 fp.close()
167 fp.close()
171
168
172 thisDatetime = lastBasicHeaderObj.datatime
169 thisDatetime = lastBasicHeaderObj.datatime
173 thisTime_last_block = thisDatetime.time()
170 thisTime_last_block = thisDatetime.time()
174
171
175 thisDatetime = firstBasicHeaderObj.datatime
172 thisDatetime = firstBasicHeaderObj.datatime
176 thisDate = thisDatetime.date()
173 thisDate = thisDatetime.date()
177 thisTime_first_block = thisDatetime.time()
174 thisTime_first_block = thisDatetime.time()
178
175
179 # General case
176 # General case
180 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
181 #-----------o----------------------------o-----------
178 #-----------o----------------------------o-----------
182 # startTime endTime
179 # startTime endTime
183
180
184 if endTime >= startTime:
181 if endTime >= startTime:
185 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
186 return None
183 return None
187
184
188 return thisDatetime
185 return thisDatetime
189
186
190 # If endTime < startTime then endTime belongs to the next day
187 # If endTime < startTime then endTime belongs to the next day
191
188
192 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #<<<<<<<<<<<o o>>>>>>>>>>>
193 #-----------o----------------------------o-----------
190 #-----------o----------------------------o-----------
194 # endTime startTime
191 # endTime startTime
195
192
196 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
197 return None
194 return None
198
195
199 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
200 return None
197 return None
201
198
202 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
203 return None
200 return None
204
201
205 return thisDatetime
202 return thisDatetime
206
203
207
204
208 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 def isFolderInDateRange(folder, startDate=None, endDate=None):
209 """
206 """
210 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
211
208
212 Inputs:
209 Inputs:
213 folder : nombre completo del directorio.
210 folder : nombre completo del directorio.
214 Su formato deberia ser "/path_root/?YYYYDDD"
211 Su formato deberia ser "/path_root/?YYYYDDD"
215
212
216 siendo:
213 siendo:
217 YYYY : Anio (ejemplo 2015)
214 YYYY : Anio (ejemplo 2015)
218 DDD : Dia del anio (ejemplo 305)
215 DDD : Dia del anio (ejemplo 305)
219
216
220 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 startDate : fecha inicial del rango seleccionado en formato datetime.date
221
218
222 endDate : fecha final del rango seleccionado en formato datetime.date
219 endDate : fecha final del rango seleccionado en formato datetime.date
223
220
224 Return:
221 Return:
225 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
226 fecha especificado, de lo contrario retorna False.
223 fecha especificado, de lo contrario retorna False.
227 Excepciones:
224 Excepciones:
228 Si el directorio no tiene el formato adecuado
225 Si el directorio no tiene el formato adecuado
229 """
226 """
230
227
231 basename = os.path.basename(folder)
228 basename = os.path.basename(folder)
232
229
233 if not isRadarFolder(basename):
230 if not isRadarFolder(basename):
234 print "The folder %s has not the rigth format" % folder
231 print "The folder %s has not the rigth format" % folder
235 return 0
232 return 0
236
233
237 if startDate and endDate:
234 if startDate and endDate:
238 thisDate = getDateFromRadarFolder(basename)
235 thisDate = getDateFromRadarFolder(basename)
239
236
240 if thisDate < startDate:
237 if thisDate < startDate:
241 return 0
238 return 0
242
239
243 if thisDate > endDate:
240 if thisDate > endDate:
244 return 0
241 return 0
245
242
246 return 1
243 return 1
247
244
248
245
249 def isFileInDateRange(filename, startDate=None, endDate=None):
246 def isFileInDateRange(filename, startDate=None, endDate=None):
250 """
247 """
251 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
248 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
252
249
253 Inputs:
250 Inputs:
254 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
251 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
255
252
256 Su formato deberia ser "?YYYYDDDsss"
253 Su formato deberia ser "?YYYYDDDsss"
257
254
258 siendo:
255 siendo:
259 YYYY : Anio (ejemplo 2015)
256 YYYY : Anio (ejemplo 2015)
260 DDD : Dia del anio (ejemplo 305)
257 DDD : Dia del anio (ejemplo 305)
261 sss : set
258 sss : set
262
259
263 startDate : fecha inicial del rango seleccionado en formato datetime.date
260 startDate : fecha inicial del rango seleccionado en formato datetime.date
264
261
265 endDate : fecha final del rango seleccionado en formato datetime.date
262 endDate : fecha final del rango seleccionado en formato datetime.date
266
263
267 Return:
264 Return:
268 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
265 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
269 fecha especificado, de lo contrario retorna False.
266 fecha especificado, de lo contrario retorna False.
270 Excepciones:
267 Excepciones:
271 Si el archivo no tiene el formato adecuado
268 Si el archivo no tiene el formato adecuado
272 """
269 """
273
270
274 basename = os.path.basename(filename)
271 basename = os.path.basename(filename)
275
272
276 if not isRadarFile(basename):
273 if not isRadarFile(basename):
277 print "The filename %s has not the rigth format" % filename
274 print "The filename %s has not the rigth format" % filename
278 return 0
275 return 0
279
276
280 if startDate and endDate:
277 if startDate and endDate:
281 thisDate = getDateFromRadarFile(basename)
278 thisDate = getDateFromRadarFile(basename)
282
279
283 if thisDate < startDate:
280 if thisDate < startDate:
284 return 0
281 return 0
285
282
286 if thisDate > endDate:
283 if thisDate > endDate:
287 return 0
284 return 0
288
285
289 return 1
286 return 1
290
287
291
288
292 def getFileFromSet(path, ext, set):
289 def getFileFromSet(path, ext, set):
293 validFilelist = []
290 validFilelist = []
294 fileList = os.listdir(path)
291 fileList = os.listdir(path)
295
292
296 # 0 1234 567 89A BCDE
293 # 0 1234 567 89A BCDE
297 # H YYYY DDD SSS .ext
294 # H YYYY DDD SSS .ext
298
295
299 for thisFile in fileList:
296 for thisFile in fileList:
300 try:
297 try:
301 year = int(thisFile[1:5])
298 year = int(thisFile[1:5])
302 doy = int(thisFile[5:8])
299 doy = int(thisFile[5:8])
303 except:
300 except:
304 continue
301 continue
305
302
306 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
303 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 continue
304 continue
308
305
309 validFilelist.append(thisFile)
306 validFilelist.append(thisFile)
310
307
311 myfile = fnmatch.filter(
308 myfile = fnmatch.filter(
312 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
309 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
313
310
314 if len(myfile) != 0:
311 if len(myfile) != 0:
315 return myfile[0]
312 return myfile[0]
316 else:
313 else:
317 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
314 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
318 print 'the filename %s does not exist' % filename
315 print 'the filename %s does not exist' % filename
319 print '...going to the last file: '
316 print '...going to the last file: '
320
317
321 if validFilelist:
318 if validFilelist:
322 validFilelist = sorted(validFilelist, key=str.lower)
319 validFilelist = sorted(validFilelist, key=str.lower)
323 return validFilelist[-1]
320 return validFilelist[-1]
324
321
325 return None
322 return None
326
323
327
324
328 def getlastFileFromPath(path, ext):
325 def getlastFileFromPath(path, ext):
329 """
326 """
330 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
327 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
331 al final de la depuracion devuelve el ultimo file de la lista que quedo.
328 al final de la depuracion devuelve el ultimo file de la lista que quedo.
332
329
333 Input:
330 Input:
334 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
331 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
335 ext : extension de los files contenidos en una carpeta
332 ext : extension de los files contenidos en una carpeta
336
333
337 Return:
334 Return:
338 El ultimo file de una determinada carpeta, no se considera el path.
335 El ultimo file de una determinada carpeta, no se considera el path.
339 """
336 """
340 validFilelist = []
337 validFilelist = []
341 fileList = os.listdir(path)
338 fileList = os.listdir(path)
342
339
343 # 0 1234 567 89A BCDE
340 # 0 1234 567 89A BCDE
344 # H YYYY DDD SSS .ext
341 # H YYYY DDD SSS .ext
345
342
346 for thisFile in fileList:
343 for thisFile in fileList:
347
344
348 year = thisFile[1:5]
345 year = thisFile[1:5]
349 if not isNumber(year):
346 if not isNumber(year):
350 continue
347 continue
351
348
352 doy = thisFile[5:8]
349 doy = thisFile[5:8]
353 if not isNumber(doy):
350 if not isNumber(doy):
354 continue
351 continue
355
352
356 year = int(year)
353 year = int(year)
357 doy = int(doy)
354 doy = int(doy)
358
355
359 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
356 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
360 continue
357 continue
361
358
362 validFilelist.append(thisFile)
359 validFilelist.append(thisFile)
363
360
364 if validFilelist:
361 if validFilelist:
365 validFilelist = sorted(validFilelist, key=str.lower)
362 validFilelist = sorted(validFilelist, key=str.lower)
366 return validFilelist[-1]
363 return validFilelist[-1]
367
364
368 return None
365 return None
369
366
370
367
371 def checkForRealPath(path, foldercounter, year, doy, set, ext):
368 def checkForRealPath(path, foldercounter, year, doy, set, ext):
372 """
369 """
373 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
370 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
374 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
371 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
375 el path exacto de un determinado file.
372 el path exacto de un determinado file.
376
373
377 Example :
374 Example :
378 nombre correcto del file es .../.../D2009307/P2009307367.ext
375 nombre correcto del file es .../.../D2009307/P2009307367.ext
379
376
380 Entonces la funcion prueba con las siguientes combinaciones
377 Entonces la funcion prueba con las siguientes combinaciones
381 .../.../y2009307367.ext
378 .../.../y2009307367.ext
382 .../.../Y2009307367.ext
379 .../.../Y2009307367.ext
383 .../.../x2009307/y2009307367.ext
380 .../.../x2009307/y2009307367.ext
384 .../.../x2009307/Y2009307367.ext
381 .../.../x2009307/Y2009307367.ext
385 .../.../X2009307/y2009307367.ext
382 .../.../X2009307/y2009307367.ext
386 .../.../X2009307/Y2009307367.ext
383 .../.../X2009307/Y2009307367.ext
387 siendo para este caso, la ultima combinacion de letras, identica al file buscado
384 siendo para este caso, la ultima combinacion de letras, identica al file buscado
388
385
389 Return:
386 Return:
390 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
387 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
391 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
388 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
392 para el filename
389 para el filename
393 """
390 """
394 fullfilename = None
391 fullfilename = None
395 find_flag = False
392 find_flag = False
396 filename = None
393 filename = None
397
394
398 prefixDirList = [None, 'd', 'D']
395 prefixDirList = [None, 'd', 'D']
399 if ext.lower() == ".r": # voltage
396 if ext.lower() == ".r": # voltage
400 prefixFileList = ['d', 'D']
397 prefixFileList = ['d', 'D']
401 elif ext.lower() == ".pdata": # spectra
398 elif ext.lower() == ".pdata": # spectra
402 prefixFileList = ['p', 'P']
399 prefixFileList = ['p', 'P']
403 else:
400 else:
404 return None, filename
401 return None, filename
405
402
406 # barrido por las combinaciones posibles
403 # barrido por las combinaciones posibles
407 for prefixDir in prefixDirList:
404 for prefixDir in prefixDirList:
408 thispath = path
405 thispath = path
409 if prefixDir != None:
406 if prefixDir != None:
410 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
407 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
411 if foldercounter == 0:
408 if foldercounter == 0:
412 thispath = os.path.join(path, "%s%04d%03d" %
409 thispath = os.path.join(path, "%s%04d%03d" %
413 (prefixDir, year, doy))
410 (prefixDir, year, doy))
414 else:
411 else:
415 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
412 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
416 prefixDir, year, doy, foldercounter))
413 prefixDir, year, doy, foldercounter))
417 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
414 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
418 # formo el nombre del file xYYYYDDDSSS.ext
415 # formo el nombre del file xYYYYDDDSSS.ext
419 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
416 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
420 fullfilename = os.path.join(
417 fullfilename = os.path.join(
421 thispath, filename) # formo el path completo
418 thispath, filename) # formo el path completo
422
419
423 if os.path.exists(fullfilename): # verifico que exista
420 if os.path.exists(fullfilename): # verifico que exista
424 find_flag = True
421 find_flag = True
425 break
422 break
426 if find_flag:
423 if find_flag:
427 break
424 break
428
425
429 if not(find_flag):
426 if not(find_flag):
430 return None, filename
427 return None, filename
431
428
432 return fullfilename, filename
429 return fullfilename, filename
433
430
434
431
435 def isRadarFolder(folder):
432 def isRadarFolder(folder):
436 try:
433 try:
437 year = int(folder[1:5])
434 year = int(folder[1:5])
438 doy = int(folder[5:8])
435 doy = int(folder[5:8])
439 except:
436 except:
440 return 0
437 return 0
441
438
442 return 1
439 return 1
443
440
444
441
445 def isRadarFile(file):
442 def isRadarFile(file):
446 try:
443 try:
447 year = int(file[1:5])
444 year = int(file[1:5])
448 doy = int(file[5:8])
445 doy = int(file[5:8])
449 set = int(file[8:11])
446 set = int(file[8:11])
450 except:
447 except:
451 return 0
448 return 0
452
449
453 return 1
450 return 1
454
451
455
452
456 def getDateFromRadarFile(file):
453 def getDateFromRadarFile(file):
457 try:
454 try:
458 year = int(file[1:5])
455 year = int(file[1:5])
459 doy = int(file[5:8])
456 doy = int(file[5:8])
460 set = int(file[8:11])
457 set = int(file[8:11])
461 except:
458 except:
462 return None
459 return None
463
460
464 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
461 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
465 return thisDate
462 return thisDate
466
463
467
464
468 def getDateFromRadarFolder(folder):
465 def getDateFromRadarFolder(folder):
469 try:
466 try:
470 year = int(folder[1:5])
467 year = int(folder[1:5])
471 doy = int(folder[5:8])
468 doy = int(folder[5:8])
472 except:
469 except:
473 return None
470 return None
474
471
475 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
472 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
476 return thisDate
473 return thisDate
477
474
478
475
479 class JRODataIO:
476 class JRODataIO:
480
477
481 c = 3E8
478 c = 3E8
482
479
483 isConfig = False
480 isConfig = False
484
481
485 basicHeaderObj = None
482 basicHeaderObj = None
486
483
487 systemHeaderObj = None
484 systemHeaderObj = None
488
485
489 radarControllerHeaderObj = None
486 radarControllerHeaderObj = None
490
487
491 processingHeaderObj = None
488 processingHeaderObj = None
492
489
493 dtype = None
490 dtype = None
494
491
495 pathList = []
492 pathList = []
496
493
497 filenameList = []
494 filenameList = []
498
495
499 filename = None
496 filename = None
500
497
501 ext = None
498 ext = None
502
499
503 flagIsNewFile = 1
500 flagIsNewFile = 1
504
501
505 flagDiscontinuousBlock = 0
502 flagDiscontinuousBlock = 0
506
503
507 flagIsNewBlock = 0
504 flagIsNewBlock = 0
508
505
509 fp = None
506 fp = None
510
507
511 firstHeaderSize = 0
508 firstHeaderSize = 0
512
509
513 basicHeaderSize = 24
510 basicHeaderSize = 24
514
511
515 versionFile = 1103
512 versionFile = 1103
516
513
517 fileSize = None
514 fileSize = None
518
515
519 # ippSeconds = None
516 # ippSeconds = None
520
517
521 fileSizeByHeader = None
518 fileSizeByHeader = None
522
519
523 fileIndex = None
520 fileIndex = None
524
521
525 profileIndex = None
522 profileIndex = None
526
523
527 blockIndex = None
524 blockIndex = None
528
525
529 nTotalBlocks = None
526 nTotalBlocks = None
530
527
531 maxTimeStep = 30
528 maxTimeStep = 30
532
529
533 lastUTTime = None
530 lastUTTime = None
534
531
535 datablock = None
532 datablock = None
536
533
537 dataOut = None
534 dataOut = None
538
535
539 blocksize = None
536 blocksize = None
540
537
541 getByBlock = False
538 getByBlock = False
542
539
543 def __init__(self):
540 def __init__(self):
544
541
545 raise NotImplementedError
542 raise NotImplementedError
546
543
547 def run(self):
544 def run(self):
548
545
549 raise NotImplementedError
546 raise NotImplementedError
550
547
551 def getDtypeWidth(self):
548 def getDtypeWidth(self):
552
549
553 dtype_index = get_dtype_index(self.dtype)
550 dtype_index = get_dtype_index(self.dtype)
554 dtype_width = get_dtype_width(dtype_index)
551 dtype_width = get_dtype_width(dtype_index)
555
552
556 return dtype_width
553 return dtype_width
557
554
558 def getAllowedArgs(self):
555 def getAllowedArgs(self):
559 return inspect.getargspec(self.run).args
556 return inspect.getargspec(self.run).args
560
557
561
558
562 class JRODataReader(JRODataIO):
559 class JRODataReader(JRODataIO):
563
560
564 online = 0
561 online = 0
565
562
566 realtime = 0
563 realtime = 0
567
564
568 nReadBlocks = 0
565 nReadBlocks = 0
569
566
570 delay = 10 # number of seconds waiting a new file
567 delay = 10 # number of seconds waiting a new file
571
568
572 nTries = 3 # quantity tries
569 nTries = 3 # quantity tries
573
570
574 nFiles = 3 # number of files for searching
571 nFiles = 3 # number of files for searching
575
572
576 path = None
573 path = None
577
574
578 foldercounter = 0
575 foldercounter = 0
579
576
580 flagNoMoreFiles = 0
577 flagNoMoreFiles = 0
581
578
582 datetimeList = []
579 datetimeList = []
583
580
584 __isFirstTimeOnline = 1
581 __isFirstTimeOnline = 1
585
582
586 __printInfo = True
583 __printInfo = True
587
584
588 profileIndex = None
585 profileIndex = None
589
586
590 nTxs = 1
587 nTxs = 1
591
588
592 txIndex = None
589 txIndex = None
593
590
594 # Added--------------------
591 # Added--------------------
595
592
596 selBlocksize = None
593 selBlocksize = None
597
594
598 selBlocktime = None
595 selBlocktime = None
599
596
600 def __init__(self):
597 def __init__(self):
601 """
598 """
602 This class is used to find data files
599 This class is used to find data files
603
600
604 Example:
601 Example:
605 reader = JRODataReader()
602 reader = JRODataReader()
606 fileList = reader.findDataFiles()
603 fileList = reader.findDataFiles()
607
604
608 """
605 """
609 pass
606 pass
610
607
611 def createObjByDefault(self):
608 def createObjByDefault(self):
612 """
609 """
613
610
614 """
611 """
615 raise NotImplementedError
612 raise NotImplementedError
616
613
617 def getBlockDimension(self):
614 def getBlockDimension(self):
618
615
619 raise NotImplementedError
616 raise NotImplementedError
620
617
621 def searchFilesOffLine(self,
618 def searchFilesOffLine(self,
622 path,
619 path,
623 startDate=None,
620 startDate=None,
624 endDate=None,
621 endDate=None,
625 startTime=datetime.time(0, 0, 0),
622 startTime=datetime.time(0, 0, 0),
626 endTime=datetime.time(23, 59, 59),
623 endTime=datetime.time(23, 59, 59),
627 set=None,
624 set=None,
628 expLabel='',
625 expLabel='',
629 ext='.r',
626 ext='.r',
630 cursor=None,
627 cursor=None,
631 skip=None,
628 skip=None,
632 walk=True):
629 walk=True):
633
630
634 self.filenameList = []
631 self.filenameList = []
635 self.datetimeList = []
632 self.datetimeList = []
636
633
637 pathList = []
634 pathList = []
638
635
639 dateList, pathList = self.findDatafiles(
636 dateList, pathList = self.findDatafiles(
640 path, startDate, endDate, expLabel, ext, walk, include_path=True)
637 path, startDate, endDate, expLabel, ext, walk, include_path=True)
641
638
642 if dateList == []:
639 if dateList == []:
643 return [], []
640 return [], []
644
641
645 if len(dateList) > 1:
642 if len(dateList) > 1:
646 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
643 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
647 else:
644 else:
648 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
645 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
649
646
650 filenameList = []
647 filenameList = []
651 datetimeList = []
648 datetimeList = []
652
649
653 for thisPath in pathList:
650 for thisPath in pathList:
654
651
655 fileList = glob.glob1(thisPath, "*%s" % ext)
652 fileList = glob.glob1(thisPath, "*%s" % ext)
656 fileList.sort()
653 fileList.sort()
657
654
658 skippedFileList = []
655 skippedFileList = []
659
656
660 if cursor is not None and skip is not None:
657 if cursor is not None and skip is not None:
661
658
662 if skip == 0:
659 if skip == 0:
663 skippedFileList = []
660 skippedFileList = []
664 else:
661 else:
665 skippedFileList = fileList[cursor *
662 skippedFileList = fileList[cursor *
666 skip: cursor * skip + skip]
663 skip: cursor * skip + skip]
667
664
668 else:
665 else:
669 skippedFileList = fileList
666 skippedFileList = fileList
670
667
671 for file in skippedFileList:
668 for file in skippedFileList:
672
669
673 filename = os.path.join(thisPath, file)
670 filename = os.path.join(thisPath, file)
674
671
675 if not isFileInDateRange(filename, startDate, endDate):
672 if not isFileInDateRange(filename, startDate, endDate):
676 continue
673 continue
677
674
678 thisDatetime = isFileInTimeRange(
675 thisDatetime = isFileInTimeRange(
679 filename, startDate, endDate, startTime, endTime)
676 filename, startDate, endDate, startTime, endTime)
680
677
681 if not(thisDatetime):
678 if not(thisDatetime):
682 continue
679 continue
683
680
684 filenameList.append(filename)
681 filenameList.append(filename)
685 datetimeList.append(thisDatetime)
682 datetimeList.append(thisDatetime)
686
683
687 if not(filenameList):
684 if not(filenameList):
688 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
685 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
689 return [], []
686 return [], []
690
687
691 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
688 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
692 print
689 print
693
690
694 # for i in range(len(filenameList)):
691 # for i in range(len(filenameList)):
695 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
692 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
696
693
697 self.filenameList = filenameList
694 self.filenameList = filenameList
698 self.datetimeList = datetimeList
695 self.datetimeList = datetimeList
699
696
700 return pathList, filenameList
697 return pathList, filenameList
701
698
702 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
699 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
703 """
700 """
704 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
701 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
705 devuelve el archivo encontrado ademas de otros datos.
702 devuelve el archivo encontrado ademas de otros datos.
706
703
707 Input:
704 Input:
708 path : carpeta donde estan contenidos los files que contiene data
705 path : carpeta donde estan contenidos los files que contiene data
709
706
710 expLabel : Nombre del subexperimento (subfolder)
707 expLabel : Nombre del subexperimento (subfolder)
711
708
712 ext : extension de los files
709 ext : extension de los files
713
710
714 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
711 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
715
712
716 Return:
713 Return:
717 directory : eL directorio donde esta el file encontrado
714 directory : eL directorio donde esta el file encontrado
718 filename : el ultimo file de una determinada carpeta
715 filename : el ultimo file de una determinada carpeta
719 year : el anho
716 year : el anho
720 doy : el numero de dia del anho
717 doy : el numero de dia del anho
721 set : el set del archivo
718 set : el set del archivo
722
719
723
720
724 """
721 """
725 if not os.path.isdir(path):
722 if not os.path.isdir(path):
726 return None, None, None, None, None, None
723 return None, None, None, None, None, None
727
724
728 dirList = []
725 dirList = []
729
726
730 if not walk:
727 if not walk:
731 fullpath = path
728 fullpath = path
732 foldercounter = 0
729 foldercounter = 0
733 else:
730 else:
734 # Filtra solo los directorios
731 # Filtra solo los directorios
735 for thisPath in os.listdir(path):
732 for thisPath in os.listdir(path):
736 if not os.path.isdir(os.path.join(path, thisPath)):
733 if not os.path.isdir(os.path.join(path, thisPath)):
737 continue
734 continue
738 if not isRadarFolder(thisPath):
735 if not isRadarFolder(thisPath):
739 continue
736 continue
740
737
741 dirList.append(thisPath)
738 dirList.append(thisPath)
742
739
743 if not(dirList):
740 if not(dirList):
744 return None, None, None, None, None, None
741 return None, None, None, None, None, None
745
742
746 dirList = sorted(dirList, key=str.lower)
743 dirList = sorted(dirList, key=str.lower)
747
744
748 doypath = dirList[-1]
745 doypath = dirList[-1]
749 foldercounter = int(doypath.split('_')[1]) if len(
746 foldercounter = int(doypath.split('_')[1]) if len(
750 doypath.split('_')) > 1 else 0
747 doypath.split('_')) > 1 else 0
751 fullpath = os.path.join(path, doypath, expLabel)
748 fullpath = os.path.join(path, doypath, expLabel)
752
749
753 print "[Reading] %s folder was found: " % (fullpath)
750 print "[Reading] %s folder was found: " % (fullpath)
754
751
755 if set == None:
752 if set == None:
756 filename = getlastFileFromPath(fullpath, ext)
753 filename = getlastFileFromPath(fullpath, ext)
757 else:
754 else:
758 filename = getFileFromSet(fullpath, ext, set)
755 filename = getFileFromSet(fullpath, ext, set)
759
756
760 if not(filename):
757 if not(filename):
761 return None, None, None, None, None, None
758 return None, None, None, None, None, None
762
759
763 print "[Reading] %s file was found" % (filename)
760 print "[Reading] %s file was found" % (filename)
764
761
765 if not(self.__verifyFile(os.path.join(fullpath, filename))):
762 if not(self.__verifyFile(os.path.join(fullpath, filename))):
766 return None, None, None, None, None, None
763 return None, None, None, None, None, None
767
764
768 year = int(filename[1:5])
765 year = int(filename[1:5])
769 doy = int(filename[5:8])
766 doy = int(filename[5:8])
770 set = int(filename[8:11])
767 set = int(filename[8:11])
771
768
772 return fullpath, foldercounter, filename, year, doy, set
769 return fullpath, foldercounter, filename, year, doy, set
773
770
774 def __setNextFileOffline(self):
771 def __setNextFileOffline(self):
775
772
776 idFile = self.fileIndex
773 idFile = self.fileIndex
777
774
778 while (True):
775 while (True):
779 idFile += 1
776 idFile += 1
780 if not(idFile < len(self.filenameList)):
777 if not(idFile < len(self.filenameList)):
781 self.flagNoMoreFiles = 1
778 self.flagNoMoreFiles = 1
782 # print "[Reading] No more Files"
779 # print "[Reading] No more Files"
783 return 0
780 return 0
784
781
785 filename = self.filenameList[idFile]
782 filename = self.filenameList[idFile]
786
783
787 if not(self.__verifyFile(filename)):
784 if not(self.__verifyFile(filename)):
788 continue
785 continue
789
786
790 fileSize = os.path.getsize(filename)
787 fileSize = os.path.getsize(filename)
791 fp = open(filename, 'rb')
788 fp = open(filename, 'rb')
792 break
789 break
793
790
794 self.flagIsNewFile = 1
791 self.flagIsNewFile = 1
795 self.fileIndex = idFile
792 self.fileIndex = idFile
796 self.filename = filename
793 self.filename = filename
797 self.fileSize = fileSize
794 self.fileSize = fileSize
798 self.fp = fp
795 self.fp = fp
799
796
800 # print "[Reading] Setting the file: %s"%self.filename
797 # print "[Reading] Setting the file: %s"%self.filename
801
798
802 return 1
799 return 1
803
800
804 def __setNextFileOnline(self):
801 def __setNextFileOnline(self):
805 """
802 """
806 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
803 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
807 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
804 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
808 siguientes.
805 siguientes.
809
806
810 Affected:
807 Affected:
811 self.flagIsNewFile
808 self.flagIsNewFile
812 self.filename
809 self.filename
813 self.fileSize
810 self.fileSize
814 self.fp
811 self.fp
815 self.set
812 self.set
816 self.flagNoMoreFiles
813 self.flagNoMoreFiles
817
814
818 Return:
815 Return:
819 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
816 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
820 1 : si el file fue abierto con exito y esta listo a ser leido
817 1 : si el file fue abierto con exito y esta listo a ser leido
821
818
822 Excepciones:
819 Excepciones:
823 Si un determinado file no puede ser abierto
820 Si un determinado file no puede ser abierto
824 """
821 """
825 nFiles = 0
822 nFiles = 0
826 fileOk_flag = False
823 fileOk_flag = False
827 firstTime_flag = True
824 firstTime_flag = True
828
825
829 self.set += 1
826 self.set += 1
830
827
831 if self.set > 999:
828 if self.set > 999:
832 self.set = 0
829 self.set = 0
833 self.foldercounter += 1
830 self.foldercounter += 1
834
831
835 # busca el 1er file disponible
832 # busca el 1er file disponible
836 fullfilename, filename = checkForRealPath(
833 fullfilename, filename = checkForRealPath(
837 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
834 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
838 if fullfilename:
835 if fullfilename:
839 if self.__verifyFile(fullfilename, False):
836 if self.__verifyFile(fullfilename, False):
840 fileOk_flag = True
837 fileOk_flag = True
841
838
842 # si no encuentra un file entonces espera y vuelve a buscar
839 # si no encuentra un file entonces espera y vuelve a buscar
843 if not(fileOk_flag):
840 if not(fileOk_flag):
844 # busco en los siguientes self.nFiles+1 files posibles
841 # busco en los siguientes self.nFiles+1 files posibles
845 for nFiles in range(self.nFiles + 1):
842 for nFiles in range(self.nFiles + 1):
846
843
847 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
844 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
848 tries = self.nTries
845 tries = self.nTries
849 else:
846 else:
850 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
847 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
851
848
852 for nTries in range(tries):
849 for nTries in range(tries):
853 if firstTime_flag:
850 if firstTime_flag:
854 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
851 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
855 sleep(self.delay)
852 sleep(self.delay)
856 else:
853 else:
857 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
854 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
858
855
859 fullfilename, filename = checkForRealPath(
856 fullfilename, filename = checkForRealPath(
860 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
857 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
861 if fullfilename:
858 if fullfilename:
862 if self.__verifyFile(fullfilename):
859 if self.__verifyFile(fullfilename):
863 fileOk_flag = True
860 fileOk_flag = True
864 break
861 break
865
862
866 if fileOk_flag:
863 if fileOk_flag:
867 break
864 break
868
865
869 firstTime_flag = False
866 firstTime_flag = False
870
867
871 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
868 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
872 self.set += 1
869 self.set += 1
873
870
874 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
871 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
875 if nFiles == (self.nFiles - 1):
872 if nFiles == (self.nFiles - 1):
876 self.set = 0
873 self.set = 0
877 self.doy += 1
874 self.doy += 1
878 self.foldercounter = 0
875 self.foldercounter = 0
879
876
880 if fileOk_flag:
877 if fileOk_flag:
881 self.fileSize = os.path.getsize(fullfilename)
878 self.fileSize = os.path.getsize(fullfilename)
882 self.filename = fullfilename
879 self.filename = fullfilename
883 self.flagIsNewFile = 1
880 self.flagIsNewFile = 1
884 if self.fp != None:
881 if self.fp != None:
885 self.fp.close()
882 self.fp.close()
886 self.fp = open(fullfilename, 'rb')
883 self.fp = open(fullfilename, 'rb')
887 self.flagNoMoreFiles = 0
884 self.flagNoMoreFiles = 0
888 # print '[Reading] Setting the file: %s' % fullfilename
885 # print '[Reading] Setting the file: %s' % fullfilename
889 else:
886 else:
890 self.fileSize = 0
887 self.fileSize = 0
891 self.filename = None
888 self.filename = None
892 self.flagIsNewFile = 0
889 self.flagIsNewFile = 0
893 self.fp = None
890 self.fp = None
894 self.flagNoMoreFiles = 1
891 self.flagNoMoreFiles = 1
895 # print '[Reading] No more files to read'
892 # print '[Reading] No more files to read'
896
893
897 return fileOk_flag
894 return fileOk_flag
898
895
899 def setNextFile(self):
896 def setNextFile(self):
900 if self.fp != None:
897 if self.fp != None:
901 self.fp.close()
898 self.fp.close()
902
899
903 if self.online:
900 if self.online:
904 newFile = self.__setNextFileOnline()
901 newFile = self.__setNextFileOnline()
905 else:
902 else:
906 newFile = self.__setNextFileOffline()
903 newFile = self.__setNextFileOffline()
907
904
908 if not(newFile):
905 if not(newFile):
909 print '[Reading] No more files to read'
906 print '[Reading] No more files to read'
910 return 0
907 return 0
911
908
912 if self.verbose:
909 if self.verbose:
913 print '[Reading] Setting the file: %s' % self.filename
910 print '[Reading] Setting the file: %s' % self.filename
914
911
915 self.__readFirstHeader()
912 self.__readFirstHeader()
916 self.nReadBlocks = 0
913 self.nReadBlocks = 0
917 return 1
914 return 1
918
915
919 def __waitNewBlock(self):
916 def __waitNewBlock(self):
920 """
917 """
921 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
918 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
922
919
923 Si el modo de lectura es OffLine siempre retorn 0
920 Si el modo de lectura es OffLine siempre retorn 0
924 """
921 """
925 if not self.online:
922 if not self.online:
926 return 0
923 return 0
927
924
928 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
925 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
929 return 0
926 return 0
930
927
931 currentPointer = self.fp.tell()
928 currentPointer = self.fp.tell()
932
929
933 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
930 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
934
931
935 for nTries in range(self.nTries):
932 for nTries in range(self.nTries):
936
933
937 self.fp.close()
934 self.fp.close()
938 self.fp = open(self.filename, 'rb')
935 self.fp = open(self.filename, 'rb')
939 self.fp.seek(currentPointer)
936 self.fp.seek(currentPointer)
940
937
941 self.fileSize = os.path.getsize(self.filename)
938 self.fileSize = os.path.getsize(self.filename)
942 currentSize = self.fileSize - currentPointer
939 currentSize = self.fileSize - currentPointer
943
940
944 if (currentSize >= neededSize):
941 if (currentSize >= neededSize):
945 self.basicHeaderObj.read(self.fp)
942 self.basicHeaderObj.read(self.fp)
946 return 1
943 return 1
947
944
948 if self.fileSize == self.fileSizeByHeader:
945 if self.fileSize == self.fileSizeByHeader:
949 # self.flagEoF = True
946 # self.flagEoF = True
950 return 0
947 return 0
951
948
952 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
949 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
953 sleep(self.delay)
950 sleep(self.delay)
954
951
955 return 0
952 return 0
956
953
957 def waitDataBlock(self, pointer_location):
954 def waitDataBlock(self, pointer_location):
958
955
959 currentPointer = pointer_location
956 currentPointer = pointer_location
960
957
961 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
958 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
962
959
963 for nTries in range(self.nTries):
960 for nTries in range(self.nTries):
964 self.fp.close()
961 self.fp.close()
965 self.fp = open(self.filename, 'rb')
962 self.fp = open(self.filename, 'rb')
966 self.fp.seek(currentPointer)
963 self.fp.seek(currentPointer)
967
964
968 self.fileSize = os.path.getsize(self.filename)
965 self.fileSize = os.path.getsize(self.filename)
969 currentSize = self.fileSize - currentPointer
966 currentSize = self.fileSize - currentPointer
970
967
971 if (currentSize >= neededSize):
968 if (currentSize >= neededSize):
972 return 1
969 return 1
973
970
974 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
971 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
975 sleep(self.delay)
972 sleep(self.delay)
976
973
977 return 0
974 return 0
978
975
979 def __jumpToLastBlock(self):
976 def __jumpToLastBlock(self):
980
977
981 if not(self.__isFirstTimeOnline):
978 if not(self.__isFirstTimeOnline):
982 return
979 return
983
980
984 csize = self.fileSize - self.fp.tell()
981 csize = self.fileSize - self.fp.tell()
985 blocksize = self.processingHeaderObj.blockSize
982 blocksize = self.processingHeaderObj.blockSize
986
983
987 # salta el primer bloque de datos
984 # salta el primer bloque de datos
988 if csize > self.processingHeaderObj.blockSize:
985 if csize > self.processingHeaderObj.blockSize:
989 self.fp.seek(self.fp.tell() + blocksize)
986 self.fp.seek(self.fp.tell() + blocksize)
990 else:
987 else:
991 return
988 return
992
989
993 csize = self.fileSize - self.fp.tell()
990 csize = self.fileSize - self.fp.tell()
994 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
991 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 while True:
992 while True:
996
993
997 if self.fp.tell() < self.fileSize:
994 if self.fp.tell() < self.fileSize:
998 self.fp.seek(self.fp.tell() + neededsize)
995 self.fp.seek(self.fp.tell() + neededsize)
999 else:
996 else:
1000 self.fp.seek(self.fp.tell() - neededsize)
997 self.fp.seek(self.fp.tell() - neededsize)
1001 break
998 break
1002
999
1003 # csize = self.fileSize - self.fp.tell()
1000 # csize = self.fileSize - self.fp.tell()
1004 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1005 # factor = int(csize/neededsize)
1002 # factor = int(csize/neededsize)
1006 # if factor > 0:
1003 # if factor > 0:
1007 # self.fp.seek(self.fp.tell() + factor*neededsize)
1004 # self.fp.seek(self.fp.tell() + factor*neededsize)
1008
1005
1009 self.flagIsNewFile = 0
1006 self.flagIsNewFile = 0
1010 self.__isFirstTimeOnline = 0
1007 self.__isFirstTimeOnline = 0
1011
1008
1012 def __setNewBlock(self):
1009 def __setNewBlock(self):
1013 # if self.server is None:
1010 # if self.server is None:
1014 if self.fp == None:
1011 if self.fp == None:
1015 return 0
1012 return 0
1016
1013
1017 # if self.online:
1014 # if self.online:
1018 # self.__jumpToLastBlock()
1015 # self.__jumpToLastBlock()
1019
1016
1020 if self.flagIsNewFile:
1017 if self.flagIsNewFile:
1021 self.lastUTTime = self.basicHeaderObj.utc
1018 self.lastUTTime = self.basicHeaderObj.utc
1022 return 1
1019 return 1
1023
1020
1024 if self.realtime:
1021 if self.realtime:
1025 self.flagDiscontinuousBlock = 1
1022 self.flagDiscontinuousBlock = 1
1026 if not(self.setNextFile()):
1023 if not(self.setNextFile()):
1027 return 0
1024 return 0
1028 else:
1025 else:
1029 return 1
1026 return 1
1030 # if self.server is None:
1027 # if self.server is None:
1031 currentSize = self.fileSize - self.fp.tell()
1028 currentSize = self.fileSize - self.fp.tell()
1032 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1029 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1033 if (currentSize >= neededSize):
1030 if (currentSize >= neededSize):
1034 self.basicHeaderObj.read(self.fp)
1031 self.basicHeaderObj.read(self.fp)
1035 self.lastUTTime = self.basicHeaderObj.utc
1032 self.lastUTTime = self.basicHeaderObj.utc
1036 return 1
1033 return 1
1037 # else:
1034 # else:
1038 # self.basicHeaderObj.read(self.zHeader)
1035 # self.basicHeaderObj.read(self.zHeader)
1039 # self.lastUTTime = self.basicHeaderObj.utc
1036 # self.lastUTTime = self.basicHeaderObj.utc
1040 # return 1
1037 # return 1
1041 if self.__waitNewBlock():
1038 if self.__waitNewBlock():
1042 self.lastUTTime = self.basicHeaderObj.utc
1039 self.lastUTTime = self.basicHeaderObj.utc
1043 return 1
1040 return 1
1044 # if self.server is None:
1041 # if self.server is None:
1045 if not(self.setNextFile()):
1042 if not(self.setNextFile()):
1046 return 0
1043 return 0
1047
1044
1048 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1045 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1049 self.lastUTTime = self.basicHeaderObj.utc
1046 self.lastUTTime = self.basicHeaderObj.utc
1050
1047
1051 self.flagDiscontinuousBlock = 0
1048 self.flagDiscontinuousBlock = 0
1052
1049
1053 if deltaTime > self.maxTimeStep:
1050 if deltaTime > self.maxTimeStep:
1054 self.flagDiscontinuousBlock = 1
1051 self.flagDiscontinuousBlock = 1
1055
1052
1056 return 1
1053 return 1
1057
1054
1058 def readNextBlock(self):
1055 def readNextBlock(self):
1059
1056
1060 # Skip block out of startTime and endTime
1057 # Skip block out of startTime and endTime
1061 while True:
1058 while True:
1062 if not(self.__setNewBlock()):
1059 if not(self.__setNewBlock()):
1063 return 0
1060 return 0
1064
1061
1065 if not(self.readBlock()):
1062 if not(self.readBlock()):
1066 return 0
1063 return 0
1067
1064
1068 self.getBasicHeader()
1065 self.getBasicHeader()
1069
1066 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1070 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1071
1072 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1067 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1073 self.processingHeaderObj.dataBlocksPerFile,
1068 self.processingHeaderObj.dataBlocksPerFile,
1074 self.dataOut.datatime.ctime())
1069 self.dataOut.datatime.ctime())
1075 continue
1070 continue
1076
1071
1077 break
1072 break
1078
1073
1079 if self.verbose:
1074 if self.verbose:
1080 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1075 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1081 self.processingHeaderObj.dataBlocksPerFile,
1076 self.processingHeaderObj.dataBlocksPerFile,
1082 self.dataOut.datatime.ctime())
1077 self.dataOut.datatime.ctime())
1083 return 1
1078 return 1
1084
1079
1085 def __readFirstHeader(self):
1080 def __readFirstHeader(self):
1086
1081
1087 self.basicHeaderObj.read(self.fp)
1082 self.basicHeaderObj.read(self.fp)
1088 self.systemHeaderObj.read(self.fp)
1083 self.systemHeaderObj.read(self.fp)
1089 self.radarControllerHeaderObj.read(self.fp)
1084 self.radarControllerHeaderObj.read(self.fp)
1090 self.processingHeaderObj.read(self.fp)
1085 self.processingHeaderObj.read(self.fp)
1091
1086
1092 self.firstHeaderSize = self.basicHeaderObj.size
1087 self.firstHeaderSize = self.basicHeaderObj.size
1093
1088
1094 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1089 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1095 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1090 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1096 if datatype == 0:
1091 if datatype == 0:
1097 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1092 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1098 elif datatype == 1:
1093 elif datatype == 1:
1099 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1094 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1100 elif datatype == 2:
1095 elif datatype == 2:
1101 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1096 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1102 elif datatype == 3:
1097 elif datatype == 3:
1103 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1098 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1104 elif datatype == 4:
1099 elif datatype == 4:
1105 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1100 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1106 elif datatype == 5:
1101 elif datatype == 5:
1107 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1102 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1108 else:
1103 else:
1109 raise ValueError, 'Data type was not defined'
1104 raise ValueError, 'Data type was not defined'
1110
1105
1111 self.dtype = datatype_str
1106 self.dtype = datatype_str
1112 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1107 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1113 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1108 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1114 self.firstHeaderSize + self.basicHeaderSize * \
1109 self.firstHeaderSize + self.basicHeaderSize * \
1115 (self.processingHeaderObj.dataBlocksPerFile - 1)
1110 (self.processingHeaderObj.dataBlocksPerFile - 1)
1116 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1111 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1117 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1112 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1118 self.getBlockDimension()
1113 self.getBlockDimension()
1119
1114
1120 def __verifyFile(self, filename, msgFlag=True):
1115 def __verifyFile(self, filename, msgFlag=True):
1121
1116
1122 msg = None
1117 msg = None
1123
1118
1124 try:
1119 try:
1125 fp = open(filename, 'rb')
1120 fp = open(filename, 'rb')
1126 except IOError:
1121 except IOError:
1127
1122
1128 if msgFlag:
1123 if msgFlag:
1129 print "[Reading] File %s can't be opened" % (filename)
1124 print "[Reading] File %s can't be opened" % (filename)
1130
1125
1131 return False
1126 return False
1132
1127
1133 currentPosition = fp.tell()
1128 currentPosition = fp.tell()
1134 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1129 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1135
1130
1136 if neededSize == 0:
1131 if neededSize == 0:
1137 basicHeaderObj = BasicHeader(LOCALTIME)
1132 basicHeaderObj = BasicHeader(LOCALTIME)
1138 systemHeaderObj = SystemHeader()
1133 systemHeaderObj = SystemHeader()
1139 radarControllerHeaderObj = RadarControllerHeader()
1134 radarControllerHeaderObj = RadarControllerHeader()
1140 processingHeaderObj = ProcessingHeader()
1135 processingHeaderObj = ProcessingHeader()
1141
1136
1142 if not(basicHeaderObj.read(fp)):
1137 if not(basicHeaderObj.read(fp)):
1143 fp.close()
1138 fp.close()
1144 return False
1139 return False
1145
1140
1146 if not(systemHeaderObj.read(fp)):
1141 if not(systemHeaderObj.read(fp)):
1147 fp.close()
1142 fp.close()
1148 return False
1143 return False
1149
1144
1150 if not(radarControllerHeaderObj.read(fp)):
1145 if not(radarControllerHeaderObj.read(fp)):
1151 fp.close()
1146 fp.close()
1152 return False
1147 return False
1153
1148
1154 if not(processingHeaderObj.read(fp)):
1149 if not(processingHeaderObj.read(fp)):
1155 fp.close()
1150 fp.close()
1156 return False
1151 return False
1157
1152
1158 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1153 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1159 else:
1154 else:
1160 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1155 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1161
1156
1162 fp.close()
1157 fp.close()
1163
1158
1164 fileSize = os.path.getsize(filename)
1159 fileSize = os.path.getsize(filename)
1165 currentSize = fileSize - currentPosition
1160 currentSize = fileSize - currentPosition
1166
1161
1167 if currentSize < neededSize:
1162 if currentSize < neededSize:
1168 if msgFlag and (msg != None):
1163 if msgFlag and (msg != None):
1169 print msg
1164 print msg
1170 return False
1165 return False
1171
1166
1172 return True
1167 return True
1173
1168
1174 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1169 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1175
1170
1176 path_empty = True
1171 path_empty = True
1177
1172
1178 dateList = []
1173 dateList = []
1179 pathList = []
1174 pathList = []
1180
1175
1181 multi_path = path.split(',')
1176 multi_path = path.split(',')
1182
1177
1183 if not walk:
1178 if not walk:
1184
1179
1185 for single_path in multi_path:
1180 for single_path in multi_path:
1186
1181
1187 if not os.path.isdir(single_path):
1182 if not os.path.isdir(single_path):
1188 continue
1183 continue
1189
1184
1190 fileList = glob.glob1(single_path, "*" + ext)
1185 fileList = glob.glob1(single_path, "*" + ext)
1191
1186
1192 if not fileList:
1187 if not fileList:
1193 continue
1188 continue
1194
1189
1195 path_empty = False
1190 path_empty = False
1196
1191
1197 fileList.sort()
1192 fileList.sort()
1198
1193
1199 for thisFile in fileList:
1194 for thisFile in fileList:
1200
1195
1201 if not os.path.isfile(os.path.join(single_path, thisFile)):
1196 if not os.path.isfile(os.path.join(single_path, thisFile)):
1202 continue
1197 continue
1203
1198
1204 if not isRadarFile(thisFile):
1199 if not isRadarFile(thisFile):
1205 continue
1200 continue
1206
1201
1207 if not isFileInDateRange(thisFile, startDate, endDate):
1202 if not isFileInDateRange(thisFile, startDate, endDate):
1208 continue
1203 continue
1209
1204
1210 thisDate = getDateFromRadarFile(thisFile)
1205 thisDate = getDateFromRadarFile(thisFile)
1211
1206
1212 if thisDate in dateList:
1207 if thisDate in dateList:
1213 continue
1208 continue
1214
1209
1215 dateList.append(thisDate)
1210 dateList.append(thisDate)
1216 pathList.append(single_path)
1211 pathList.append(single_path)
1217
1212
1218 else:
1213 else:
1219 for single_path in multi_path:
1214 for single_path in multi_path:
1220
1215
1221 if not os.path.isdir(single_path):
1216 if not os.path.isdir(single_path):
1222 continue
1217 continue
1223
1218
1224 dirList = []
1219 dirList = []
1225
1220
1226 for thisPath in os.listdir(single_path):
1221 for thisPath in os.listdir(single_path):
1227
1222
1228 if not os.path.isdir(os.path.join(single_path, thisPath)):
1223 if not os.path.isdir(os.path.join(single_path, thisPath)):
1229 continue
1224 continue
1230
1225
1231 if not isRadarFolder(thisPath):
1226 if not isRadarFolder(thisPath):
1232 continue
1227 continue
1233
1228
1234 if not isFolderInDateRange(thisPath, startDate, endDate):
1229 if not isFolderInDateRange(thisPath, startDate, endDate):
1235 continue
1230 continue
1236
1231
1237 dirList.append(thisPath)
1232 dirList.append(thisPath)
1238
1233
1239 if not dirList:
1234 if not dirList:
1240 continue
1235 continue
1241
1236
1242 dirList.sort()
1237 dirList.sort()
1243
1238
1244 for thisDir in dirList:
1239 for thisDir in dirList:
1245
1240
1246 datapath = os.path.join(single_path, thisDir, expLabel)
1241 datapath = os.path.join(single_path, thisDir, expLabel)
1247 fileList = glob.glob1(datapath, "*" + ext)
1242 fileList = glob.glob1(datapath, "*" + ext)
1248
1243
1249 if not fileList:
1244 if not fileList:
1250 continue
1245 continue
1251
1246
1252 path_empty = False
1247 path_empty = False
1253
1248
1254 thisDate = getDateFromRadarFolder(thisDir)
1249 thisDate = getDateFromRadarFolder(thisDir)
1255
1250
1256 pathList.append(datapath)
1251 pathList.append(datapath)
1257 dateList.append(thisDate)
1252 dateList.append(thisDate)
1258
1253
1259 dateList.sort()
1254 dateList.sort()
1260
1255
1261 if walk:
1256 if walk:
1262 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1257 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1263 else:
1258 else:
1264 pattern_path = multi_path[0]
1259 pattern_path = multi_path[0]
1265
1260
1266 if path_empty:
1261 if path_empty:
1267 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1262 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1268 else:
1263 else:
1269 if not dateList:
1264 if not dateList:
1270 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1265 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1271
1266
1272 if include_path:
1267 if include_path:
1273 return dateList, pathList
1268 return dateList, pathList
1274
1269
1275 return dateList
1270 return dateList
1276
1271
1277 def setup(self,
1272 def setup(self,
1278 path=None,
1273 path=None,
1279 startDate=None,
1274 startDate=None,
1280 endDate=None,
1275 endDate=None,
1281 startTime=datetime.time(0, 0, 0),
1276 startTime=datetime.time(0, 0, 0),
1282 endTime=datetime.time(23, 59, 59),
1277 endTime=datetime.time(23, 59, 59),
1283 set=None,
1278 set=None,
1284 expLabel="",
1279 expLabel="",
1285 ext=None,
1280 ext=None,
1286 online=False,
1281 online=False,
1287 delay=60,
1282 delay=60,
1288 walk=True,
1283 walk=True,
1289 getblock=False,
1284 getblock=False,
1290 nTxs=1,
1285 nTxs=1,
1291 realtime=False,
1286 realtime=False,
1292 blocksize=None,
1287 blocksize=None,
1293 blocktime=None,
1288 blocktime=None,
1294 skip=None,
1289 skip=None,
1295 cursor=None,
1290 cursor=None,
1296 warnings=True,
1291 warnings=True,
1297 verbose=True,
1292 verbose=True,
1298 server=None,
1293 server=None,
1299 format=None,
1294 format=None,
1300 oneDDict=None,
1295 oneDDict=None,
1301 twoDDict=None,
1296 twoDDict=None,
1302 ind2DList=None):
1297 ind2DList=None):
1303 if server is not None:
1298 if server is not None:
1304 if 'tcp://' in server:
1299 if 'tcp://' in server:
1305 address = server
1300 address = server
1306 else:
1301 else:
1307 address = 'ipc:///tmp/%s' % server
1302 address = 'ipc:///tmp/%s' % server
1308 self.server = address
1303 self.server = address
1309 self.context = zmq.Context()
1304 self.context = zmq.Context()
1310 self.receiver = self.context.socket(zmq.PULL)
1305 self.receiver = self.context.socket(zmq.PULL)
1311 self.receiver.connect(self.server)
1306 self.receiver.connect(self.server)
1312 time.sleep(0.5)
1307 time.sleep(0.5)
1313 print '[Starting] ReceiverData from {}'.format(self.server)
1308 print '[Starting] ReceiverData from {}'.format(self.server)
1314 else:
1309 else:
1315 self.server = None
1310 self.server = None
1316 if path == None:
1311 if path == None:
1317 raise ValueError, "[Reading] The path is not valid"
1312 raise ValueError, "[Reading] The path is not valid"
1318
1313
1319 if ext == None:
1314 if ext == None:
1320 ext = self.ext
1315 ext = self.ext
1321
1316
1322 if online:
1317 if online:
1323 print "[Reading] Searching files in online mode..."
1318 print "[Reading] Searching files in online mode..."
1324
1319
1325 for nTries in range(self.nTries):
1320 for nTries in range(self.nTries):
1326 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1321 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1327 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1322 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1328
1323
1329 if fullpath:
1324 if fullpath:
1330 break
1325 break
1331
1326
1332 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1327 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1333 sleep(self.delay)
1328 sleep(self.delay)
1334
1329
1335 if not(fullpath):
1330 if not(fullpath):
1336 print "[Reading] There 'isn't any valid file in %s" % path
1331 print "[Reading] There 'isn't any valid file in %s" % path
1337 return
1332 return
1338
1333
1339 self.year = year
1334 self.year = year
1340 self.doy = doy
1335 self.doy = doy
1341 self.set = set - 1
1336 self.set = set - 1
1342 self.path = path
1337 self.path = path
1343 self.foldercounter = foldercounter
1338 self.foldercounter = foldercounter
1344 last_set = None
1339 last_set = None
1345 else:
1340 else:
1346 print "[Reading] Searching files in offline mode ..."
1341 print "[Reading] Searching files in offline mode ..."
1347 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1342 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1348 startTime=startTime, endTime=endTime,
1343 startTime=startTime, endTime=endTime,
1349 set=set, expLabel=expLabel, ext=ext,
1344 set=set, expLabel=expLabel, ext=ext,
1350 walk=walk, cursor=cursor,
1345 walk=walk, cursor=cursor,
1351 skip=skip)
1346 skip=skip)
1352
1347
1353 if not(pathList):
1348 if not(pathList):
1354 self.fileIndex = -1
1349 self.fileIndex = -1
1355 self.pathList = []
1350 self.pathList = []
1356 self.filenameList = []
1351 self.filenameList = []
1357 return
1352 return
1358
1353
1359 self.fileIndex = -1
1354 self.fileIndex = -1
1360 self.pathList = pathList
1355 self.pathList = pathList
1361 self.filenameList = filenameList
1356 self.filenameList = filenameList
1362 file_name = os.path.basename(filenameList[-1])
1357 file_name = os.path.basename(filenameList[-1])
1363 basename, ext = os.path.splitext(file_name)
1358 basename, ext = os.path.splitext(file_name)
1364 last_set = int(basename[-3:])
1359 last_set = int(basename[-3:])
1365
1360
1366 self.online = online
1361 self.online = online
1367 self.realtime = realtime
1362 self.realtime = realtime
1368 self.delay = delay
1363 self.delay = delay
1369 ext = ext.lower()
1364 ext = ext.lower()
1370 self.ext = ext
1365 self.ext = ext
1371 self.getByBlock = getblock
1366 self.getByBlock = getblock
1372 self.nTxs = nTxs
1367 self.nTxs = nTxs
1373 self.startTime = startTime
1368 self.startTime = startTime
1374 self.endTime = endTime
1369 self.endTime = endTime
1375
1370 self.endDate = endDate
1371 self.startDate = startDate
1376 # Added-----------------
1372 # Added-----------------
1377 self.selBlocksize = blocksize
1373 self.selBlocksize = blocksize
1378 self.selBlocktime = blocktime
1374 self.selBlocktime = blocktime
1379
1375
1380 # Verbose-----------
1376 # Verbose-----------
1381 self.verbose = verbose
1377 self.verbose = verbose
1382 self.warnings = warnings
1378 self.warnings = warnings
1383
1379
1384 if not(self.setNextFile()):
1380 if not(self.setNextFile()):
1385 if (startDate != None) and (endDate != None):
1381 if (startDate != None) and (endDate != None):
1386 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1382 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1387 elif startDate != None:
1383 elif startDate != None:
1388 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1384 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1389 else:
1385 else:
1390 print "[Reading] No files"
1386 print "[Reading] No files"
1391
1387
1392 self.fileIndex = -1
1388 self.fileIndex = -1
1393 self.pathList = []
1389 self.pathList = []
1394 self.filenameList = []
1390 self.filenameList = []
1395 return
1391 return
1396
1392
1397 # self.getBasicHeader()
1393 # self.getBasicHeader()
1398
1394
1399 if last_set != None:
1395 if last_set != None:
1400 self.dataOut.last_block = last_set * \
1396 self.dataOut.last_block = last_set * \
1401 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1397 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1402 return
1398 return
1403
1399
1404 def getBasicHeader(self):
1400 def getBasicHeader(self):
1405
1401
1406 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1402 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1407 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1403 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1408
1404
1409 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1405 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1410
1406
1411 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1407 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1412
1408
1413 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1409 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1414
1410
1415 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1411 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1416
1412
1417 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1413 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1418
1414
1419 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1415 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1420
1416
1421 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1417 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1422
1418
1423 def getFirstHeader(self):
1419 def getFirstHeader(self):
1424
1420
1425 raise NotImplementedError
1421 raise NotImplementedError
1426
1422
1427 def getData(self):
1423 def getData(self):
1428
1424
1429 raise NotImplementedError
1425 raise NotImplementedError
1430
1426
1431 def hasNotDataInBuffer(self):
1427 def hasNotDataInBuffer(self):
1432
1428
1433 raise NotImplementedError
1429 raise NotImplementedError
1434
1430
1435 def readBlock(self):
1431 def readBlock(self):
1436
1432
1437 raise NotImplementedError
1433 raise NotImplementedError
1438
1434
1439 def isEndProcess(self):
1435 def isEndProcess(self):
1440
1436
1441 return self.flagNoMoreFiles
1437 return self.flagNoMoreFiles
1442
1438
1443 def printReadBlocks(self):
1439 def printReadBlocks(self):
1444
1440
1445 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1441 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1446
1442
1447 def printTotalBlocks(self):
1443 def printTotalBlocks(self):
1448
1444
1449 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1445 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1450
1446
1451 def printNumberOfBlock(self):
1447 def printNumberOfBlock(self):
1452 'SPAM!'
1448 'SPAM!'
1453
1449
1454 # if self.flagIsNewBlock:
1450 # if self.flagIsNewBlock:
1455 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1451 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1456 # self.processingHeaderObj.dataBlocksPerFile,
1452 # self.processingHeaderObj.dataBlocksPerFile,
1457 # self.dataOut.datatime.ctime())
1453 # self.dataOut.datatime.ctime())
1458
1454
1459 def printInfo(self):
1455 def printInfo(self):
1460
1456
1461 if self.__printInfo == False:
1457 if self.__printInfo == False:
1462 return
1458 return
1463
1459
1464 self.basicHeaderObj.printInfo()
1460 self.basicHeaderObj.printInfo()
1465 self.systemHeaderObj.printInfo()
1461 self.systemHeaderObj.printInfo()
1466 self.radarControllerHeaderObj.printInfo()
1462 self.radarControllerHeaderObj.printInfo()
1467 self.processingHeaderObj.printInfo()
1463 self.processingHeaderObj.printInfo()
1468
1464
1469 self.__printInfo = False
1465 self.__printInfo = False
1470
1466
1471 def run(self,
1467 def run(self,
1472 path=None,
1468 path=None,
1473 startDate=None,
1469 startDate=None,
1474 endDate=None,
1470 endDate=None,
1475 startTime=datetime.time(0, 0, 0),
1471 startTime=datetime.time(0, 0, 0),
1476 endTime=datetime.time(23, 59, 59),
1472 endTime=datetime.time(23, 59, 59),
1477 set=None,
1473 set=None,
1478 expLabel="",
1474 expLabel="",
1479 ext=None,
1475 ext=None,
1480 online=False,
1476 online=False,
1481 delay=60,
1477 delay=60,
1482 walk=True,
1478 walk=True,
1483 getblock=False,
1479 getblock=False,
1484 nTxs=1,
1480 nTxs=1,
1485 realtime=False,
1481 realtime=False,
1486 blocksize=None,
1482 blocksize=None,
1487 blocktime=None,
1483 blocktime=None,
1488 skip=None,
1484 skip=None,
1489 cursor=None,
1485 cursor=None,
1490 warnings=True,
1486 warnings=True,
1491 server=None,
1487 server=None,
1492 verbose=True,
1488 verbose=True,
1493 format=None,
1489 format=None,
1494 oneDDict=None,
1490 oneDDict=None,
1495 twoDDict=None,
1491 twoDDict=None,
1496 ind2DList=None, **kwargs):
1492 ind2DList=None, **kwargs):
1497
1493
1498 if not(self.isConfig):
1494 if not(self.isConfig):
1499 self.setup(path=path,
1495 self.setup(path=path,
1500 startDate=startDate,
1496 startDate=startDate,
1501 endDate=endDate,
1497 endDate=endDate,
1502 startTime=startTime,
1498 startTime=startTime,
1503 endTime=endTime,
1499 endTime=endTime,
1504 set=set,
1500 set=set,
1505 expLabel=expLabel,
1501 expLabel=expLabel,
1506 ext=ext,
1502 ext=ext,
1507 online=online,
1503 online=online,
1508 delay=delay,
1504 delay=delay,
1509 walk=walk,
1505 walk=walk,
1510 getblock=getblock,
1506 getblock=getblock,
1511 nTxs=nTxs,
1507 nTxs=nTxs,
1512 realtime=realtime,
1508 realtime=realtime,
1513 blocksize=blocksize,
1509 blocksize=blocksize,
1514 blocktime=blocktime,
1510 blocktime=blocktime,
1515 skip=skip,
1511 skip=skip,
1516 cursor=cursor,
1512 cursor=cursor,
1517 warnings=warnings,
1513 warnings=warnings,
1518 server=server,
1514 server=server,
1519 verbose=verbose,
1515 verbose=verbose,
1520 format=format,
1516 format=format,
1521 oneDDict=oneDDict,
1517 oneDDict=oneDDict,
1522 twoDDict=twoDDict,
1518 twoDDict=twoDDict,
1523 ind2DList=ind2DList)
1519 ind2DList=ind2DList)
1524 self.isConfig = True
1520 self.isConfig = True
1525 if server is None:
1521 if server is None:
1526 self.getData()
1522 self.getData()
1527 else:
1523 else:
1528 self.getFromServer()
1524 self.getFromServer()
1529
1525
1530
1526
1531 class JRODataWriter(JRODataIO):
1527 class JRODataWriter(JRODataIO):
1532
1528
1533 """
1529 """
1534 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1530 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1535 de los datos siempre se realiza por bloques.
1531 de los datos siempre se realiza por bloques.
1536 """
1532 """
1537
1533
1538 blockIndex = 0
1534 blockIndex = 0
1539
1535
1540 path = None
1536 path = None
1541
1537
1542 setFile = None
1538 setFile = None
1543
1539
1544 profilesPerBlock = None
1540 profilesPerBlock = None
1545
1541
1546 blocksPerFile = None
1542 blocksPerFile = None
1547
1543
1548 nWriteBlocks = 0
1544 nWriteBlocks = 0
1549
1545
1550 fileDate = None
1546 fileDate = None
1551
1547
1552 def __init__(self, dataOut=None):
1548 def __init__(self, dataOut=None):
1553 raise NotImplementedError
1549 raise NotImplementedError
1554
1550
1555 def hasAllDataInBuffer(self):
1551 def hasAllDataInBuffer(self):
1556 raise NotImplementedError
1552 raise NotImplementedError
1557
1553
1558 def setBlockDimension(self):
1554 def setBlockDimension(self):
1559 raise NotImplementedError
1555 raise NotImplementedError
1560
1556
1561 def writeBlock(self):
1557 def writeBlock(self):
1562 raise NotImplementedError
1558 raise NotImplementedError
1563
1559
1564 def putData(self):
1560 def putData(self):
1565 raise NotImplementedError
1561 raise NotImplementedError
1566
1562
1567 def getProcessFlags(self):
1563 def getProcessFlags(self):
1568
1564
1569 processFlags = 0
1565 processFlags = 0
1570
1566
1571 dtype_index = get_dtype_index(self.dtype)
1567 dtype_index = get_dtype_index(self.dtype)
1572 procflag_dtype = get_procflag_dtype(dtype_index)
1568 procflag_dtype = get_procflag_dtype(dtype_index)
1573
1569
1574 processFlags += procflag_dtype
1570 processFlags += procflag_dtype
1575
1571
1576 if self.dataOut.flagDecodeData:
1572 if self.dataOut.flagDecodeData:
1577 processFlags += PROCFLAG.DECODE_DATA
1573 processFlags += PROCFLAG.DECODE_DATA
1578
1574
1579 if self.dataOut.flagDeflipData:
1575 if self.dataOut.flagDeflipData:
1580 processFlags += PROCFLAG.DEFLIP_DATA
1576 processFlags += PROCFLAG.DEFLIP_DATA
1581
1577
1582 if self.dataOut.code is not None:
1578 if self.dataOut.code is not None:
1583 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1579 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1584
1580
1585 if self.dataOut.nCohInt > 1:
1581 if self.dataOut.nCohInt > 1:
1586 processFlags += PROCFLAG.COHERENT_INTEGRATION
1582 processFlags += PROCFLAG.COHERENT_INTEGRATION
1587
1583
1588 if self.dataOut.type == "Spectra":
1584 if self.dataOut.type == "Spectra":
1589 if self.dataOut.nIncohInt > 1:
1585 if self.dataOut.nIncohInt > 1:
1590 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1586 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1591
1587
1592 if self.dataOut.data_dc is not None:
1588 if self.dataOut.data_dc is not None:
1593 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1589 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1594
1590
1595 if self.dataOut.flagShiftFFT:
1591 if self.dataOut.flagShiftFFT:
1596 processFlags += PROCFLAG.SHIFT_FFT_DATA
1592 processFlags += PROCFLAG.SHIFT_FFT_DATA
1597
1593
1598 return processFlags
1594 return processFlags
1599
1595
1600 def setBasicHeader(self):
1596 def setBasicHeader(self):
1601
1597
1602 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1598 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1603 self.basicHeaderObj.version = self.versionFile
1599 self.basicHeaderObj.version = self.versionFile
1604 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1600 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1605
1601
1606 utc = numpy.floor(self.dataOut.utctime)
1602 utc = numpy.floor(self.dataOut.utctime)
1607 milisecond = (self.dataOut.utctime - utc) * 1000.0
1603 milisecond = (self.dataOut.utctime - utc) * 1000.0
1608
1604
1609 self.basicHeaderObj.utc = utc
1605 self.basicHeaderObj.utc = utc
1610 self.basicHeaderObj.miliSecond = milisecond
1606 self.basicHeaderObj.miliSecond = milisecond
1611 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1607 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1612 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1608 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1613 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1609 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1614
1610
1615 def setFirstHeader(self):
1611 def setFirstHeader(self):
1616 """
1612 """
1617 Obtiene una copia del First Header
1613 Obtiene una copia del First Header
1618
1614
1619 Affected:
1615 Affected:
1620
1616
1621 self.basicHeaderObj
1617 self.basicHeaderObj
1622 self.systemHeaderObj
1618 self.systemHeaderObj
1623 self.radarControllerHeaderObj
1619 self.radarControllerHeaderObj
1624 self.processingHeaderObj self.
1620 self.processingHeaderObj self.
1625
1621
1626 Return:
1622 Return:
1627 None
1623 None
1628 """
1624 """
1629
1625
1630 raise NotImplementedError
1626 raise NotImplementedError
1631
1627
1632 def __writeFirstHeader(self):
1628 def __writeFirstHeader(self):
1633 """
1629 """
1634 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1630 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1635
1631
1636 Affected:
1632 Affected:
1637 __dataType
1633 __dataType
1638
1634
1639 Return:
1635 Return:
1640 None
1636 None
1641 """
1637 """
1642
1638
1643 # CALCULAR PARAMETROS
1639 # CALCULAR PARAMETROS
1644
1640
1645 sizeLongHeader = self.systemHeaderObj.size + \
1641 sizeLongHeader = self.systemHeaderObj.size + \
1646 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1642 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1647 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1643 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1648
1644
1649 self.basicHeaderObj.write(self.fp)
1645 self.basicHeaderObj.write(self.fp)
1650 self.systemHeaderObj.write(self.fp)
1646 self.systemHeaderObj.write(self.fp)
1651 self.radarControllerHeaderObj.write(self.fp)
1647 self.radarControllerHeaderObj.write(self.fp)
1652 self.processingHeaderObj.write(self.fp)
1648 self.processingHeaderObj.write(self.fp)
1653
1649
1654 def __setNewBlock(self):
1650 def __setNewBlock(self):
1655 """
1651 """
1656 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1652 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1657
1653
1658 Return:
1654 Return:
1659 0 : si no pudo escribir nada
1655 0 : si no pudo escribir nada
1660 1 : Si escribio el Basic el First Header
1656 1 : Si escribio el Basic el First Header
1661 """
1657 """
1662 if self.fp == None:
1658 if self.fp == None:
1663 self.setNextFile()
1659 self.setNextFile()
1664
1660
1665 if self.flagIsNewFile:
1661 if self.flagIsNewFile:
1666 return 1
1662 return 1
1667
1663
1668 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1664 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1669 self.basicHeaderObj.write(self.fp)
1665 self.basicHeaderObj.write(self.fp)
1670 return 1
1666 return 1
1671
1667
1672 if not(self.setNextFile()):
1668 if not(self.setNextFile()):
1673 return 0
1669 return 0
1674
1670
1675 return 1
1671 return 1
1676
1672
1677 def writeNextBlock(self):
1673 def writeNextBlock(self):
1678 """
1674 """
1679 Selecciona el bloque siguiente de datos y los escribe en un file
1675 Selecciona el bloque siguiente de datos y los escribe en un file
1680
1676
1681 Return:
1677 Return:
1682 0 : Si no hizo pudo escribir el bloque de datos
1678 0 : Si no hizo pudo escribir el bloque de datos
1683 1 : Si no pudo escribir el bloque de datos
1679 1 : Si no pudo escribir el bloque de datos
1684 """
1680 """
1685 if not(self.__setNewBlock()):
1681 if not(self.__setNewBlock()):
1686 return 0
1682 return 0
1687
1683
1688 self.writeBlock()
1684 self.writeBlock()
1689
1685
1690 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1686 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1691 self.processingHeaderObj.dataBlocksPerFile)
1687 self.processingHeaderObj.dataBlocksPerFile)
1692
1688
1693 return 1
1689 return 1
1694
1690
1695 def setNextFile(self):
1691 def setNextFile(self):
1696 """
1692 """
1697 Determina el siguiente file que sera escrito
1693 Determina el siguiente file que sera escrito
1698
1694
1699 Affected:
1695 Affected:
1700 self.filename
1696 self.filename
1701 self.subfolder
1697 self.subfolder
1702 self.fp
1698 self.fp
1703 self.setFile
1699 self.setFile
1704 self.flagIsNewFile
1700 self.flagIsNewFile
1705
1701
1706 Return:
1702 Return:
1707 0 : Si el archivo no puede ser escrito
1703 0 : Si el archivo no puede ser escrito
1708 1 : Si el archivo esta listo para ser escrito
1704 1 : Si el archivo esta listo para ser escrito
1709 """
1705 """
1710 ext = self.ext
1706 ext = self.ext
1711 path = self.path
1707 path = self.path
1712
1708
1713 if self.fp != None:
1709 if self.fp != None:
1714 self.fp.close()
1710 self.fp.close()
1715
1711
1716 timeTuple = time.localtime(self.dataOut.utctime)
1712 timeTuple = time.localtime(self.dataOut.utctime)
1717 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1713 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1718
1714
1719 fullpath = os.path.join(path, subfolder)
1715 fullpath = os.path.join(path, subfolder)
1720 setFile = self.setFile
1716 setFile = self.setFile
1721
1717
1722 if not(os.path.exists(fullpath)):
1718 if not(os.path.exists(fullpath)):
1723 os.mkdir(fullpath)
1719 os.mkdir(fullpath)
1724 setFile = -1 # inicializo mi contador de seteo
1720 setFile = -1 # inicializo mi contador de seteo
1725 else:
1721 else:
1726 filesList = os.listdir(fullpath)
1722 filesList = os.listdir(fullpath)
1727 if len(filesList) > 0:
1723 if len(filesList) > 0:
1728 filesList = sorted(filesList, key=str.lower)
1724 filesList = sorted(filesList, key=str.lower)
1729 filen = filesList[-1]
1725 filen = filesList[-1]
1730 # el filename debera tener el siguiente formato
1726 # el filename debera tener el siguiente formato
1731 # 0 1234 567 89A BCDE (hex)
1727 # 0 1234 567 89A BCDE (hex)
1732 # x YYYY DDD SSS .ext
1728 # x YYYY DDD SSS .ext
1733 if isNumber(filen[8:11]):
1729 if isNumber(filen[8:11]):
1734 # inicializo mi contador de seteo al seteo del ultimo file
1730 # inicializo mi contador de seteo al seteo del ultimo file
1735 setFile = int(filen[8:11])
1731 setFile = int(filen[8:11])
1736 else:
1732 else:
1737 setFile = -1
1733 setFile = -1
1738 else:
1734 else:
1739 setFile = -1 # inicializo mi contador de seteo
1735 setFile = -1 # inicializo mi contador de seteo
1740
1736
1741 setFile += 1
1737 setFile += 1
1742
1738
1743 # If this is a new day it resets some values
1739 # If this is a new day it resets some values
1744 if self.dataOut.datatime.date() > self.fileDate:
1740 if self.dataOut.datatime.date() > self.fileDate:
1745 setFile = 0
1741 setFile = 0
1746 self.nTotalBlocks = 0
1742 self.nTotalBlocks = 0
1747
1743
1748 filen = '%s%4.4d%3.3d%3.3d%s' % (
1744 filen = '%s%4.4d%3.3d%3.3d%s' % (
1749 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1745 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1750
1746
1751 filename = os.path.join(path, subfolder, filen)
1747 filename = os.path.join(path, subfolder, filen)
1752
1748
1753 fp = open(filename, 'wb')
1749 fp = open(filename, 'wb')
1754
1750
1755 self.blockIndex = 0
1751 self.blockIndex = 0
1756
1752
1757 # guardando atributos
1753 # guardando atributos
1758 self.filename = filename
1754 self.filename = filename
1759 self.subfolder = subfolder
1755 self.subfolder = subfolder
1760 self.fp = fp
1756 self.fp = fp
1761 self.setFile = setFile
1757 self.setFile = setFile
1762 self.flagIsNewFile = 1
1758 self.flagIsNewFile = 1
1763 self.fileDate = self.dataOut.datatime.date()
1759 self.fileDate = self.dataOut.datatime.date()
1764
1760
1765 self.setFirstHeader()
1761 self.setFirstHeader()
1766
1762
1767 print '[Writing] Opening file: %s' % self.filename
1763 print '[Writing] Opening file: %s' % self.filename
1768
1764
1769 self.__writeFirstHeader()
1765 self.__writeFirstHeader()
1770
1766
1771 return 1
1767 return 1
1772
1768
1773 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1769 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1774 """
1770 """
1775 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1771 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1776
1772
1777 Inputs:
1773 Inputs:
1778 path : directory where data will be saved
1774 path : directory where data will be saved
1779 profilesPerBlock : number of profiles per block
1775 profilesPerBlock : number of profiles per block
1780 set : initial file set
1776 set : initial file set
1781 datatype : An integer number that defines data type:
1777 datatype : An integer number that defines data type:
1782 0 : int8 (1 byte)
1778 0 : int8 (1 byte)
1783 1 : int16 (2 bytes)
1779 1 : int16 (2 bytes)
1784 2 : int32 (4 bytes)
1780 2 : int32 (4 bytes)
1785 3 : int64 (8 bytes)
1781 3 : int64 (8 bytes)
1786 4 : float32 (4 bytes)
1782 4 : float32 (4 bytes)
1787 5 : double64 (8 bytes)
1783 5 : double64 (8 bytes)
1788
1784
1789 Return:
1785 Return:
1790 0 : Si no realizo un buen seteo
1786 0 : Si no realizo un buen seteo
1791 1 : Si realizo un buen seteo
1787 1 : Si realizo un buen seteo
1792 """
1788 """
1793
1789
1794 if ext == None:
1790 if ext == None:
1795 ext = self.ext
1791 ext = self.ext
1796
1792
1797 self.ext = ext.lower()
1793 self.ext = ext.lower()
1798
1794
1799 self.path = path
1795 self.path = path
1800
1796
1801 if set is None:
1797 if set is None:
1802 self.setFile = -1
1798 self.setFile = -1
1803 else:
1799 else:
1804 self.setFile = set - 1
1800 self.setFile = set - 1
1805
1801
1806 self.blocksPerFile = blocksPerFile
1802 self.blocksPerFile = blocksPerFile
1807
1803
1808 self.profilesPerBlock = profilesPerBlock
1804 self.profilesPerBlock = profilesPerBlock
1809
1805
1810 self.dataOut = dataOut
1806 self.dataOut = dataOut
1811 self.fileDate = self.dataOut.datatime.date()
1807 self.fileDate = self.dataOut.datatime.date()
1812 # By default
1808 # By default
1813 self.dtype = self.dataOut.dtype
1809 self.dtype = self.dataOut.dtype
1814
1810
1815 if datatype is not None:
1811 if datatype is not None:
1816 self.dtype = get_numpy_dtype(datatype)
1812 self.dtype = get_numpy_dtype(datatype)
1817
1813
1818 if not(self.setNextFile()):
1814 if not(self.setNextFile()):
1819 print "[Writing] There isn't a next file"
1815 print "[Writing] There isn't a next file"
1820 return 0
1816 return 0
1821
1817
1822 self.setBlockDimension()
1818 self.setBlockDimension()
1823
1819
1824 return 1
1820 return 1
1825
1821
1826 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1822 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1827
1823
1828 if not(self.isConfig):
1824 if not(self.isConfig):
1829
1825
1830 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1826 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1831 set=set, ext=ext, datatype=datatype, **kwargs)
1827 set=set, ext=ext, datatype=datatype, **kwargs)
1832 self.isConfig = True
1828 self.isConfig = True
1833
1829
1834 self.putData()
1830 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now