##// END OF EJS Templates
schain
RSS
1.0
José Chávez -
r944:0efe9dc73bde
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,34
1 from schainpy.controller import Project
2
3 desc = "A schain project"
4
5 controller = Project()
6 controller.setup(id='191', name="project", description=desc)
7
8 readUnitConf = controller.addReadUnit(datatype='VoltageReader',
9 path="/home/nanosat/schain",
10 startDate="1970/01/01",
11 endDate="2017/12/31",
12 startTime="00:00:00",
13 endTime="23:59:59",
14 online=0,
15 verbose=1,
16 walk=1,
17 )
18
19 procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId())
20
21 opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other')
22 opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist')
23
24 opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other')
25 opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
26 opObj11.addParameter(name='showprofile', value='0', format='int')
27 opObj11.addParameter(name='xmin', value='0', format='int')
28 opObj11.addParameter(name='xmax', value='24', format='int')
29 opObj11.addParameter(name='figpath', value="/home/nanosat/schain/figs", format='str')
30 opObj11.addParameter(name='wr_period', value='5', format='int')
31 opObj11.addParameter(name='exp_code', value='22', format='int')
32
33
34 controller.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="Voltage" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="plotter"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 You should install "digital_rf_hdf5" module if you want to read USRP data
Show file before | Show file after | Hide comments
@@ -1,99 +1,180
1 1 import click
2 2 import schainpy
3 3 import subprocess
4 4 import os
5 5 import sys
6 6 import glob
7 save_stdout = sys.stdout
8 sys.stdout = open('trash', 'w')
7 9 from multiprocessing import cpu_count
8 10 from schaincli import templates
9 11 from schainpy import controller_api
12 from schainpy.model import Operation, ProcessingUnit
10 13 from schainpy.utils import log
14 from importlib import import_module
15 from pydoc import locate
16 from fuzzywuzzy import process
17 sys.stdout = save_stdout
18
11 19
12 20 def print_version(ctx, param, value):
13 21 if not value or ctx.resilient_parsing:
14 22 return
15 23 click.echo(schainpy.__version__)
16 24 ctx.exit()
17 25
26
18 27 cliLogger = log.makelogger('schain cli')
28 PREFIX = 'experiment'
29
19 30
20 31 @click.command()
21 32 @click.option('--version', '-v', is_flag=True, callback=print_version, help='SChain version', type=str)
22 33 @click.option('--xml', '-x', default=None, help='run an XML file', type=click.Path(exists=True, resolve_path=True))
23 34 @click.argument('command', default='run', required=True)
24 @click.argument('nextcommand', default=None, required=False, type=click.Path(exists=True, resolve_path=True))
35 @click.argument('nextcommand', default=None, required=False, type=str)
25 36 def main(command, nextcommand, version, xml):
26 37 """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY"""
27 38 if xml is not None:
28 39 runFromXML(xml)
29 40 elif command == 'generate':
30 41 generate()
31 42 elif command == 'test':
32 43 test()
33 44 elif command == 'run':
45 runschain(nextcommand)
46 elif command == 'search':
47 search(nextcommand)
48 else:
49 log.error('Command {} is not defined'.format(command))
50
51 def check_module(possible, instance):
52 def check(x):
53 try:
54 instancia = locate('schainpy.model.{}'.format(x))
55 return isinstance(instancia(), instance)
56 except Exception as e:
57 return False
58 clean = clean_modules(possible)
59 return [x for x in clean if check(x)]
60
61
62 def clean_modules(module):
63 noEndsUnder = [x for x in module if not x.endswith('__')]
64 noStartUnder = [x for x in noEndsUnder if not x.startswith('__')]
65 noFullUpper = [x for x in noStartUnder if not x.isupper()]
66 return noFullUpper
67
68
69 def search(nextcommand):
70
71 if nextcommand is None:
72 log.error('There is no Operation/ProcessingUnit to search')
73 elif nextcommand == 'procs':
74 module = dir(import_module('schainpy.model'))
75 procs = check_module(module, ProcessingUnit)
76 try:
77 procs.remove('ProcessingUnit')
78 except Exception as e:
79 pass
80 log.success('Current ProcessingUnits are:\n\033[1m{}\033[0m'.format('\n'.join(procs)))
81
82 elif nextcommand == 'operations':
83 module = dir(import_module('schainpy.model'))
84 noProcs = [x for x in module if not x.endswith('Proc')]
85 operations = check_module(noProcs, Operation)
86 try:
87 operations.remove('Operation')
88 except Exception as e:
89 pass
90 log.success('Current Operations are:\n\033[1m{}\033[0m'.format('\n'.join(operations)))
91 else:
92 try:
93 module = locate('schainpy.model.{}'.format(nextcommand))
94 log.warning('Use this feature with caution. It may not return all the allowed arguments')
95 args = module().getAllowedArgs()
96 try:
97 args.remove('self')
98 except Exception as e:
99 pass
100 try:
101 args.remove('dataOut')
102 except Exception as e:
103 pass
104 log.success('Showing arguments of {} are:\n\033[1m{}\033[0m'.format(nextcommand, '\n'.join(args)))
105 except Exception as e:
106 log.error('Module {} does not exists'.format(nextcommand))
107 allModules = dir(import_module('schainpy.model'))
108 module = check_module(allModules, Operation)
109 module.extend(check_module(allModules, ProcessingUnit))
110 similar = process.extractOne(nextcommand, module)[0]
111 search(similar)
112
113
114 def runschain(nextcommand):
34 115 if nextcommand is None:
35 currentfiles = glob.glob('./*.py')
116 currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
36 117 numberfiles = len(currentfiles)
37 print currentfiles
38 118 if numberfiles > 1:
39 119 log.error('There is more than one file to run')
40 120 elif numberfiles == 1:
41 121 subprocess.call(['python ' + currentfiles[0]], shell=True)
42 122 else:
43 log.error('There is no file to run.')
123 log.error('There is no file to run')
44 124 else:
125 try:
45 126 subprocess.call(['python ' + nextcommand], shell=True)
46 else:
47 log.error('Command is not defined.')
127 except Exception as e:
128 log.error("I cannot run the file. Does it exists?")
48 129
49 130
50 131 def basicInputs():
51 132 inputs = {}
52 133 inputs['desc'] = click.prompt('Enter a description', default="A schain project", type=str)
53 134 inputs['name'] = click.prompt('Name of the project', default="project", type=str)
54 135 inputs['path'] = click.prompt('Data path', default=os.getcwd(), type=click.Path(exists=True, resolve_path=True))
55 136 inputs['startDate'] = click.prompt('Start date', default='1970/01/01', type=str)
56 137 inputs['endDate'] = click.prompt('End date', default='2017/12/31', type=str)
57 138 inputs['startHour'] = click.prompt('Start hour', default='00:00:00', type=str)
58 139 inputs['endHour'] = click.prompt('End hour', default='23:59:59', type=str)
59 140 inputs['figpath'] = inputs['path'] + '/figs'
60 141 return inputs
61 142
62 143
63 144 def generate():
64 145 inputs = basicInputs()
65 146 inputs['multiprocess'] = click.confirm('Is this a multiprocess script?')
66 147 if inputs['multiprocess']:
67 148 inputs['nProcess'] = click.prompt('How many process?', default=cpu_count(), type=int)
68 149 current = templates.multiprocess.format(**inputs)
69 150 else:
70 151 current = templates.basic.format(**inputs)
71 scriptname = inputs['name'] + ".py"
152 scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
72 153 script = open(scriptname, 'w')
73 154 try:
74 155 script.write(current)
75 log.success('Script {file} generated'.format(file=scriptname))
156 log.success('Script {} generated'.format(scriptname))
76 157 except Exception as e:
77 158 log.error('I cannot create the file. Do you have writing permissions?')
78 159
79 160
80 161 def test():
81 162 log.warning('testing')
82 163
83 164
84 165 def runFromXML(filename):
85 166 controller = controller_api.ControllerThread()
86 167 if not controller.readXml(filename):
87 168 return
88 169
89 170 plotterObj = controller.useExternalPlotter()
90 171
91 172 controller.start()
92 173 plotterObj.start()
93 174
94 cliLogger("Finishing all processes ...")
175 cliLogger("Finishing all processes")
95 176
96 177 controller.join(5)
97 178
98 179 cliLogger("End of script")
99 180 return
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
Show file before | Show file after | Hide comments
@@ -1,1750 +1,1794
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 import inspect
12 13 import time, datetime
13 14 #import h5py
14 15 import traceback
15 16
16 17 try:
17 18 from gevent import sleep
18 19 except:
19 20 from time import sleep
20 21
21 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 24
24 25 LOCALTIME = True
25 26
26 27 def isNumber(cad):
27 28 """
28 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 30
30 31 Excepciones:
31 32 Si un determinado string no puede ser convertido a numero
32 33 Input:
33 34 str, string al cual se le analiza para determinar si convertible a un numero o no
34 35
35 36 Return:
36 37 True : si el string es uno numerico
37 38 False : no es un string numerico
38 39 """
39 40 try:
40 41 float( cad )
41 42 return True
42 43 except:
43 44 return False
44 45
45 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 47 """
47 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 49
49 50 Inputs:
50 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 52
52 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 54 segundos contados desde 01/01/1970.
54 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 56 segundos contados desde 01/01/1970.
56 57
57 58 Return:
58 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 60 fecha especificado, de lo contrario retorna False.
60 61
61 62 Excepciones:
62 63 Si el archivo no existe o no puede ser abierto
63 64 Si la cabecera no puede ser leida.
64 65
65 66 """
66 67 basicHeaderObj = BasicHeader(LOCALTIME)
67 68
68 69 try:
69 70 fp = open(filename,'rb')
70 71 except IOError:
71 72 print "The file %s can't be opened" %(filename)
72 73 return 0
73 74
74 75 sts = basicHeaderObj.read(fp)
75 76 fp.close()
76 77
77 78 if not(sts):
78 79 print "Skipping the file %s because it has not a valid header" %(filename)
79 80 return 0
80 81
81 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 83 return 0
83 84
84 85 return 1
85 86
86 87 def isTimeInRange(thisTime, startTime, endTime):
87 88
88 89 if endTime >= startTime:
89 90 if (thisTime < startTime) or (thisTime > endTime):
90 91 return 0
91 92
92 93 return 1
93 94 else:
94 95 if (thisTime < startTime) and (thisTime > endTime):
95 96 return 0
96 97
97 98 return 1
98 99
99 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 101 """
101 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 103
103 104 Inputs:
104 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 106
106 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 108
108 109 endDate : fecha final del rango seleccionado en formato datetime.date
109 110
110 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 112
112 113 endTime : tiempo final del rango seleccionado en formato datetime.time
113 114
114 115 Return:
115 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 117 fecha especificado, de lo contrario retorna False.
117 118
118 119 Excepciones:
119 120 Si el archivo no existe o no puede ser abierto
120 121 Si la cabecera no puede ser leida.
121 122
122 123 """
123 124
124 125
125 126 try:
126 127 fp = open(filename,'rb')
127 128 except IOError:
128 129 print "The file %s can't be opened" %(filename)
129 130 return None
130 131
131 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 133 systemHeaderObj = SystemHeader()
133 134 radarControllerHeaderObj = RadarControllerHeader()
134 135 processingHeaderObj = ProcessingHeader()
135 136
136 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 138
138 139 sts = firstBasicHeaderObj.read(fp)
139 140
140 141 if not(sts):
141 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 143 return None
143 144
144 145 if not systemHeaderObj.read(fp):
145 146 return None
146 147
147 148 if not radarControllerHeaderObj.read(fp):
148 149 return None
149 150
150 151 if not processingHeaderObj.read(fp):
151 152 return None
152 153
153 154 filesize = os.path.getsize(filename)
154 155
155 156 offset = processingHeaderObj.blockSize + 24 #header size
156 157
157 158 if filesize <= offset:
158 159 print "[Reading] %s: This file has not enough data" %filename
159 160 return None
160 161
161 162 fp.seek(-offset, 2)
162 163
163 164 sts = lastBasicHeaderObj.read(fp)
164 165
165 166 fp.close()
166 167
167 168 thisDatetime = lastBasicHeaderObj.datatime
168 169 thisTime_last_block = thisDatetime.time()
169 170
170 171 thisDatetime = firstBasicHeaderObj.datatime
171 172 thisDate = thisDatetime.date()
172 173 thisTime_first_block = thisDatetime.time()
173 174
174 175 #General case
175 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 177 #-----------o----------------------------o-----------
177 178 # startTime endTime
178 179
179 180 if endTime >= startTime:
180 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 182 return None
182 183
183 184 return thisDatetime
184 185
185 186 #If endTime < startTime then endTime belongs to the next day
186 187
187 188
188 189 #<<<<<<<<<<<o o>>>>>>>>>>>
189 190 #-----------o----------------------------o-----------
190 191 # endTime startTime
191 192
192 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 194 return None
194 195
195 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 197 return None
197 198
198 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 200 return None
200 201
201 202 return thisDatetime
202 203
203 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 205 """
205 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 207
207 208 Inputs:
208 209 folder : nombre completo del directorio.
209 210 Su formato deberia ser "/path_root/?YYYYDDD"
210 211
211 212 siendo:
212 213 YYYY : Anio (ejemplo 2015)
213 214 DDD : Dia del anio (ejemplo 305)
214 215
215 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 217
217 218 endDate : fecha final del rango seleccionado en formato datetime.date
218 219
219 220 Return:
220 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 222 fecha especificado, de lo contrario retorna False.
222 223 Excepciones:
223 224 Si el directorio no tiene el formato adecuado
224 225 """
225 226
226 227 basename = os.path.basename(folder)
227 228
228 229 if not isRadarFolder(basename):
229 230 print "The folder %s has not the rigth format" %folder
230 231 return 0
231 232
232 233 if startDate and endDate:
233 234 thisDate = getDateFromRadarFolder(basename)
234 235
235 236 if thisDate < startDate:
236 237 return 0
237 238
238 239 if thisDate > endDate:
239 240 return 0
240 241
241 242 return 1
242 243
243 244 def isFileInDateRange(filename, startDate=None, endDate=None):
244 245 """
245 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 247
247 248 Inputs:
248 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 250
250 251 Su formato deberia ser "?YYYYDDDsss"
251 252
252 253 siendo:
253 254 YYYY : Anio (ejemplo 2015)
254 255 DDD : Dia del anio (ejemplo 305)
255 256 sss : set
256 257
257 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 259
259 260 endDate : fecha final del rango seleccionado en formato datetime.date
260 261
261 262 Return:
262 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 264 fecha especificado, de lo contrario retorna False.
264 265 Excepciones:
265 266 Si el archivo no tiene el formato adecuado
266 267 """
267 268
268 269 basename = os.path.basename(filename)
269 270
270 271 if not isRadarFile(basename):
271 272 print "The filename %s has not the rigth format" %filename
272 273 return 0
273 274
274 275 if startDate and endDate:
275 276 thisDate = getDateFromRadarFile(basename)
276 277
277 278 if thisDate < startDate:
278 279 return 0
279 280
280 281 if thisDate > endDate:
281 282 return 0
282 283
283 284 return 1
284 285
285 286 def getFileFromSet(path, ext, set):
286 287 validFilelist = []
287 288 fileList = os.listdir(path)
288 289
289 290 # 0 1234 567 89A BCDE
290 291 # H YYYY DDD SSS .ext
291 292
292 293 for thisFile in fileList:
293 294 try:
294 295 year = int(thisFile[1:5])
295 296 doy = int(thisFile[5:8])
296 297 except:
297 298 continue
298 299
299 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 301 continue
301 302
302 303 validFilelist.append(thisFile)
303 304
304 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 306
306 307 if len(myfile)!= 0:
307 308 return myfile[0]
308 309 else:
309 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 311 print 'the filename %s does not exist'%filename
311 312 print '...going to the last file: '
312 313
313 314 if validFilelist:
314 315 validFilelist = sorted( validFilelist, key=str.lower )
315 316 return validFilelist[-1]
316 317
317 318 return None
318 319
319 320 def getlastFileFromPath(path, ext):
320 321 """
321 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 324
324 325 Input:
325 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 327 ext : extension de los files contenidos en una carpeta
327 328
328 329 Return:
329 330 El ultimo file de una determinada carpeta, no se considera el path.
330 331 """
331 332 validFilelist = []
332 333 fileList = os.listdir(path)
333 334
334 335 # 0 1234 567 89A BCDE
335 336 # H YYYY DDD SSS .ext
336 337
337 338 for thisFile in fileList:
338 339
339 340 year = thisFile[1:5]
340 341 if not isNumber(year):
341 342 continue
342 343
343 344 doy = thisFile[5:8]
344 345 if not isNumber(doy):
345 346 continue
346 347
347 348 year = int(year)
348 349 doy = int(doy)
349 350
350 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 352 continue
352 353
353 354 validFilelist.append(thisFile)
354 355
355 356 if validFilelist:
356 357 validFilelist = sorted( validFilelist, key=str.lower )
357 358 return validFilelist[-1]
358 359
359 360 return None
360 361
361 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 363 """
363 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 366 el path exacto de un determinado file.
366 367
367 368 Example :
368 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 370
370 371 Entonces la funcion prueba con las siguientes combinaciones
371 372 .../.../y2009307367.ext
372 373 .../.../Y2009307367.ext
373 374 .../.../x2009307/y2009307367.ext
374 375 .../.../x2009307/Y2009307367.ext
375 376 .../.../X2009307/y2009307367.ext
376 377 .../.../X2009307/Y2009307367.ext
377 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 379
379 380 Return:
380 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 383 para el filename
383 384 """
384 385 fullfilename = None
385 386 find_flag = False
386 387 filename = None
387 388
388 389 prefixDirList = [None,'d','D']
389 390 if ext.lower() == ".r": #voltage
390 391 prefixFileList = ['d','D']
391 392 elif ext.lower() == ".pdata": #spectra
392 393 prefixFileList = ['p','P']
393 394 else:
394 395 return None, filename
395 396
396 397 #barrido por las combinaciones posibles
397 398 for prefixDir in prefixDirList:
398 399 thispath = path
399 400 if prefixDir != None:
400 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 402 if foldercounter == 0:
402 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 404 else:
404 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 409
409 410 if os.path.exists( fullfilename ): #verifico que exista
410 411 find_flag = True
411 412 break
412 413 if find_flag:
413 414 break
414 415
415 416 if not(find_flag):
416 417 return None, filename
417 418
418 419 return fullfilename, filename
419 420
420 421 def isRadarFolder(folder):
421 422 try:
422 423 year = int(folder[1:5])
423 424 doy = int(folder[5:8])
424 425 except:
425 426 return 0
426 427
427 428 return 1
428 429
429 430 def isRadarFile(file):
430 431 try:
431 432 year = int(file[1:5])
432 433 doy = int(file[5:8])
433 434 set = int(file[8:11])
434 435 except:
435 436 return 0
436 437
437 438 return 1
438 439
439 440 def getDateFromRadarFile(file):
440 441 try:
441 442 year = int(file[1:5])
442 443 doy = int(file[5:8])
443 444 set = int(file[8:11])
444 445 except:
445 446 return None
446 447
447 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 449 return thisDate
449 450
450 451 def getDateFromRadarFolder(folder):
451 452 try:
452 453 year = int(folder[1:5])
453 454 doy = int(folder[5:8])
454 455 except:
455 456 return None
456 457
457 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 459 return thisDate
459 460
460 461 class JRODataIO:
461 462
462 463 c = 3E8
463 464
464 465 isConfig = False
465 466
466 467 basicHeaderObj = None
467 468
468 469 systemHeaderObj = None
469 470
470 471 radarControllerHeaderObj = None
471 472
472 473 processingHeaderObj = None
473 474
474 475 dtype = None
475 476
476 477 pathList = []
477 478
478 479 filenameList = []
479 480
480 481 filename = None
481 482
482 483 ext = None
483 484
484 485 flagIsNewFile = 1
485 486
486 487 flagDiscontinuousBlock = 0
487 488
488 489 flagIsNewBlock = 0
489 490
490 491 fp = None
491 492
492 493 firstHeaderSize = 0
493 494
494 495 basicHeaderSize = 24
495 496
496 497 versionFile = 1103
497 498
498 499 fileSize = None
499 500
500 501 # ippSeconds = None
501 502
502 503 fileSizeByHeader = None
503 504
504 505 fileIndex = None
505 506
506 507 profileIndex = None
507 508
508 509 blockIndex = None
509 510
510 511 nTotalBlocks = None
511 512
512 513 maxTimeStep = 30
513 514
514 515 lastUTTime = None
515 516
516 517 datablock = None
517 518
518 519 dataOut = None
519 520
520 521 blocksize = None
521 522
522 523 getByBlock = False
523 524
524 525 def __init__(self):
525 526
526 527 raise NotImplementedError
527 528
528 529 def run(self):
529 530
530 531 raise NotImplementedError
531 532
532 533 def getDtypeWidth(self):
533 534
534 535 dtype_index = get_dtype_index(self.dtype)
535 536 dtype_width = get_dtype_width(dtype_index)
536 537
537 538 return dtype_width
538 539
540 def getAllowedArgs(self):
541 return inspect.getargspec(self.run).args
542
539 543 class JRODataReader(JRODataIO):
540 544
541 545
542 546 online = 0
543 547
544 548 realtime = 0
545 549
546 550 nReadBlocks = 0
547 551
548 552 delay = 10 #number of seconds waiting a new file
549 553
550 554 nTries = 3 #quantity tries
551 555
552 556 nFiles = 3 #number of files for searching
553 557
554 558 path = None
555 559
556 560 foldercounter = 0
557 561
558 562 flagNoMoreFiles = 0
559 563
560 564 datetimeList = []
561 565
562 566 __isFirstTimeOnline = 1
563 567
564 568 __printInfo = True
565 569
566 570 profileIndex = None
567 571
568 572 nTxs = 1
569 573
570 574 txIndex = None
571 575
572 576 #Added--------------------
573 577
574 578 selBlocksize = None
575 579
576 580 selBlocktime = None
577 581
578 582
579 583 def __init__(self):
580 584
581 585 """
582 586 This class is used to find data files
583 587
584 588 Example:
585 589 reader = JRODataReader()
586 590 fileList = reader.findDataFiles()
587 591
588 592 """
589 593 pass
590 594
591 595
592 596 def createObjByDefault(self):
593 597 """
594 598
595 599 """
596 600 raise NotImplementedError
597 601
598 602 def getBlockDimension(self):
599 603
600 604 raise NotImplementedError
601 605
602 606 def __searchFilesOffLine(self,
603 607 path,
604 608 startDate=None,
605 609 endDate=None,
606 610 startTime=datetime.time(0,0,0),
607 611 endTime=datetime.time(23,59,59),
608 612 set=None,
609 613 expLabel='',
610 614 ext='.r',
611 615 queue=None,
612 616 cursor=None,
613 617 skip=None,
614 618 walk=True):
615 619
616 620 self.filenameList = []
617 621 self.datetimeList = []
618 622
619 623 pathList = []
620 624
621 625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
622 626
623 627 if dateList == []:
624 628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
625 629 return None, None
626 630
627 631 if len(dateList) > 1:
628 632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
629 633 else:
630 634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
631 635
632 636 filenameList = []
633 637 datetimeList = []
634 638
635 639 for thisPath in pathList:
636 640 # thisPath = pathList[pathDict[file]]
637 641
638 642 fileList = glob.glob1(thisPath, "*%s" %ext)
639 643 fileList.sort()
640 644
641 645 skippedFileList = []
642 646
643 647 if cursor is not None and skip is not None:
644 648 # if cursor*skip > len(fileList):
645 649 if skip == 0:
646 650 if queue is not None:
647 651 queue.put(len(fileList))
648 652 skippedFileList = []
649 653 else:
650 654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
651 655
652 656 else:
653 657 skippedFileList = fileList
654 658
655 659 for file in skippedFileList:
656 660
657 661 filename = os.path.join(thisPath,file)
658 662
659 663 if not isFileInDateRange(filename, startDate, endDate):
660 664 continue
661 665
662 666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
663 667
664 668 if not(thisDatetime):
665 669 continue
666 670
667 671 filenameList.append(filename)
668 672 datetimeList.append(thisDatetime)
669 673
670 674 if not(filenameList):
671 675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
672 676 return None, None
673 677
674 678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
675 679 print
676 680
677 681 for i in range(len(filenameList)):
678 682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
679 683
680 684 self.filenameList = filenameList
681 685 self.datetimeList = datetimeList
682 686
683 687 return pathList, filenameList
684 688
685 689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
686 690
687 691 """
688 692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
689 693 devuelve el archivo encontrado ademas de otros datos.
690 694
691 695 Input:
692 696 path : carpeta donde estan contenidos los files que contiene data
693 697
694 698 expLabel : Nombre del subexperimento (subfolder)
695 699
696 700 ext : extension de los files
697 701
698 702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
699 703
700 704 Return:
701 705 directory : eL directorio donde esta el file encontrado
702 706 filename : el ultimo file de una determinada carpeta
703 707 year : el anho
704 708 doy : el numero de dia del anho
705 709 set : el set del archivo
706 710
707 711
708 712 """
709 713 if not os.path.isdir(path):
710 714 return None, None, None, None, None, None
711 715
712 716 dirList = []
713 717
714 718 if not walk:
715 719 fullpath = path
716 720 foldercounter = 0
717 721 else:
718 722 #Filtra solo los directorios
719 723 for thisPath in os.listdir(path):
720 724 if not os.path.isdir(os.path.join(path,thisPath)):
721 725 continue
722 726 if not isRadarFolder(thisPath):
723 727 continue
724 728
725 729 dirList.append(thisPath)
726 730
727 731 if not(dirList):
728 732 return None, None, None, None, None, None
729 733
730 734 dirList = sorted( dirList, key=str.lower )
731 735
732 736 doypath = dirList[-1]
733 737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
734 738 fullpath = os.path.join(path, doypath, expLabel)
735 739
736 740
737 741 print "[Reading] %s folder was found: " %(fullpath )
738 742
739 743 if set == None:
740 744 filename = getlastFileFromPath(fullpath, ext)
741 745 else:
742 746 filename = getFileFromSet(fullpath, ext, set)
743 747
744 748 if not(filename):
745 749 return None, None, None, None, None, None
746 750
747 751 print "[Reading] %s file was found" %(filename)
748 752
749 753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
750 754 return None, None, None, None, None, None
751 755
752 756 year = int( filename[1:5] )
753 757 doy = int( filename[5:8] )
754 758 set = int( filename[8:11] )
755 759
756 760 return fullpath, foldercounter, filename, year, doy, set
757 761
758 762 def __setNextFileOffline(self):
759 763
760 764 idFile = self.fileIndex
761 765
762 766 while (True):
763 767 idFile += 1
764 768 if not(idFile < len(self.filenameList)):
765 769 self.flagNoMoreFiles = 1
766 770 # print "[Reading] No more Files"
767 771 return 0
768 772
769 773 filename = self.filenameList[idFile]
770 774
771 775 if not(self.__verifyFile(filename)):
772 776 continue
773 777
774 778 fileSize = os.path.getsize(filename)
775 779 fp = open(filename,'rb')
776 780 break
777 781
778 782 self.flagIsNewFile = 1
779 783 self.fileIndex = idFile
780 784 self.filename = filename
781 785 self.fileSize = fileSize
782 786 self.fp = fp
783 787
784 788 # print "[Reading] Setting the file: %s"%self.filename
785 789
786 790 return 1
787 791
788 792 def __setNextFileOnline(self):
789 793 """
790 794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
791 795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
792 796 siguientes.
793 797
794 798 Affected:
795 799 self.flagIsNewFile
796 800 self.filename
797 801 self.fileSize
798 802 self.fp
799 803 self.set
800 804 self.flagNoMoreFiles
801 805
802 806 Return:
803 807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
804 808 1 : si el file fue abierto con exito y esta listo a ser leido
805 809
806 810 Excepciones:
807 811 Si un determinado file no puede ser abierto
808 812 """
809 813 nFiles = 0
810 814 fileOk_flag = False
811 815 firstTime_flag = True
812 816
813 817 self.set += 1
814 818
815 819 if self.set > 999:
816 820 self.set = 0
817 821 self.foldercounter += 1
818 822
819 823 #busca el 1er file disponible
820 824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
821 825 if fullfilename:
822 826 if self.__verifyFile(fullfilename, False):
823 827 fileOk_flag = True
824 828
825 829 #si no encuentra un file entonces espera y vuelve a buscar
826 830 if not(fileOk_flag):
827 831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
828 832
829 833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
830 834 tries = self.nTries
831 835 else:
832 836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
833 837
834 838 for nTries in range( tries ):
835 839 if firstTime_flag:
836 840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
837 841 sleep( self.delay )
838 842 else:
839 843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
840 844
841 845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
842 846 if fullfilename:
843 847 if self.__verifyFile(fullfilename):
844 848 fileOk_flag = True
845 849 break
846 850
847 851 if fileOk_flag:
848 852 break
849 853
850 854 firstTime_flag = False
851 855
852 856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
853 857 self.set += 1
854 858
855 859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
856 860 self.set = 0
857 861 self.doy += 1
858 862 self.foldercounter = 0
859 863
860 864 if fileOk_flag:
861 865 self.fileSize = os.path.getsize( fullfilename )
862 866 self.filename = fullfilename
863 867 self.flagIsNewFile = 1
864 868 if self.fp != None: self.fp.close()
865 869 self.fp = open(fullfilename, 'rb')
866 870 self.flagNoMoreFiles = 0
867 871 # print '[Reading] Setting the file: %s' % fullfilename
868 872 else:
869 873 self.fileSize = 0
870 874 self.filename = None
871 875 self.flagIsNewFile = 0
872 876 self.fp = None
873 877 self.flagNoMoreFiles = 1
874 878 # print '[Reading] No more files to read'
875 879
876 880 return fileOk_flag
877 881
878 882 def setNextFile(self):
879 883 if self.fp != None:
880 884 self.fp.close()
881 885
882 886 if self.online:
883 887 newFile = self.__setNextFileOnline()
884 888 else:
885 889 newFile = self.__setNextFileOffline()
886 890
887 891 if not(newFile):
888 892 print '[Reading] No more files to read'
889 893 return 0
890 894
891 895 if self.verbose:
892 896 print '[Reading] Setting the file: %s' % self.filename
893 897
894 898 self.__readFirstHeader()
895 899 self.nReadBlocks = 0
896 900 return 1
897 901
898 902 def __waitNewBlock(self):
899 903 """
900 904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
901 905
902 906 Si el modo de lectura es OffLine siempre retorn 0
903 907 """
904 908 if not self.online:
905 909 return 0
906 910
907 911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
908 912 return 0
909 913
910 914 currentPointer = self.fp.tell()
911 915
912 916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
913 917
914 918 for nTries in range( self.nTries ):
915 919
916 920 self.fp.close()
917 921 self.fp = open( self.filename, 'rb' )
918 922 self.fp.seek( currentPointer )
919 923
920 924 self.fileSize = os.path.getsize( self.filename )
921 925 currentSize = self.fileSize - currentPointer
922 926
923 927 if ( currentSize >= neededSize ):
924 928 self.basicHeaderObj.read(self.fp)
925 929 return 1
926 930
927 931 if self.fileSize == self.fileSizeByHeader:
928 932 # self.flagEoF = True
929 933 return 0
930 934
931 935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
932 936 sleep( self.delay )
933 937
934 938
935 939 return 0
936 940
937 941 def waitDataBlock(self,pointer_location):
938 942
939 943 currentPointer = pointer_location
940 944
941 945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
942 946
943 947 for nTries in range( self.nTries ):
944 948 self.fp.close()
945 949 self.fp = open( self.filename, 'rb' )
946 950 self.fp.seek( currentPointer )
947 951
948 952 self.fileSize = os.path.getsize( self.filename )
949 953 currentSize = self.fileSize - currentPointer
950 954
951 955 if ( currentSize >= neededSize ):
952 956 return 1
953 957
954 958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
955 959 sleep( self.delay )
956 960
957 961 return 0
958 962
959 963 def __jumpToLastBlock(self):
960 964
961 965 if not(self.__isFirstTimeOnline):
962 966 return
963 967
964 968 csize = self.fileSize - self.fp.tell()
965 969 blocksize = self.processingHeaderObj.blockSize
966 970
967 971 #salta el primer bloque de datos
968 972 if csize > self.processingHeaderObj.blockSize:
969 973 self.fp.seek(self.fp.tell() + blocksize)
970 974 else:
971 975 return
972 976
973 977 csize = self.fileSize - self.fp.tell()
974 978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
975 979 while True:
976 980
977 981 if self.fp.tell()<self.fileSize:
978 982 self.fp.seek(self.fp.tell() + neededsize)
979 983 else:
980 984 self.fp.seek(self.fp.tell() - neededsize)
981 985 break
982 986
983 987 # csize = self.fileSize - self.fp.tell()
984 988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
985 989 # factor = int(csize/neededsize)
986 990 # if factor > 0:
987 991 # self.fp.seek(self.fp.tell() + factor*neededsize)
988 992
989 993 self.flagIsNewFile = 0
990 994 self.__isFirstTimeOnline = 0
991 995
992 996 def __setNewBlock(self):
993 997
994 998 if self.fp == None:
995 999 return 0
996 1000
997 1001 # if self.online:
998 1002 # self.__jumpToLastBlock()
999 1003
1000 1004 if self.flagIsNewFile:
1001 1005 self.lastUTTime = self.basicHeaderObj.utc
1002 1006 return 1
1003 1007
1004 1008 if self.realtime:
1005 1009 self.flagDiscontinuousBlock = 1
1006 1010 if not(self.setNextFile()):
1007 1011 return 0
1008 1012 else:
1009 1013 return 1
1010 1014
1011 1015 currentSize = self.fileSize - self.fp.tell()
1012 1016 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1013 1017
1014 1018 if (currentSize >= neededSize):
1015 1019 self.basicHeaderObj.read(self.fp)
1016 1020 self.lastUTTime = self.basicHeaderObj.utc
1017 1021 return 1
1018 1022
1019 1023 if self.__waitNewBlock():
1020 1024 self.lastUTTime = self.basicHeaderObj.utc
1021 1025 return 1
1022 1026
1023 1027 if not(self.setNextFile()):
1024 1028 return 0
1025 1029
1026 1030 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1027 1031 self.lastUTTime = self.basicHeaderObj.utc
1028 1032
1029 1033 self.flagDiscontinuousBlock = 0
1030 1034
1031 1035 if deltaTime > self.maxTimeStep:
1032 1036 self.flagDiscontinuousBlock = 1
1033 1037
1034 1038 return 1
1035 1039
1036 1040 def readNextBlock(self):
1037 1041
1038 1042 #Skip block out of startTime and endTime
1039 1043 while True:
1040 1044 if not(self.__setNewBlock()):
1041 1045 return 0
1042 1046
1043 1047 if not(self.readBlock()):
1044 1048 return 0
1045 1049
1046 1050 self.getBasicHeader()
1047 1051
1048 1052 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1049 1053
1050 1054 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1051 1055 self.processingHeaderObj.dataBlocksPerFile,
1052 1056 self.dataOut.datatime.ctime())
1053 1057 continue
1054 1058
1055 1059 break
1056 1060
1057 1061 if self.verbose:
1058 1062 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1059 1063 self.processingHeaderObj.dataBlocksPerFile,
1060 1064 self.dataOut.datatime.ctime())
1061 1065 return 1
1062 1066
1063 1067 def __readFirstHeader(self):
1064 1068
1065 1069 self.basicHeaderObj.read(self.fp)
1066 1070 self.systemHeaderObj.read(self.fp)
1067 1071 self.radarControllerHeaderObj.read(self.fp)
1068 1072 self.processingHeaderObj.read(self.fp)
1069 1073
1070 1074 self.firstHeaderSize = self.basicHeaderObj.size
1071 1075
1072 1076 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1073 1077 if datatype == 0:
1074 1078 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1075 1079 elif datatype == 1:
1076 1080 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1077 1081 elif datatype == 2:
1078 1082 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1079 1083 elif datatype == 3:
1080 1084 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1081 1085 elif datatype == 4:
1082 1086 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1083 1087 elif datatype == 5:
1084 1088 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1085 1089 else:
1086 1090 raise ValueError, 'Data type was not defined'
1087 1091
1088 1092 self.dtype = datatype_str
1089 1093 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1090 1094 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1091 1095 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1092 1096 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1093 1097 self.getBlockDimension()
1094 1098
1095 1099 def __verifyFile(self, filename, msgFlag=True):
1096 1100
1097 1101 msg = None
1098 1102
1099 1103 try:
1100 1104 fp = open(filename, 'rb')
1101 1105 except IOError:
1102 1106
1103 1107 if msgFlag:
1104 1108 print "[Reading] File %s can't be opened" % (filename)
1105 1109
1106 1110 return False
1107 1111
1108 1112 currentPosition = fp.tell()
1109 1113 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1110 1114
1111 1115 if neededSize == 0:
1112 1116 basicHeaderObj = BasicHeader(LOCALTIME)
1113 1117 systemHeaderObj = SystemHeader()
1114 1118 radarControllerHeaderObj = RadarControllerHeader()
1115 1119 processingHeaderObj = ProcessingHeader()
1116 1120
1117 1121 if not( basicHeaderObj.read(fp) ):
1118 1122 fp.close()
1119 1123 return False
1120 1124
1121 1125 if not( systemHeaderObj.read(fp) ):
1122 1126 fp.close()
1123 1127 return False
1124 1128
1125 1129 if not( radarControllerHeaderObj.read(fp) ):
1126 1130 fp.close()
1127 1131 return False
1128 1132
1129 1133 if not( processingHeaderObj.read(fp) ):
1130 1134 fp.close()
1131 1135 return False
1132 1136
1133 1137 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1134 1138 else:
1135 1139 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1136 1140
1137 1141 fp.close()
1138 1142
1139 1143 fileSize = os.path.getsize(filename)
1140 1144 currentSize = fileSize - currentPosition
1141 1145
1142 1146 if currentSize < neededSize:
1143 1147 if msgFlag and (msg != None):
1144 1148 print msg
1145 1149 return False
1146 1150
1147 1151 return True
1148 1152
1149 1153 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1150 1154
1151 1155 path_empty = True
1152 1156
1153 1157 dateList = []
1154 1158 pathList = []
1155 1159
1156 1160 multi_path = path.split(',')
1157 1161
1158 1162 if not walk:
1159 1163
1160 1164 for single_path in multi_path:
1161 1165
1162 1166 if not os.path.isdir(single_path):
1163 1167 continue
1164 1168
1165 1169 fileList = glob.glob1(single_path, "*"+ext)
1166 1170
1167 1171 if not fileList:
1168 1172 continue
1169 1173
1170 1174 path_empty = False
1171 1175
1172 1176 fileList.sort()
1173 1177
1174 1178 for thisFile in fileList:
1175 1179
1176 1180 if not os.path.isfile(os.path.join(single_path, thisFile)):
1177 1181 continue
1178 1182
1179 1183 if not isRadarFile(thisFile):
1180 1184 continue
1181 1185
1182 1186 if not isFileInDateRange(thisFile, startDate, endDate):
1183 1187 continue
1184 1188
1185 1189 thisDate = getDateFromRadarFile(thisFile)
1186 1190
1187 1191 if thisDate in dateList:
1188 1192 continue
1189 1193
1190 1194 dateList.append(thisDate)
1191 1195 pathList.append(single_path)
1192 1196
1193 1197 else:
1194 1198 for single_path in multi_path:
1195 1199
1196 1200 if not os.path.isdir(single_path):
1197 1201 continue
1198 1202
1199 1203 dirList = []
1200 1204
1201 1205 for thisPath in os.listdir(single_path):
1202 1206
1203 1207 if not os.path.isdir(os.path.join(single_path,thisPath)):
1204 1208 continue
1205 1209
1206 1210 if not isRadarFolder(thisPath):
1207 1211 continue
1208 1212
1209 1213 if not isFolderInDateRange(thisPath, startDate, endDate):
1210 1214 continue
1211 1215
1212 1216 dirList.append(thisPath)
1213 1217
1214 1218 if not dirList:
1215 1219 continue
1216 1220
1217 1221 dirList.sort()
1218 1222
1219 1223 for thisDir in dirList:
1220 1224
1221 1225 datapath = os.path.join(single_path, thisDir, expLabel)
1222 1226 fileList = glob.glob1(datapath, "*"+ext)
1223 1227
1224 1228 if not fileList:
1225 1229 continue
1226 1230
1227 1231 path_empty = False
1228 1232
1229 1233 thisDate = getDateFromRadarFolder(thisDir)
1230 1234
1231 1235 pathList.append(datapath)
1232 1236 dateList.append(thisDate)
1233 1237
1234 1238 dateList.sort()
1235 1239
1236 1240 if walk:
1237 1241 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1238 1242 else:
1239 1243 pattern_path = multi_path[0]
1240 1244
1241 1245 if path_empty:
1242 1246 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1243 1247 else:
1244 1248 if not dateList:
1245 1249 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1246 1250
1247 1251 if include_path:
1248 1252 return dateList, pathList
1249 1253
1250 1254 return dateList
1251 1255
1252 1256 def setup(self,
1253 1257 path=None,
1254 1258 startDate=None,
1255 1259 endDate=None,
1256 1260 startTime=datetime.time(0,0,0),
1257 1261 endTime=datetime.time(23,59,59),
1258 1262 set=None,
1259 1263 expLabel = "",
1260 1264 ext = None,
1261 1265 online = False,
1262 1266 delay = 60,
1263 1267 walk = True,
1264 1268 getblock = False,
1265 1269 nTxs = 1,
1266 1270 realtime=False,
1267 1271 blocksize=None,
1268 1272 blocktime=None,
1269 1273 queue=None,
1270 1274 skip=None,
1271 1275 cursor=None,
1272 1276 warnings=True,
1273 1277 verbose=True):
1274 1278
1275 1279 if path == None:
1276 1280 raise ValueError, "[Reading] The path is not valid"
1277 1281
1278 1282 if ext == None:
1279 1283 ext = self.ext
1280 1284
1281 1285 if online:
1282 1286 print "[Reading] Searching files in online mode..."
1283 1287
1284 1288 for nTries in range( self.nTries ):
1285 1289 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1286 1290
1287 1291 if fullpath:
1288 1292 break
1289 1293
1290 1294 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1291 1295 sleep( self.delay )
1292 1296
1293 1297 if not(fullpath):
1294 1298 print "[Reading] There 'isn't any valid file in %s" % path
1295 1299 return
1296 1300
1297 1301 self.year = year
1298 1302 self.doy = doy
1299 1303 self.set = set - 1
1300 1304 self.path = path
1301 1305 self.foldercounter = foldercounter
1302 1306 last_set = None
1303 1307
1304 1308 else:
1305 1309 print "[Reading] Searching files in offline mode ..."
1306 1310 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1307 1311 startTime=startTime, endTime=endTime,
1308 1312 set=set, expLabel=expLabel, ext=ext,
1309 1313 walk=walk, cursor=cursor,
1310 1314 skip=skip, queue=queue)
1311 1315
1312 1316 if not(pathList):
1313 1317 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1314 1318 # datetime.datetime.combine(startDate,startTime).ctime(),
1315 1319 # datetime.datetime.combine(endDate,endTime).ctime())
1316 1320
1317 1321 # sys.exit(-1)
1318 1322
1319 1323 self.fileIndex = -1
1320 1324 self.pathList = []
1321 1325 self.filenameList = []
1322 1326 return
1323 1327
1324 1328 self.fileIndex = -1
1325 1329 self.pathList = pathList
1326 1330 self.filenameList = filenameList
1327 1331 file_name = os.path.basename(filenameList[-1])
1328 1332 basename, ext = os.path.splitext(file_name)
1329 1333 last_set = int(basename[-3:])
1330 1334
1331 1335 self.online = online
1332 1336 self.realtime = realtime
1333 1337 self.delay = delay
1334 1338 ext = ext.lower()
1335 1339 self.ext = ext
1336 1340 self.getByBlock = getblock
1337 1341 self.nTxs = nTxs
1338 1342 self.startTime = startTime
1339 1343 self.endTime = endTime
1340 1344
1341 1345 #Added-----------------
1342 1346 self.selBlocksize = blocksize
1343 1347 self.selBlocktime = blocktime
1344 1348
1345 1349 # Verbose-----------
1346 1350 self.verbose = verbose
1347 1351 self.warnings = warnings
1348 1352
1349 1353 if not(self.setNextFile()):
1350 1354 if (startDate!=None) and (endDate!=None):
1351 1355 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1352 1356 elif startDate != None:
1353 1357 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1354 1358 else:
1355 1359 print "[Reading] No files"
1356 1360
1357 1361 self.fileIndex = -1
1358 1362 self.pathList = []
1359 1363 self.filenameList = []
1360 1364 return
1361 1365
1362 1366 # self.getBasicHeader()
1363 1367
1364 1368 if last_set != None:
1365 1369 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1366 1370 return
1367 1371
1368 1372 def getBasicHeader(self):
1369 1373
1370 1374 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1371 1375
1372 1376 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1373 1377
1374 1378 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1375 1379
1376 1380 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1377 1381
1378 1382 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1379 1383
1380 1384 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1381 1385
1382 1386 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1383 1387
1384 1388 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1385 1389
1386 1390
1387 1391 def getFirstHeader(self):
1388 1392
1389 1393 raise NotImplementedError
1390 1394
1391 1395 def getData(self):
1392 1396
1393 1397 raise NotImplementedError
1394 1398
1395 1399 def hasNotDataInBuffer(self):
1396 1400
1397 1401 raise NotImplementedError
1398 1402
1399 1403 def readBlock(self):
1400 1404
1401 1405 raise NotImplementedError
1402 1406
1403 1407 def isEndProcess(self):
1404 1408
1405 1409 return self.flagNoMoreFiles
1406 1410
1407 1411 def printReadBlocks(self):
1408 1412
1409 1413 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1410 1414
1411 1415 def printTotalBlocks(self):
1412 1416
1413 1417 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1414 1418
1415 1419 def printNumberOfBlock(self):
1416 1420
1417 1421 if self.flagIsNewBlock:
1418 1422 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1419 1423 self.processingHeaderObj.dataBlocksPerFile,
1420 1424 self.dataOut.datatime.ctime())
1421 1425
1422 1426 def printInfo(self):
1423 1427
1424 1428 if self.__printInfo == False:
1425 1429 return
1426 1430
1427 1431 self.basicHeaderObj.printInfo()
1428 1432 self.systemHeaderObj.printInfo()
1429 1433 self.radarControllerHeaderObj.printInfo()
1430 1434 self.processingHeaderObj.printInfo()
1431 1435
1432 1436 self.__printInfo = False
1433 1437
1434 1438
1435 def run(self, **kwargs):
1439 def run(self,
1440 path=None,
1441 startDate=None,
1442 endDate=None,
1443 startTime=datetime.time(0,0,0),
1444 endTime=datetime.time(23,59,59),
1445 set=None,
1446 expLabel = "",
1447 ext = None,
1448 online = False,
1449 delay = 60,
1450 walk = True,
1451 getblock = False,
1452 nTxs = 1,
1453 realtime=False,
1454 blocksize=None,
1455 blocktime=None,
1456 queue=None,
1457 skip=None,
1458 cursor=None,
1459 warnings=True,
1460 verbose=True, **kwargs):
1436 1461
1437 1462 if not(self.isConfig):
1438
1439 1463 # self.dataOut = dataOut
1440 self.setup(**kwargs)
1464 self.setup( path=path,
1465 startDate=startDate,
1466 endDate=endDate,
1467 startTime=startTime,
1468 endTime=endTime,
1469 set=set,
1470 expLabel=expLabel,
1471 ext=ext,
1472 online=online,
1473 delay=delay,
1474 walk=walk,
1475 getblock=getblock,
1476 nTxs=nTxs,
1477 realtime=realtime,
1478 blocksize=blocksize,
1479 blocktime=blocktime,
1480 queue=queue,
1481 skip=skip,
1482 cursor=cursor,
1483 warnings=warnings,
1484 verbose=verbose)
1441 1485 self.isConfig = True
1442 1486
1443 1487 self.getData()
1444 1488
1445 1489 class JRODataWriter(JRODataIO):
1446 1490
1447 1491 """
1448 1492 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1449 1493 de los datos siempre se realiza por bloques.
1450 1494 """
1451 1495
1452 1496 blockIndex = 0
1453 1497
1454 1498 path = None
1455 1499
1456 1500 setFile = None
1457 1501
1458 1502 profilesPerBlock = None
1459 1503
1460 1504 blocksPerFile = None
1461 1505
1462 1506 nWriteBlocks = 0
1463 1507
1464 1508 fileDate = None
1465 1509
1466 1510 def __init__(self, dataOut=None):
1467 1511 raise NotImplementedError
1468 1512
1469 1513
1470 1514 def hasAllDataInBuffer(self):
1471 1515 raise NotImplementedError
1472 1516
1473 1517
1474 1518 def setBlockDimension(self):
1475 1519 raise NotImplementedError
1476 1520
1477 1521
1478 1522 def writeBlock(self):
1479 1523 raise NotImplementedError
1480 1524
1481 1525
1482 1526 def putData(self):
1483 1527 raise NotImplementedError
1484 1528
1485 1529
1486 1530 def getProcessFlags(self):
1487 1531
1488 1532 processFlags = 0
1489 1533
1490 1534 dtype_index = get_dtype_index(self.dtype)
1491 1535 procflag_dtype = get_procflag_dtype(dtype_index)
1492 1536
1493 1537 processFlags += procflag_dtype
1494 1538
1495 1539 if self.dataOut.flagDecodeData:
1496 1540 processFlags += PROCFLAG.DECODE_DATA
1497 1541
1498 1542 if self.dataOut.flagDeflipData:
1499 1543 processFlags += PROCFLAG.DEFLIP_DATA
1500 1544
1501 1545 if self.dataOut.code is not None:
1502 1546 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1503 1547
1504 1548 if self.dataOut.nCohInt > 1:
1505 1549 processFlags += PROCFLAG.COHERENT_INTEGRATION
1506 1550
1507 1551 if self.dataOut.type == "Spectra":
1508 1552 if self.dataOut.nIncohInt > 1:
1509 1553 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1510 1554
1511 1555 if self.dataOut.data_dc is not None:
1512 1556 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1513 1557
1514 1558 if self.dataOut.flagShiftFFT:
1515 1559 processFlags += PROCFLAG.SHIFT_FFT_DATA
1516 1560
1517 1561 return processFlags
1518 1562
1519 1563 def setBasicHeader(self):
1520 1564
1521 1565 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1522 1566 self.basicHeaderObj.version = self.versionFile
1523 1567 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1524 1568
1525 1569 utc = numpy.floor(self.dataOut.utctime)
1526 1570 milisecond = (self.dataOut.utctime - utc)* 1000.0
1527 1571
1528 1572 self.basicHeaderObj.utc = utc
1529 1573 self.basicHeaderObj.miliSecond = milisecond
1530 1574 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1531 1575 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1532 1576 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1533 1577
1534 1578 def setFirstHeader(self):
1535 1579 """
1536 1580 Obtiene una copia del First Header
1537 1581
1538 1582 Affected:
1539 1583
1540 1584 self.basicHeaderObj
1541 1585 self.systemHeaderObj
1542 1586 self.radarControllerHeaderObj
1543 1587 self.processingHeaderObj self.
1544 1588
1545 1589 Return:
1546 1590 None
1547 1591 """
1548 1592
1549 1593 raise NotImplementedError
1550 1594
1551 1595 def __writeFirstHeader(self):
1552 1596 """
1553 1597 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1554 1598
1555 1599 Affected:
1556 1600 __dataType
1557 1601
1558 1602 Return:
1559 1603 None
1560 1604 """
1561 1605
1562 1606 # CALCULAR PARAMETROS
1563 1607
1564 1608 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1565 1609 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1566 1610
1567 1611 self.basicHeaderObj.write(self.fp)
1568 1612 self.systemHeaderObj.write(self.fp)
1569 1613 self.radarControllerHeaderObj.write(self.fp)
1570 1614 self.processingHeaderObj.write(self.fp)
1571 1615
1572 1616 def __setNewBlock(self):
1573 1617 """
1574 1618 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1575 1619
1576 1620 Return:
1577 1621 0 : si no pudo escribir nada
1578 1622 1 : Si escribio el Basic el First Header
1579 1623 """
1580 1624 if self.fp == None:
1581 1625 self.setNextFile()
1582 1626
1583 1627 if self.flagIsNewFile:
1584 1628 return 1
1585 1629
1586 1630 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1587 1631 self.basicHeaderObj.write(self.fp)
1588 1632 return 1
1589 1633
1590 1634 if not( self.setNextFile() ):
1591 1635 return 0
1592 1636
1593 1637 return 1
1594 1638
1595 1639
1596 1640 def writeNextBlock(self):
1597 1641 """
1598 1642 Selecciona el bloque siguiente de datos y los escribe en un file
1599 1643
1600 1644 Return:
1601 1645 0 : Si no hizo pudo escribir el bloque de datos
1602 1646 1 : Si no pudo escribir el bloque de datos
1603 1647 """
1604 1648 if not( self.__setNewBlock() ):
1605 1649 return 0
1606 1650
1607 1651 self.writeBlock()
1608 1652
1609 1653 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1610 1654 self.processingHeaderObj.dataBlocksPerFile)
1611 1655
1612 1656 return 1
1613 1657
1614 1658 def setNextFile(self):
1615 1659 """
1616 1660 Determina el siguiente file que sera escrito
1617 1661
1618 1662 Affected:
1619 1663 self.filename
1620 1664 self.subfolder
1621 1665 self.fp
1622 1666 self.setFile
1623 1667 self.flagIsNewFile
1624 1668
1625 1669 Return:
1626 1670 0 : Si el archivo no puede ser escrito
1627 1671 1 : Si el archivo esta listo para ser escrito
1628 1672 """
1629 1673 ext = self.ext
1630 1674 path = self.path
1631 1675
1632 1676 if self.fp != None:
1633 1677 self.fp.close()
1634 1678
1635 1679 timeTuple = time.localtime( self.dataOut.utctime)
1636 1680 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1637 1681
1638 1682 fullpath = os.path.join( path, subfolder )
1639 1683 setFile = self.setFile
1640 1684
1641 1685 if not( os.path.exists(fullpath) ):
1642 1686 os.mkdir(fullpath)
1643 1687 setFile = -1 #inicializo mi contador de seteo
1644 1688 else:
1645 1689 filesList = os.listdir( fullpath )
1646 1690 if len( filesList ) > 0:
1647 1691 filesList = sorted( filesList, key=str.lower )
1648 1692 filen = filesList[-1]
1649 1693 # el filename debera tener el siguiente formato
1650 1694 # 0 1234 567 89A BCDE (hex)
1651 1695 # x YYYY DDD SSS .ext
1652 1696 if isNumber( filen[8:11] ):
1653 1697 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1654 1698 else:
1655 1699 setFile = -1
1656 1700 else:
1657 1701 setFile = -1 #inicializo mi contador de seteo
1658 1702
1659 1703 setFile += 1
1660 1704
1661 1705 #If this is a new day it resets some values
1662 1706 if self.dataOut.datatime.date() > self.fileDate:
1663 1707 setFile = 0
1664 1708 self.nTotalBlocks = 0
1665 1709
1666 1710 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1667 1711
1668 1712 filename = os.path.join( path, subfolder, filen )
1669 1713
1670 1714 fp = open( filename,'wb' )
1671 1715
1672 1716 self.blockIndex = 0
1673 1717
1674 1718 #guardando atributos
1675 1719 self.filename = filename
1676 1720 self.subfolder = subfolder
1677 1721 self.fp = fp
1678 1722 self.setFile = setFile
1679 1723 self.flagIsNewFile = 1
1680 1724 self.fileDate = self.dataOut.datatime.date()
1681 1725
1682 1726 self.setFirstHeader()
1683 1727
1684 1728 print '[Writing] Opening file: %s'%self.filename
1685 1729
1686 1730 self.__writeFirstHeader()
1687 1731
1688 1732 return 1
1689 1733
1690 1734 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1691 1735 """
1692 1736 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1693 1737
1694 1738 Inputs:
1695 1739 path : directory where data will be saved
1696 1740 profilesPerBlock : number of profiles per block
1697 1741 set : initial file set
1698 1742 datatype : An integer number that defines data type:
1699 1743 0 : int8 (1 byte)
1700 1744 1 : int16 (2 bytes)
1701 1745 2 : int32 (4 bytes)
1702 1746 3 : int64 (8 bytes)
1703 1747 4 : float32 (4 bytes)
1704 1748 5 : double64 (8 bytes)
1705 1749
1706 1750 Return:
1707 1751 0 : Si no realizo un buen seteo
1708 1752 1 : Si realizo un buen seteo
1709 1753 """
1710 1754
1711 1755 if ext == None:
1712 1756 ext = self.ext
1713 1757
1714 1758 self.ext = ext.lower()
1715 1759
1716 1760 self.path = path
1717 1761
1718 1762 if set is None:
1719 1763 self.setFile = -1
1720 1764 else:
1721 1765 self.setFile = set - 1
1722 1766
1723 1767 self.blocksPerFile = blocksPerFile
1724 1768
1725 1769 self.profilesPerBlock = profilesPerBlock
1726 1770
1727 1771 self.dataOut = dataOut
1728 1772 self.fileDate = self.dataOut.datatime.date()
1729 1773 #By default
1730 1774 self.dtype = self.dataOut.dtype
1731 1775
1732 1776 if datatype is not None:
1733 1777 self.dtype = get_numpy_dtype(datatype)
1734 1778
1735 1779 if not(self.setNextFile()):
1736 1780 print "[Writing] There isn't a next file"
1737 1781 return 0
1738 1782
1739 1783 self.setBlockDimension()
1740 1784
1741 1785 return 1
1742 1786
1743 def run(self, dataOut, **kwargs):
1787 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1744 1788
1745 1789 if not(self.isConfig):
1746 1790
1747 self.setup(dataOut, **kwargs)
1791 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1748 1792 self.isConfig = True
1749 1793
1750 1794 self.putData()
Show file before | Show file after | Hide comments
@@ -1,46 +1,45
1 1 """.
2
3 2 SCHAINPY - LOG
4 3 Simple helper for log standarization
5 4 Usage:
6 5 from schainpy.utils import log
7 6 log.error('A kitten died beacuse of you')
8 7 log.warning('You are doing it wrong but what the heck, I'll allow it)
9 8 log.succes('YOU ROCK!')
10 9 To create your own logger inside your class do it like this:
11 10 from schainpy.utils import log
12 11 awesomeLogger = log.makelogger("never gonna", bg="red", fg="white")
13 12 awesomeLogger('give you up')
14 13 which will look like this:
15 14 [NEVER GONNA] - give you up
16 15 with color red as background and white as foreground.
17 16 """
18 17
19 18 import click
20 19
21 20
22 21 def warning(message):
23 22 click.echo(click.style('[WARNING] - ' + message, fg='yellow'))
24 23 pass
25 24
26 25
27 26 def error(message):
28 click.echo(click.style('[ERROR] - ' + message, bg='red', fg='white'))
27 click.echo(click.style('[ERROR] - ' + message, fg='red'))
29 28 pass
30 29
31 30
32 31 def success(message):
33 click.echo(click.style('[SUCESS] - ' + message, bg='green', fg='white'))
32 click.echo(click.style(message, fg='green'))
34 33 pass
35 34
36 35
37 36 def log(message):
38 37 click.echo('[LOG] - ' + message)
39 38 pass
40 39
41 40
42 41 def makelogger(topic, bg='reset', fg='reset'):
43 42 def func(message):
44 43 click.echo(click.style('[{}] - '.format(topic.upper()) + message,
45 44 bg=bg, fg=fg))
46 45 return func
Show file before | Show file after | Hide comments
@@ -1,56 +1,57
1 1 """.
2 2
3 3 Created on Jul 16, 2014
4 4
5 5 @author: Miguel Urco
6 6 """
7 7
8 8 from schainpy import __version__
9 9 from setuptools import setup, Extension
10 10
11 11 setup(name="schainpy",
12 12 version=__version__,
13 13 description="Python tools to read, write and process Jicamarca data",
14 14 author="Miguel Urco",
15 15 author_email="miguel.urco@jro.igp.gob.pe",
16 16 url="http://jro.igp.gob.pe",
17 17 packages={'schainpy',
18 18 'schainpy.model',
19 19 'schainpy.model.data',
20 20 'schainpy.model.graphics',
21 21 'schainpy.model.io',
22 22 'schainpy.model.proc',
23 23 'schainpy.model.serializer',
24 24 'schainpy.model.utils',
25 25 'schainpy.gui',
26 26 'schainpy.gui.figures',
27 27 'schainpy.gui.viewcontroller',
28 28 'schainpy.gui.viewer',
29 29 'schainpy.gui.viewer.windows'},
30 30 ext_package='schainpy',
31 31 py_modules=[''],
32 32 package_data={'': ['schain.conf.template'],
33 33 'schainpy.gui.figures': ['*.png', '*.jpg'],
34 34 },
35 35 include_package_data=False,
36 36 entry_points={
37 37 'console_scripts': [
38 38 'schain = schaincli.cli:main',
39 39 ],
40 40 },
41 41 scripts=['schainpy/gui/schainGUI'],
42 42 ext_modules=[Extension("cSchain", ["schainpy/model/proc/extensions.c"])],
43 43 install_requires=[
44 44 "scipy >= 0.14.0",
45 45 "h5py >= 2.2.1",
46 46 "matplotlib >= 1.4.2",
47 47 "pyfits >= 3.4",
48 48 "numpy >= 1.11.2",
49 49 "paramiko >= 2.1.2",
50 50 "paho-mqtt >= 1.2",
51 51 "zmq",
52 52 "fuzzywuzzy",
53 53 "click",
54 "colorama"
54 "colorama",
55 "python-Levenshtein"
55 56 ],
56 57 )
General Comments 0
You need to be logged in to leave comments. Login now