##// END OF EJS Templates
schain
RSS
Merge pull request #1 from schain cli...
jespinoza -
r952:1bbb28e62202 merge
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,34
1 from schainpy.controller import Project
2
3 desc = "A schain project"
4
5 controller = Project()
6 controller.setup(id='191', name="project", description=desc)
7
8 readUnitConf = controller.addReadUnit(datatype='VoltageReader',
9 path="/home/nanosat/schain",
10 startDate="1970/01/01",
11 endDate="2017/12/31",
12 startTime="00:00:00",
13 endTime="23:59:59",
14 online=0,
15 verbose=1,
16 walk=1,
17 )
18
19 procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId())
20
21 opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other')
22 opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist')
23
24 opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other')
25 opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
26 opObj11.addParameter(name='showprofile', value='0', format='int')
27 opObj11.addParameter(name='xmin', value='0', format='int')
28 opObj11.addParameter(name='xmax', value='24', format='int')
29 opObj11.addParameter(name='figpath', value="/home/nanosat/schain/figs", format='str')
30 opObj11.addParameter(name='wr_period', value='5', format='int')
31 opObj11.addParameter(name='exp_code', value='22', format='int')
32
33
34 controller.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="Voltage" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="plotter"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,9
1 # schaing
2
3 Command Line Interface for SIGNAL CHAIN - jro
4
5 # Usage
6
7 To use it:
8
9 $ schain-cli --help
Show file before | Show file after | Hide comments
1 NO CONTENT: new file 100644
Show file before | Show file after | Hide comments
@@ -0,0 +1,34
1 from schainpy.controller import Project
2
3 desc = "asdasddsad"
4
5 controller = Project()
6 controller.setup(id='191', name="asdasd", description=desc)
7
8 readUnitConf = controller.addReadUnit(datatype='VoltageReader',
9 path="/home/nanosat/schain/schain-cli",
10 startDate="1970/01/01",
11 endDate="2017/12/31",
12 startTime="00:00:00",
13 endTime="23:59:59",
14 online=0,
15 verbose=1,
16 walk=1,
17 )
18
19 procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId())
20
21 opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other')
22 opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist')
23
24 opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other')
25 opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
26 opObj11.addParameter(name='showprofile', value='0', format='int')
27 opObj11.addParameter(name='xmin', value='0', format='int')
28 opObj11.addParameter(name='xmax', value='24', format='int')
29 opObj11.addParameter(name='figpath', value="/home/nanosat/schain/schain-cli/figs", format='str')
30 opObj11.addParameter(name='wr_period', value='5', format='int')
31 opObj11.addParameter(name='exp_code', value='22', format='int')
32
33
34 controller.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,188
1 import click
2 import schainpy
3 import subprocess
4 import os
5 import sys
6 import glob
7 save_stdout = sys.stdout
8 sys.stdout = open('trash', 'w')
9 from multiprocessing import cpu_count
10 from schaincli import templates
11 from schainpy import controller_api
12 from schainpy.model import Operation, ProcessingUnit
13 from schainpy.utils import log
14 from importlib import import_module
15 from pydoc import locate
16 from fuzzywuzzy import process
17 sys.stdout = save_stdout
18
19
20 def print_version(ctx, param, value):
21 if not value or ctx.resilient_parsing:
22 return
23 click.echo(schainpy.__version__)
24 ctx.exit()
25
26
27 cliLogger = log.makelogger('schain cli')
28 PREFIX = 'experiment'
29
30
31 @click.command()
32 @click.option('--version', '-v', is_flag=True, callback=print_version, help='SChain version', type=str)
33 @click.option('--xml', '-x', default=None, help='run an XML file', type=click.Path(exists=True, resolve_path=True))
34 @click.argument('command', default='run', required=True)
35 @click.argument('nextcommand', default=None, required=False, type=str)
36 def main(command, nextcommand, version, xml):
37 """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY \n
38 Available commands.\n
39 --xml: runs a schain XML generated file\n
40 run: runs any python script starting 'experiment_'\n
41 generate: generates a template schain script\n
42 search: return avilable operations, procs or arguments of the give operation/proc\n"""
43 if xml is not None:
44 runFromXML(xml)
45 elif command == 'generate':
46 generate()
47 elif command == 'test':
48 test()
49 elif command == 'run':
50 runschain(nextcommand)
51 elif command == 'search':
52 search(nextcommand)
53 else:
54 log.error('Command {} is not defined'.format(command))
55
56 def check_module(possible, instance):
57 def check(x):
58 try:
59 instancia = locate('schainpy.model.{}'.format(x))
60 return isinstance(instancia(), instance)
61 except Exception as e:
62 return False
63 clean = clean_modules(possible)
64 return [x for x in clean if check(x)]
65
66
67 def clean_modules(module):
68 noEndsUnder = [x for x in module if not x.endswith('__')]
69 noStartUnder = [x for x in noEndsUnder if not x.startswith('__')]
70 noFullUpper = [x for x in noStartUnder if not x.isupper()]
71 return noFullUpper
72
73
74 def search(nextcommand):
75 if nextcommand is None:
76 log.error('There is no Operation/ProcessingUnit to search')
77 elif nextcommand == 'procs':
78 module = dir(import_module('schainpy.model'))
79 procs = check_module(module, ProcessingUnit)
80 try:
81 procs.remove('ProcessingUnit')
82 except Exception as e:
83 pass
84 log.success('Current ProcessingUnits are:\n\033[1m{}\033[0m'.format('\n'.join(procs)))
85
86 elif nextcommand == 'operations':
87 module = dir(import_module('schainpy.model'))
88 noProcs = [x for x in module if not x.endswith('Proc')]
89 operations = check_module(noProcs, Operation)
90 try:
91 operations.remove('Operation')
92 except Exception as e:
93 pass
94 log.success('Current Operations are:\n\033[1m{}\033[0m'.format('\n'.join(operations)))
95 else:
96 try:
97 module = locate('schainpy.model.{}'.format(nextcommand))
98 args = module().getAllowedArgs()
99 log.warning('Use this feature with caution. It may not return all the allowed arguments')
100 try:
101 args.remove('self')
102 except Exception as e:
103 pass
104 try:
105 args.remove('dataOut')
106 except Exception as e:
107 pass
108 if len(args) == 0:
109 log.success('{} has no arguments'.format(nextcommand))
110 else:
111 log.success('Showing arguments of {} are:\n\033[1m{}\033[0m'.format(nextcommand, '\n'.join(args)))
112 except Exception as e:
113 log.error('Module {} does not exists'.format(nextcommand))
114 allModules = dir(import_module('schainpy.model'))
115 module = check_module(allModules, Operation)
116 module.extend(check_module(allModules, ProcessingUnit))
117 similar = process.extractOne(nextcommand, module)[0]
118 log.success('Searching {} instead'.format(similar))
119 search(similar)
120
121
122 def runschain(nextcommand):
123 if nextcommand is None:
124 currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
125 numberfiles = len(currentfiles)
126 if numberfiles > 1:
127 log.error('There is more than one file to run')
128 elif numberfiles == 1:
129 subprocess.call(['python ' + currentfiles[0]], shell=True)
130 else:
131 log.error('There is no file to run')
132 else:
133 try:
134 subprocess.call(['python ' + nextcommand], shell=True)
135 except Exception as e:
136 log.error("I cannot run the file. Does it exists?")
137
138
139 def basicInputs():
140 inputs = {}
141 inputs['desc'] = click.prompt('Enter a description', default="A schain project", type=str)
142 inputs['name'] = click.prompt('Name of the project', default="project", type=str)
143 inputs['path'] = click.prompt('Data path', default=os.getcwd(), type=click.Path(exists=True, resolve_path=True))
144 inputs['startDate'] = click.prompt('Start date', default='1970/01/01', type=str)
145 inputs['endDate'] = click.prompt('End date', default='2017/12/31', type=str)
146 inputs['startHour'] = click.prompt('Start hour', default='00:00:00', type=str)
147 inputs['endHour'] = click.prompt('End hour', default='23:59:59', type=str)
148 inputs['figpath'] = inputs['path'] + '/figs'
149 return inputs
150
151
152 def generate():
153 inputs = basicInputs()
154 inputs['multiprocess'] = click.confirm('Is this a multiprocess script?')
155 if inputs['multiprocess']:
156 inputs['nProcess'] = click.prompt('How many process?', default=cpu_count(), type=int)
157 current = templates.multiprocess.format(**inputs)
158 else:
159 current = templates.basic.format(**inputs)
160 scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
161 script = open(scriptname, 'w')
162 try:
163 script.write(current)
164 log.success('Script {} generated'.format(scriptname))
165 except Exception as e:
166 log.error('I cannot create the file. Do you have writing permissions?')
167
168
169 def test():
170 log.warning('testing')
171
172
173 def runFromXML(filename):
174 controller = controller_api.ControllerThread()
175 if not controller.readXml(filename):
176 return
177
178 plotterObj = controller.useExternalPlotter()
179
180 controller.start()
181 plotterObj.start()
182
183 cliLogger("Finishing all processes")
184
185 controller.join(5)
186
187 cliLogger("End of script")
188 return
Show file before | Show file after | Hide comments
@@ -0,0 +1,75
1 basic = '''from schainpy.controller import Project
2
3 desc = "{desc}"
4
5 controller = Project()
6 controller.setup(id='191', name="{name}", description=desc)
7
8 readUnitConf = controller.addReadUnit(datatype='VoltageReader',
9 path="{path}",
10 startDate="{startDate}",
11 endDate="{endDate}",
12 startTime="{startHour}",
13 endTime="{endHour}",
14 online=0,
15 verbose=1,
16 walk=1,
17 )
18
19 procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId())
20
21 opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other')
22 opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist')
23
24 opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other')
25 opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
26 opObj11.addParameter(name='showprofile', value='0', format='int')
27 opObj11.addParameter(name='xmin', value='0', format='int')
28 opObj11.addParameter(name='xmax', value='24', format='int')
29 opObj11.addParameter(name='figpath', value="{figpath}", format='str')
30 opObj11.addParameter(name='wr_period', value='5', format='int')
31 opObj11.addParameter(name='exp_code', value='22', format='int')
32
33
34 controller.start()
35 '''
36
37 multiprocess = '''from schainpy.controller import Project, multiSchain
38
39 desc = "{desc}"
40
41 def fiber(cursor, skip, q, day):
42 controller = Project()
43 controller.setup(id='191', name="{name}", description=desc)
44
45 readUnitConf = controller.addReadUnit(datatype='SpectraReader',
46 path="{path}",
47 startDate=day,
48 endDate=day,
49 startTime="{startHour}",
50 endTime="{endHour}",
51 online=0,
52 queue=q,
53 cursor=cursor,
54 skip=skip,
55 verbose=1,
56 walk=1,
57 )
58
59 procUnitConf1 = controller.addProcUnit(datatype='Spectra', inputId=readUnitConf.getId())
60
61 procUnitConf2 = controller.addProcUnit(datatype='ParametersProc', inputId=readUnitConf.getId())
62 opObj11 = procUnitConf2.addOperation(name='SpectralMoments', optype='other')
63
64 opObj12 = procUnitConf2.addOperation(name='PublishData', optype='other')
65 opObj12.addParameter(name='zeromq', value=1, format='int')
66 opObj12.addParameter(name='verbose', value=0, format='bool')
67
68 controller.start()
69
70
71 if __name__ == '__main__':
72 multiSchain(fiber, nProcess={nProcess}, startDate="{startDate}", endDate="{endDate}")
73
74
75 '''
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1
Show file before | Show file after | Hide comments
@@ -0,0 +1,29
1 import pytest
2 from click.testing import CliRunner
3 from schaincli import cli
4
5
6 @pytest.fixture
7 def runner():
8 return CliRunner()
9
10
11 def test_cli(runner):
12 result = runner.invoke(cli.main)
13 assert result.exit_code == 0
14 assert not result.exception
15 assert result.output.strip() == 'Hello, world.'
16
17
18 def test_cli_with_option(runner):
19 result = runner.invoke(cli.main, ['--as-cowboy'])
20 assert not result.exception
21 assert result.exit_code == 0
22 assert result.output.strip() == 'Howdy, world.'
23
24
25 def test_cli_with_arg(runner):
26 result = runner.invoke(cli.main, ['Jicamarca'])
27 assert result.exit_code == 0
28 assert not result.exception
29 assert result.output.strip() == 'Hello, Jicamarca.'
Show file before | Show file after | Hide comments
@@ -0,0 +1,34
1 from schainpy.controller import Project
2
3 desc = "A schain project"
4
5 controller = Project()
6 controller.setup(id='191', name="project", description=desc)
7
8 readUnitConf = controller.addReadUnit(datatype='VoltageReader',
9 path="/home/nanosat/schain/schainpy",
10 startDate="1970/01/01",
11 endDate="2017/12/31",
12 startTime="00:00:00",
13 endTime="23:59:59",
14 online=0,
15 verbose=1,
16 walk=1,
17 )
18
19 procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId())
20
21 opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other')
22 opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist')
23
24 opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other')
25 opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
26 opObj11.addParameter(name='showprofile', value='0', format='int')
27 opObj11.addParameter(name='xmin', value='0', format='int')
28 opObj11.addParameter(name='xmax', value='24', format='int')
29 opObj11.addParameter(name='figpath', value="/home/nanosat/schain/schainpy/figs", format='str')
30 opObj11.addParameter(name='wr_period', value='5', format='int')
31 opObj11.addParameter(name='exp_code', value='22', format='int')
32
33
34 controller.start()
Show file before | Show file after | Hide comments
@@ -0,0 +1,33
1 from schainpy.controller import Project
2
3 desc = "A schain project"
4
5 controller = Project()
6 controller.setup(id='191', name="project", description=desc)
7
8 readUnitConf = controller.addReadUnit(datatype='VoltageReader',
9 path="/home/nanosat/schain/schainpy/scripts",
10 startDate="1970/01/01",
11 endDate="2017/12/31",
12 startTime="00:00:00",
13 endTime="23:59:59",
14 online=0,
15 walk=1,
16 )
17
18 procUnitConf1 = controller.addProcUnit(datatype='VoltageProc', inputId=readUnitConf.getId())
19
20 opObj11 = procUnitConf1.addOperation(name='ProfileSelector', optype='other')
21 opObj11.addParameter(name='profileRangeList', value='120,183', format='intlist')
22
23 opObj11 = procUnitConf1.addOperation(name='RTIPlot', optype='other')
24 opObj11.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
25 opObj11.addParameter(name='showprofile', value='0', format='int')
26 opObj11.addParameter(name='xmin', value='0', format='int')
27 opObj11.addParameter(name='xmax', value='24', format='int')
28 opObj11.addParameter(name='figpath', value="/home/nanosat/schain/schainpy/scripts/figs", format='str')
29 opObj11.addParameter(name='wr_period', value='5', format='int')
30 opObj11.addParameter(name='exp_code', value='22', format='int')
31
32
33 controller.start()
Show file before | Show file after | Hide comments
1 NO CONTENT: new file 100644
Show file before | Show file after | Hide comments
@@ -0,0 +1,45
1 """.
2 SCHAINPY - LOG
3 Simple helper for log standarization
4 Usage:
5 from schainpy.utils import log
6 log.error('A kitten died beacuse of you')
7 log.warning('You are doing it wrong but what the heck, I'll allow it)
8 log.succes('YOU ROCK!')
9 To create your own logger inside your class do it like this:
10 from schainpy.utils import log
11 awesomeLogger = log.makelogger("never gonna", bg="red", fg="white")
12 awesomeLogger('give you up')
13 which will look like this:
14 [NEVER GONNA] - give you up
15 with color red as background and white as foreground.
16 """
17
18 import click
19
20
21 def warning(message):
22 click.echo(click.style('[WARNING] - ' + message, fg='yellow'))
23 pass
24
25
26 def error(message):
27 click.echo(click.style('[ERROR] - ' + message, fg='red'))
28 pass
29
30
31 def success(message):
32 click.echo(click.style(message, fg='green'))
33 pass
34
35
36 def log(message):
37 click.echo('[LOG] - ' + message)
38 pass
39
40
41 def makelogger(topic, bg='reset', fg='reset'):
42 def func(message):
43 click.echo(click.style('[{}] - '.format(topic.upper()) + message,
44 bg=bg, fg=fg))
45 return func
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 You should install "digital_rf_hdf5" module if you want to read USRP data
Show file before | Show file after | Hide comments
@@ -1,1321 +1,1321
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, Queue, cpu_count
13 13
14 14 import schainpy
15 15 import schainpy.admin
16 16
17 17 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 18 from xml.dom import minidom
19 19
20 20 from schainpy.model import *
21 21 from time import sleep
22 22
23 23 def prettify(elem):
24 24 """Return a pretty-printed XML string for the Element.
25 25 """
26 26 rough_string = tostring(elem, 'utf-8')
27 27 reparsed = minidom.parseString(rough_string)
28 28 return reparsed.toprettyxml(indent=" ")
29 29
30 30 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
31 31 skip = 0
32 32 cursor = 0
33 33 nFiles = None
34 34 processes = []
35 35 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
36 36 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
37 37 days = (dt2 - dt1).days
38 38
39 39 for day in range(days+1):
40 40 skip = 0
41 41 cursor = 0
42 42 q = Queue()
43 43 processes = []
44 44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
45 45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
46 46 firstProcess.start()
47 47 if by_day:
48 48 continue
49 49 nFiles = q.get()
50 50 firstProcess.terminate()
51 51 skip = int(math.ceil(nFiles/nProcess))
52 52 while True:
53 53 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
54 54 processes[cursor].start()
55 55 if nFiles < cursor*skip:
56 56 break
57 57 cursor += 1
58 58
59 59 def beforeExit(exctype, value, trace):
60 60 for process in processes:
61 61 process.terminate()
62 62 process.join()
63 63 print traceback.print_tb(trace)
64 64
65 65 sys.excepthook = beforeExit
66 66
67 67 for process in processes:
68 68 process.join()
69 69 process.terminate()
70 70 time.sleep(3)
71 71
72 72 class ParameterConf():
73 73
74 74 id = None
75 75 name = None
76 76 value = None
77 77 format = None
78 78
79 79 __formated_value = None
80 80
81 81 ELEMENTNAME = 'Parameter'
82 82
83 83 def __init__(self):
84 84
85 85 self.format = 'str'
86 86
87 87 def getElementName(self):
88 88
89 89 return self.ELEMENTNAME
90 90
91 91 def getValue(self):
92 92
93 93 value = self.value
94 94 format = self.format
95 95
96 96 if self.__formated_value != None:
97 97
98 98 return self.__formated_value
99 99
100 100 if format == 'obj':
101 101 return value
102 102
103 103 if format == 'str':
104 104 self.__formated_value = str(value)
105 105 return self.__formated_value
106 106
107 107 if value == '':
108 108 raise ValueError, "%s: This parameter value is empty" %self.name
109 109
110 110 if format == 'list':
111 111 strList = value.split(',')
112 112
113 113 self.__formated_value = strList
114 114
115 115 return self.__formated_value
116 116
117 117 if format == 'intlist':
118 118 """
119 119 Example:
120 120 value = (0,1,2)
121 121 """
122 122
123 123 new_value = ast.literal_eval(value)
124 124
125 125 if type(new_value) not in (tuple, list):
126 126 new_value = [int(new_value)]
127 127
128 128 self.__formated_value = new_value
129 129
130 130 return self.__formated_value
131 131
132 132 if format == 'floatlist':
133 133 """
134 134 Example:
135 135 value = (0.5, 1.4, 2.7)
136 136 """
137 137
138 138 new_value = ast.literal_eval(value)
139 139
140 140 if type(new_value) not in (tuple, list):
141 141 new_value = [float(new_value)]
142 142
143 143 self.__formated_value = new_value
144 144
145 145 return self.__formated_value
146 146
147 147 if format == 'date':
148 148 strList = value.split('/')
149 149 intList = [int(x) for x in strList]
150 150 date = datetime.date(intList[0], intList[1], intList[2])
151 151
152 152 self.__formated_value = date
153 153
154 154 return self.__formated_value
155 155
156 156 if format == 'time':
157 157 strList = value.split(':')
158 158 intList = [int(x) for x in strList]
159 159 time = datetime.time(intList[0], intList[1], intList[2])
160 160
161 161 self.__formated_value = time
162 162
163 163 return self.__formated_value
164 164
165 165 if format == 'pairslist':
166 166 """
167 167 Example:
168 168 value = (0,1),(1,2)
169 169 """
170 170
171 171 new_value = ast.literal_eval(value)
172 172
173 173 if type(new_value) not in (tuple, list):
174 174 raise ValueError, "%s has to be a tuple or list of pairs" %value
175 175
176 176 if type(new_value[0]) not in (tuple, list):
177 177 if len(new_value) != 2:
178 178 raise ValueError, "%s has to be a tuple or list of pairs" %value
179 179 new_value = [new_value]
180 180
181 181 for thisPair in new_value:
182 182 if len(thisPair) != 2:
183 183 raise ValueError, "%s has to be a tuple or list of pairs" %value
184 184
185 185 self.__formated_value = new_value
186 186
187 187 return self.__formated_value
188 188
189 189 if format == 'multilist':
190 190 """
191 191 Example:
192 192 value = (0,1,2),(3,4,5)
193 193 """
194 194 multiList = ast.literal_eval(value)
195 195
196 196 if type(multiList[0]) == int:
197 197 multiList = ast.literal_eval("(" + value + ")")
198 198
199 199 self.__formated_value = multiList
200 200
201 201 return self.__formated_value
202 202
203 203 if format == 'bool':
204 204 value = int(value)
205 205
206 206 if format == 'int':
207 207 value = float(value)
208 208
209 209 format_func = eval(format)
210 210
211 211 self.__formated_value = format_func(value)
212 212
213 213 return self.__formated_value
214 214
215 215 def updateId(self, new_id):
216 216
217 217 self.id = str(new_id)
218 218
219 219 def setup(self, id, name, value, format='str'):
220 220
221 221 self.id = str(id)
222 222 self.name = name
223 223 if format == 'obj':
224 224 self.value = value
225 225 else:
226 226 self.value = str(value)
227 227 self.format = str.lower(format)
228 228
229 229 self.getValue()
230 230
231 231 return 1
232 232
233 233 def update(self, name, value, format='str'):
234 234
235 235 self.name = name
236 236 self.value = str(value)
237 237 self.format = format
238 238
239 239 def makeXml(self, opElement):
240 240 if self.name not in ('queue',):
241 241 parmElement = SubElement(opElement, self.ELEMENTNAME)
242 242 parmElement.set('id', str(self.id))
243 243 parmElement.set('name', self.name)
244 244 parmElement.set('value', self.value)
245 245 parmElement.set('format', self.format)
246 246
247 247 def readXml(self, parmElement):
248 248
249 249 self.id = parmElement.get('id')
250 250 self.name = parmElement.get('name')
251 251 self.value = parmElement.get('value')
252 252 self.format = str.lower(parmElement.get('format'))
253 253
254 254 #Compatible with old signal chain version
255 255 if self.format == 'int' and self.name == 'idfigure':
256 256 self.name = 'id'
257 257
258 258 def printattr(self):
259 259
260 260 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
261 261
262 262 class OperationConf():
263 263
264 264 id = None
265 265 name = None
266 266 priority = None
267 267 type = None
268 268
269 269 parmConfObjList = []
270 270
271 271 ELEMENTNAME = 'Operation'
272 272
273 273 def __init__(self):
274 274
275 275 self.id = '0'
276 276 self.name = None
277 277 self.priority = None
278 278 self.type = 'self'
279 279
280 280
281 281 def __getNewId(self):
282 282
283 283 return int(self.id)*10 + len(self.parmConfObjList) + 1
284 284
285 285 def updateId(self, new_id):
286 286
287 287 self.id = str(new_id)
288 288
289 289 n = 1
290 290 for parmObj in self.parmConfObjList:
291 291
292 292 idParm = str(int(new_id)*10 + n)
293 293 parmObj.updateId(idParm)
294 294
295 295 n += 1
296 296
297 297 def getElementName(self):
298 298
299 299 return self.ELEMENTNAME
300 300
301 301 def getParameterObjList(self):
302 302
303 303 return self.parmConfObjList
304 304
305 305 def getParameterObj(self, parameterName):
306 306
307 307 for parmConfObj in self.parmConfObjList:
308 308
309 309 if parmConfObj.name != parameterName:
310 310 continue
311 311
312 312 return parmConfObj
313 313
314 314 return None
315 315
316 316 def getParameterObjfromValue(self, parameterValue):
317 317
318 318 for parmConfObj in self.parmConfObjList:
319 319
320 320 if parmConfObj.getValue() != parameterValue:
321 321 continue
322 322
323 323 return parmConfObj.getValue()
324 324
325 325 return None
326 326
327 327 def getParameterValue(self, parameterName):
328 328
329 329 parameterObj = self.getParameterObj(parameterName)
330 330
331 331 # if not parameterObj:
332 332 # return None
333 333
334 334 value = parameterObj.getValue()
335 335
336 336 return value
337 337
338 338
339 339 def getKwargs(self):
340 340
341 341 kwargs = {}
342 342
343 343 for parmConfObj in self.parmConfObjList:
344 344 if self.name == 'run' and parmConfObj.name == 'datatype':
345 345 continue
346 346
347 347 kwargs[parmConfObj.name] = parmConfObj.getValue()
348 348
349 349 return kwargs
350 350
351 351 def setup(self, id, name, priority, type):
352 352
353 353 self.id = str(id)
354 354 self.name = name
355 355 self.type = type
356 356 self.priority = priority
357 357
358 358 self.parmConfObjList = []
359 359
360 360 def removeParameters(self):
361 361
362 362 for obj in self.parmConfObjList:
363 363 del obj
364 364
365 365 self.parmConfObjList = []
366 366
367 367 def addParameter(self, name, value, format='str'):
368 368
369 369 id = self.__getNewId()
370 370
371 371 parmConfObj = ParameterConf()
372 372 if not parmConfObj.setup(id, name, value, format):
373 373 return None
374 374
375 375 self.parmConfObjList.append(parmConfObj)
376 376
377 377 return parmConfObj
378 378
379 379 def changeParameter(self, name, value, format='str'):
380 380
381 381 parmConfObj = self.getParameterObj(name)
382 382 parmConfObj.update(name, value, format)
383 383
384 384 return parmConfObj
385 385
386 386 def makeXml(self, procUnitElement):
387 387
388 388 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
389 389 opElement.set('id', str(self.id))
390 390 opElement.set('name', self.name)
391 391 opElement.set('type', self.type)
392 392 opElement.set('priority', str(self.priority))
393 393
394 394 for parmConfObj in self.parmConfObjList:
395 395 parmConfObj.makeXml(opElement)
396 396
397 397 def readXml(self, opElement):
398 398
399 399 self.id = opElement.get('id')
400 400 self.name = opElement.get('name')
401 401 self.type = opElement.get('type')
402 402 self.priority = opElement.get('priority')
403 403
404 404 #Compatible with old signal chain version
405 405 #Use of 'run' method instead 'init'
406 406 if self.type == 'self' and self.name == 'init':
407 407 self.name = 'run'
408 408
409 409 self.parmConfObjList = []
410 410
411 411 parmElementList = opElement.iter(ParameterConf().getElementName())
412 412
413 413 for parmElement in parmElementList:
414 414 parmConfObj = ParameterConf()
415 415 parmConfObj.readXml(parmElement)
416 416
417 417 #Compatible with old signal chain version
418 418 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
419 419 if self.type != 'self' and self.name == 'Plot':
420 420 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
421 421 self.name = parmConfObj.value
422 422 continue
423 423
424 424 self.parmConfObjList.append(parmConfObj)
425 425
426 426 def printattr(self):
427 427
428 428 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
429 429 self.id,
430 430 self.name,
431 431 self.type,
432 432 self.priority)
433 433
434 434 for parmConfObj in self.parmConfObjList:
435 435 parmConfObj.printattr()
436 436
437 437 def createObject(self, plotter_queue=None):
438 438
439 439
440 440 if self.type == 'self':
441 441 raise ValueError, "This operation type cannot be created"
442 442
443 443 if self.type == 'plotter':
444 444 #Plotter(plotter_name)
445 445 if not plotter_queue:
446 446 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
447 447
448 448 opObj = Plotter(self.name, plotter_queue)
449 449
450 450 if self.type == 'external' or self.type == 'other':
451 451
452 452 className = eval(self.name)
453 453 kwargs = self.getKwargs()
454 454
455 455 opObj = className(**kwargs)
456 456
457 457 return opObj
458 458
459 459
460 460 class ProcUnitConf():
461 461
462 462 id = None
463 463 name = None
464 464 datatype = None
465 465 inputId = None
466 466 parentId = None
467 467
468 468 opConfObjList = []
469 469
470 470 procUnitObj = None
471 471 opObjList = []
472 472
473 473 ELEMENTNAME = 'ProcUnit'
474 474
475 475 def __init__(self):
476 476
477 477 self.id = None
478 478 self.datatype = None
479 479 self.name = None
480 480 self.inputId = None
481 481
482 482 self.opConfObjList = []
483 483
484 484 self.procUnitObj = None
485 485 self.opObjDict = {}
486 486
487 487 def __getPriority(self):
488 488
489 489 return len(self.opConfObjList)+1
490 490
491 491 def __getNewId(self):
492 492
493 493 return int(self.id)*10 + len(self.opConfObjList) + 1
494 494
495 495 def getElementName(self):
496 496
497 497 return self.ELEMENTNAME
498 498
499 499 def getId(self):
500 500
501 501 return self.id
502 502
503 503 def updateId(self, new_id, parentId=parentId):
504 504
505 505
506 506 new_id = int(parentId)*10 + (int(self.id) % 10)
507 507 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
508 508
509 509 #If this proc unit has not inputs
510 510 if self.inputId == '0':
511 511 new_inputId = 0
512 512
513 513 n = 1
514 514 for opConfObj in self.opConfObjList:
515 515
516 516 idOp = str(int(new_id)*10 + n)
517 517 opConfObj.updateId(idOp)
518 518
519 519 n += 1
520 520
521 521 self.parentId = str(parentId)
522 522 self.id = str(new_id)
523 523 self.inputId = str(new_inputId)
524 524
525 525
526 526 def getInputId(self):
527 527
528 528 return self.inputId
529 529
530 530 def getOperationObjList(self):
531 531
532 532 return self.opConfObjList
533 533
534 534 def getOperationObj(self, name=None):
535 535
536 536 for opConfObj in self.opConfObjList:
537 537
538 538 if opConfObj.name != name:
539 539 continue
540 540
541 541 return opConfObj
542 542
543 543 return None
544 544
545 545 def getOpObjfromParamValue(self, value=None):
546 546
547 547 for opConfObj in self.opConfObjList:
548 548 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
549 549 continue
550 550 return opConfObj
551 551 return None
552 552
553 553 def getProcUnitObj(self):
554 554
555 555 return self.procUnitObj
556 556
557 557 def setup(self, id, name, datatype, inputId, parentId=None):
558 558
559 559 #Compatible with old signal chain version
560 560 if datatype==None and name==None:
561 561 raise ValueError, "datatype or name should be defined"
562 562
563 563 if name==None:
564 564 if 'Proc' in datatype:
565 565 name = datatype
566 566 else:
567 567 name = '%sProc' %(datatype)
568 568
569 569 if datatype==None:
570 570 datatype = name.replace('Proc','')
571 571
572 572 self.id = str(id)
573 573 self.name = name
574 574 self.datatype = datatype
575 575 self.inputId = inputId
576 576 self.parentId = parentId
577 577
578 578 self.opConfObjList = []
579 579
580 580 self.addOperation(name='run', optype='self')
581 581
582 582 def removeOperations(self):
583 583
584 584 for obj in self.opConfObjList:
585 585 del obj
586 586
587 587 self.opConfObjList = []
588 588 self.addOperation(name='run')
589 589
590 590 def addParameter(self, **kwargs):
591 591 '''
592 592 Add parameters to "run" operation
593 593 '''
594 594 opObj = self.opConfObjList[0]
595 595
596 596 opObj.addParameter(**kwargs)
597 597
598 598 return opObj
599 599
600 600 def addOperation(self, name, optype='self'):
601 601
602 602 id = self.__getNewId()
603 603 priority = self.__getPriority()
604 604
605 605 opConfObj = OperationConf()
606 606 opConfObj.setup(id, name=name, priority=priority, type=optype)
607 607
608 608 self.opConfObjList.append(opConfObj)
609 609
610 610 return opConfObj
611 611
612 612 def makeXml(self, projectElement):
613 613
614 614 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
615 615 procUnitElement.set('id', str(self.id))
616 616 procUnitElement.set('name', self.name)
617 617 procUnitElement.set('datatype', self.datatype)
618 618 procUnitElement.set('inputId', str(self.inputId))
619 619
620 620 for opConfObj in self.opConfObjList:
621 621 opConfObj.makeXml(procUnitElement)
622 622
623 623 def readXml(self, upElement):
624 624
625 625 self.id = upElement.get('id')
626 626 self.name = upElement.get('name')
627 627 self.datatype = upElement.get('datatype')
628 628 self.inputId = upElement.get('inputId')
629 629
630 630 if self.ELEMENTNAME == "ReadUnit":
631 631 self.datatype = self.datatype.replace("Reader", "")
632 632
633 633 if self.ELEMENTNAME == "ProcUnit":
634 634 self.datatype = self.datatype.replace("Proc", "")
635 635
636 636 if self.inputId == 'None':
637 637 self.inputId = '0'
638 638
639 639 self.opConfObjList = []
640 640
641 641 opElementList = upElement.iter(OperationConf().getElementName())
642 642
643 643 for opElement in opElementList:
644 644 opConfObj = OperationConf()
645 645 opConfObj.readXml(opElement)
646 646 self.opConfObjList.append(opConfObj)
647 647
648 648 def printattr(self):
649 649
650 650 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
651 651 self.id,
652 652 self.name,
653 653 self.datatype,
654 654 self.inputId)
655 655
656 656 for opConfObj in self.opConfObjList:
657 657 opConfObj.printattr()
658 658
659 659
660 660 def getKwargs(self):
661 661
662 662 opObj = self.opConfObjList[0]
663 663 kwargs = opObj.getKwargs()
664 664
665 665 return kwargs
666 666
667 667 def createObjects(self, plotter_queue=None):
668 668
669 669 className = eval(self.name)
670 670 kwargs = self.getKwargs()
671 671 procUnitObj = className(**kwargs)
672 672
673 673 for opConfObj in self.opConfObjList:
674 674
675 675 if opConfObj.type=='self' and self.name=='run':
676 676 continue
677 677 elif opConfObj.type=='self':
678 678 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
679 679 continue
680 680
681 681 opObj = opConfObj.createObject(plotter_queue)
682 682
683 683 self.opObjDict[opConfObj.id] = opObj
684 684
685 685 procUnitObj.addOperation(opObj, opConfObj.id)
686 686
687 687 self.procUnitObj = procUnitObj
688 688
689 689 return procUnitObj
690 690
691 691 def run(self):
692 692
693 693 is_ok = False
694 694
695 695 for opConfObj in self.opConfObjList:
696 696
697 697 kwargs = {}
698 698 for parmConfObj in opConfObj.getParameterObjList():
699 699 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
700 700 continue
701 701
702 702 kwargs[parmConfObj.name] = parmConfObj.getValue()
703 703
704 704 #ini = time.time()
705 705
706 706 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
707 707 sts = self.procUnitObj.call(opType = opConfObj.type,
708 708 opName = opConfObj.name,
709 709 opId = opConfObj.id,
710 710 )
711 711
712 712 # total_time = time.time() - ini
713 713 #
714 714 # if total_time > 0.002:
715 715 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
716 716
717 717 is_ok = is_ok or sts
718 718
719 719 return is_ok
720 720
721 721 def close(self):
722 722
723 723 for opConfObj in self.opConfObjList:
724 724 if opConfObj.type == 'self':
725 725 continue
726 726
727 727 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
728 728 opObj.close()
729 729
730 730 self.procUnitObj.close()
731 731
732 732 return
733 733
734 734 class ReadUnitConf(ProcUnitConf):
735 735
736 736 path = None
737 737 startDate = None
738 738 endDate = None
739 739 startTime = None
740 740 endTime = None
741 741
742 742 ELEMENTNAME = 'ReadUnit'
743 743
744 744 def __init__(self):
745 745
746 746 self.id = None
747 747 self.datatype = None
748 748 self.name = None
749 749 self.inputId = None
750 750
751 751 self.parentId = None
752 752
753 753 self.opConfObjList = []
754 754 self.opObjList = []
755 755
756 756 def getElementName(self):
757 757
758 758 return self.ELEMENTNAME
759 759
760 760 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
761 761
762 762 #Compatible with old signal chain version
763 763 if datatype==None and name==None:
764 764 raise ValueError, "datatype or name should be defined"
765 765
766 766 if name==None:
767 767 if 'Reader' in datatype:
768 768 name = datatype
769 769 else:
770 770 name = '%sReader' %(datatype)
771 771
772 772 if datatype==None:
773 773 datatype = name.replace('Reader','')
774 774
775 775 self.id = id
776 776 self.name = name
777 777 self.datatype = datatype
778 778
779 779 self.path = os.path.abspath(path)
780 780 self.startDate = startDate
781 781 self.endDate = endDate
782 782 self.startTime = startTime
783 783 self.endTime = endTime
784 784
785 785 self.inputId = '0'
786 786 self.parentId = parentId
787 787 self.queue = queue
788 788 self.addRunOperation(**kwargs)
789 789
790 790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
791 791
792 792 #Compatible with old signal chain version
793 793 if datatype==None and name==None:
794 794 raise ValueError, "datatype or name should be defined"
795 795
796 796 if name==None:
797 797 if 'Reader' in datatype:
798 798 name = datatype
799 799 else:
800 800 name = '%sReader' %(datatype)
801 801
802 802 if datatype==None:
803 803 datatype = name.replace('Reader','')
804 804
805 805 self.datatype = datatype
806 806 self.name = name
807 807 self.path = path
808 808 self.startDate = startDate
809 809 self.endDate = endDate
810 810 self.startTime = startTime
811 811 self.endTime = endTime
812 812
813 813 self.inputId = '0'
814 814 self.parentId = parentId
815 815
816 816 self.updateRunOperation(**kwargs)
817 817
818 818 def removeOperations(self):
819 819
820 820 for obj in self.opConfObjList:
821 821 del obj
822 822
823 823 self.opConfObjList = []
824 824
825 825 def addRunOperation(self, **kwargs):
826 826
827 827 opObj = self.addOperation(name = 'run', optype = 'self')
828 828
829 829 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
830 830 opObj.addParameter(name='path' , value=self.path, format='str')
831 831 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
832 832 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
833 833 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
834 834 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
835 835 opObj.addParameter(name='queue' , value=self.queue, format='obj')
836 836
837 837 for key, value in kwargs.items():
838 838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
839 839
840 840 return opObj
841 841
842 842 def updateRunOperation(self, **kwargs):
843 843
844 844 opObj = self.getOperationObj(name = 'run')
845 845 opObj.removeParameters()
846 846
847 847 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
848 848 opObj.addParameter(name='path' , value=self.path, format='str')
849 849 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
850 850 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
851 851 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
852 852 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
853 853
854 854 for key, value in kwargs.items():
855 855 opObj.addParameter(name=key, value=value, format=type(value).__name__)
856 856
857 857 return opObj
858 858
859 859 # def makeXml(self, projectElement):
860 860 #
861 861 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
862 862 # procUnitElement.set('id', str(self.id))
863 863 # procUnitElement.set('name', self.name)
864 864 # procUnitElement.set('datatype', self.datatype)
865 865 # procUnitElement.set('inputId', str(self.inputId))
866 866 #
867 867 # for opConfObj in self.opConfObjList:
868 868 # opConfObj.makeXml(procUnitElement)
869 869
870 870 def readXml(self, upElement):
871 871
872 872 self.id = upElement.get('id')
873 873 self.name = upElement.get('name')
874 874 self.datatype = upElement.get('datatype')
875 875 self.inputId = upElement.get('inputId')
876 876
877 877 if self.ELEMENTNAME == "ReadUnit":
878 878 self.datatype = self.datatype.replace("Reader", "")
879 879
880 880 if self.inputId == 'None':
881 881 self.inputId = '0'
882 882
883 883 self.opConfObjList = []
884 884
885 885 opElementList = upElement.iter(OperationConf().getElementName())
886 886
887 887 for opElement in opElementList:
888 888 opConfObj = OperationConf()
889 889 opConfObj.readXml(opElement)
890 890 self.opConfObjList.append(opConfObj)
891 891
892 892 if opConfObj.name == 'run':
893 893 self.path = opConfObj.getParameterValue('path')
894 894 self.startDate = opConfObj.getParameterValue('startDate')
895 895 self.endDate = opConfObj.getParameterValue('endDate')
896 896 self.startTime = opConfObj.getParameterValue('startTime')
897 897 self.endTime = opConfObj.getParameterValue('endTime')
898 898
899 899 class Project():
900 900
901 901 id = None
902 902 name = None
903 903 description = None
904 904 filename = None
905 905
906 906 procUnitConfObjDict = None
907 907
908 908 ELEMENTNAME = 'Project'
909 909
910 910 plotterQueue = None
911 911
912 912 def __init__(self, plotter_queue=None):
913 913
914 914 self.id = None
915 915 self.name = None
916 916 self.description = None
917 917
918 918 self.plotterQueue = plotter_queue
919 919
920 920 self.procUnitConfObjDict = {}
921 921
922 922 def __getNewId(self):
923 923
924 924 idList = self.procUnitConfObjDict.keys()
925 925
926 926 id = int(self.id)*10
927 927
928 928 while True:
929 929 id += 1
930 930
931 931 if str(id) in idList:
932 932 continue
933 933
934 934 break
935 935
936 936 return str(id)
937 937
938 938 def getElementName(self):
939 939
940 940 return self.ELEMENTNAME
941 941
942 942 def getId(self):
943 943
944 944 return self.id
945 945
946 946 def updateId(self, new_id):
947 947
948 948 self.id = str(new_id)
949 949
950 950 keyList = self.procUnitConfObjDict.keys()
951 951 keyList.sort()
952 952
953 953 n = 1
954 954 newProcUnitConfObjDict = {}
955 955
956 956 for procKey in keyList:
957 957
958 958 procUnitConfObj = self.procUnitConfObjDict[procKey]
959 959 idProcUnit = str(int(self.id)*10 + n)
960 960 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
961 961
962 962 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
963 963 n += 1
964 964
965 965 self.procUnitConfObjDict = newProcUnitConfObjDict
966 966
967 967 def setup(self, id, name, description):
968 968
969 969 self.id = str(id)
970 970 self.name = name
971 971 self.description = description
972 972
973 973 def update(self, name, description):
974 974
975 975 self.name = name
976 976 self.description = description
977 977
978 978 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
979 979
980 980 if id is None:
981 981 idReadUnit = self.__getNewId()
982 982 else:
983 983 idReadUnit = str(id)
984 984
985 985 readUnitConfObj = ReadUnitConf()
986 986 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
987 987
988 988 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
989 989
990 990 return readUnitConfObj
991 991
992 992 def addProcUnit(self, inputId='0', datatype=None, name=None):
993 993
994 994 idProcUnit = self.__getNewId()
995 995
996 996 procUnitConfObj = ProcUnitConf()
997 997 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
998 998
999 999 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1000 1000
1001 1001 return procUnitConfObj
1002 1002
1003 1003 def removeProcUnit(self, id):
1004 1004
1005 1005 if id in self.procUnitConfObjDict.keys():
1006 1006 self.procUnitConfObjDict.pop(id)
1007 1007
1008 1008 def getReadUnitId(self):
1009 1009
1010 1010 readUnitConfObj = self.getReadUnitObj()
1011 1011
1012 1012 return readUnitConfObj.id
1013 1013
1014 1014 def getReadUnitObj(self):
1015 1015
1016 1016 for obj in self.procUnitConfObjDict.values():
1017 1017 if obj.getElementName() == "ReadUnit":
1018 1018 return obj
1019 1019
1020 1020 return None
1021 1021
1022 1022 def getProcUnitObj(self, id=None, name=None):
1023 1023
1024 1024 if id != None:
1025 1025 return self.procUnitConfObjDict[id]
1026 1026
1027 1027 if name != None:
1028 1028 return self.getProcUnitObjByName(name)
1029 1029
1030 1030 return None
1031 1031
1032 1032 def getProcUnitObjByName(self, name):
1033 1033
1034 1034 for obj in self.procUnitConfObjDict.values():
1035 1035 if obj.name == name:
1036 1036 return obj
1037 1037
1038 1038 return None
1039 1039
1040 1040 def procUnitItems(self):
1041 1041
1042 1042 return self.procUnitConfObjDict.items()
1043 1043
1044 1044 def makeXml(self):
1045 1045
1046 1046 projectElement = Element('Project')
1047 1047 projectElement.set('id', str(self.id))
1048 1048 projectElement.set('name', self.name)
1049 1049 projectElement.set('description', self.description)
1050 1050
1051 1051 for procUnitConfObj in self.procUnitConfObjDict.values():
1052 1052 procUnitConfObj.makeXml(projectElement)
1053 1053
1054 1054 self.projectElement = projectElement
1055 1055
1056 1056 def writeXml(self, filename=None):
1057 1057
1058 1058 if filename == None:
1059 1059 if self.filename:
1060 1060 filename = self.filename
1061 1061 else:
1062 1062 filename = "schain.xml"
1063 1063
1064 1064 if not filename:
1065 1065 print "filename has not been defined. Use setFilename(filename) for do it."
1066 1066 return 0
1067 1067
1068 1068 abs_file = os.path.abspath(filename)
1069 1069
1070 1070 if not os.access(os.path.dirname(abs_file), os.W_OK):
1071 1071 print "No write permission on %s" %os.path.dirname(abs_file)
1072 1072 return 0
1073 1073
1074 1074 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1075 1075 print "File %s already exists and it could not be overwriten" %abs_file
1076 1076 return 0
1077 1077
1078 1078 self.makeXml()
1079 1079
1080 1080 ElementTree(self.projectElement).write(abs_file, method='xml')
1081 1081
1082 1082 self.filename = abs_file
1083 1083
1084 1084 return 1
1085 1085
1086 1086 def readXml(self, filename = None):
1087 1087
1088 1088 if not filename:
1089 1089 print "filename is not defined"
1090 1090 return 0
1091 1091
1092 1092 abs_file = os.path.abspath(filename)
1093 1093
1094 1094 if not os.path.isfile(abs_file):
1095 1095 print "%s file does not exist" %abs_file
1096 1096 return 0
1097 1097
1098 1098 self.projectElement = None
1099 1099 self.procUnitConfObjDict = {}
1100 1100
1101 1101 try:
1102 1102 self.projectElement = ElementTree().parse(abs_file)
1103 1103 except:
1104 1104 print "Error reading %s, verify file format" %filename
1105 1105 return 0
1106 1106
1107 1107 self.project = self.projectElement.tag
1108 1108
1109 1109 self.id = self.projectElement.get('id')
1110 1110 self.name = self.projectElement.get('name')
1111 1111 self.description = self.projectElement.get('description')
1112 1112
1113 1113 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1114 1114
1115 1115 for readUnitElement in readUnitElementList:
1116 1116 readUnitConfObj = ReadUnitConf()
1117 1117 readUnitConfObj.readXml(readUnitElement)
1118 1118
1119 1119 if readUnitConfObj.parentId == None:
1120 1120 readUnitConfObj.parentId = self.id
1121 1121
1122 1122 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1123 1123
1124 1124 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1125 1125
1126 1126 for procUnitElement in procUnitElementList:
1127 1127 procUnitConfObj = ProcUnitConf()
1128 1128 procUnitConfObj.readXml(procUnitElement)
1129 1129
1130 1130 if procUnitConfObj.parentId == None:
1131 1131 procUnitConfObj.parentId = self.id
1132 1132
1133 1133 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1134 1134
1135 1135 self.filename = abs_file
1136 1136
1137 1137 return 1
1138 1138
1139 1139 def printattr(self):
1140 1140
1141 1141 print "Project[%s]: name = %s, description = %s" %(self.id,
1142 1142 self.name,
1143 1143 self.description)
1144 1144
1145 1145 for procUnitConfObj in self.procUnitConfObjDict.values():
1146 1146 procUnitConfObj.printattr()
1147 1147
1148 1148 def createObjects(self):
1149 1149
1150 1150 for procUnitConfObj in self.procUnitConfObjDict.values():
1151 1151 procUnitConfObj.createObjects(self.plotterQueue)
1152 1152
1153 1153 def __connect(self, objIN, thisObj):
1154 1154
1155 1155 thisObj.setInput(objIN.getOutputObj())
1156 1156
1157 1157 def connectObjects(self):
1158 1158
1159 1159 for thisPUConfObj in self.procUnitConfObjDict.values():
1160 1160
1161 1161 inputId = thisPUConfObj.getInputId()
1162 1162
1163 1163 if int(inputId) == 0:
1164 1164 continue
1165 1165
1166 1166 #Get input object
1167 1167 puConfINObj = self.procUnitConfObjDict[inputId]
1168 1168 puObjIN = puConfINObj.getProcUnitObj()
1169 1169
1170 1170 #Get current object
1171 1171 thisPUObj = thisPUConfObj.getProcUnitObj()
1172 1172
1173 1173 self.__connect(puObjIN, thisPUObj)
1174 1174
1175 1175 def __handleError(self, procUnitConfObj, send_email=True):
1176 1176
1177 1177 import socket
1178 1178
1179 1179 err = traceback.format_exception(sys.exc_info()[0],
1180 1180 sys.exc_info()[1],
1181 1181 sys.exc_info()[2])
1182 1182
1183 1183 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1184 1184 print "***** %s" %err[-1]
1185 1185
1186 1186 message = "".join(err)
1187 1187
1188 1188 sys.stderr.write(message)
1189 1189
1190 1190 if not send_email:
1191 1191 return
1192 1192
1193 1193 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1194 1194
1195 1195 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1196 1196 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1197 1197 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1198 1198 subtitle += "Configuration file: %s\n" %self.filename
1199 1199 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1200 1200
1201 1201 readUnitConfObj = self.getReadUnitObj()
1202 1202 if readUnitConfObj:
1203 1203 subtitle += "\nInput parameters:\n"
1204 1204 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1205 1205 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1206 1206 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1207 1207 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1208 1208 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1209 1209 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1210 1210
1211 1211 adminObj = schainpy.admin.SchainNotify()
1212 1212 adminObj.sendAlert(message=message,
1213 1213 subject=subject,
1214 1214 subtitle=subtitle,
1215 1215 filename=self.filename)
1216 1216
1217 1217 def isPaused(self):
1218 1218 return 0
1219 1219
1220 1220 def isStopped(self):
1221 1221 return 0
1222 1222
1223 1223 def runController(self):
1224 1224 """
1225 1225 returns 0 when this process has been stopped, 1 otherwise
1226 1226 """
1227 1227
1228 1228 if self.isPaused():
1229 1229 print "Process suspended"
1230 1230
1231 1231 while True:
1232 1232 sleep(0.1)
1233 1233
1234 1234 if not self.isPaused():
1235 1235 break
1236 1236
1237 1237 if self.isStopped():
1238 1238 break
1239 1239
1240 1240 print "Process reinitialized"
1241 1241
1242 1242 if self.isStopped():
1243 1243 print "Process stopped"
1244 1244 return 0
1245 1245
1246 1246 return 1
1247 1247
1248 1248 def setFilename(self, filename):
1249 1249
1250 1250 self.filename = filename
1251 1251
1252 1252 def setPlotterQueue(self, plotter_queue):
1253 1253
1254 1254 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1255 1255
1256 1256 def getPlotterQueue(self):
1257 1257
1258 1258 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1259 1259
1260 1260 def useExternalPlotter(self):
1261 1261
1262 1262 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1263 1263
1264 1264 def run(self):
1265 1265
1266 1266 print
1267 1267 print "*"*60
1268 1268 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1269 1269 print "*"*60
1270 1270 print
1271 1271
1272 1272 keyList = self.procUnitConfObjDict.keys()
1273 1273 keyList.sort()
1274 1274
1275 1275 while(True):
1276 1276
1277 1277 is_ok = False
1278 1278
1279 1279 for procKey in keyList:
1280 1280 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1281 1281
1282 1282 procUnitConfObj = self.procUnitConfObjDict[procKey]
1283 1283
1284 1284 try:
1285 1285 sts = procUnitConfObj.run()
1286 1286 is_ok = is_ok or sts
1287 1287 except KeyboardInterrupt:
1288 1288 is_ok = False
1289 1289 break
1290 1290 except ValueError, e:
1291 1291 sleep(0.5)
1292 1292 self.__handleError(procUnitConfObj, send_email=True)
1293 1293 is_ok = False
1294 1294 break
1295 1295 except:
1296 1296 sleep(0.5)
1297 1297 self.__handleError(procUnitConfObj)
1298 1298 is_ok = False
1299 1299 break
1300 1300
1301 1301 #If every process unit finished so end process
1302 1302 if not(is_ok):
1303 1303 # print "Every process unit have finished"
1304 1304 break
1305 1305
1306 1306 if not self.runController():
1307 1307 break
1308 1308
1309 1309 #Closing every process
1310 1310 for procKey in keyList:
1311 1311 procUnitConfObj = self.procUnitConfObjDict[procKey]
1312 1312 procUnitConfObj.close()
1313 1313
1314 1314 print "Process finished"
1315 1315
1316 def start(self):
1316 def start(self, filename=None):
1317 1317
1318 self.writeXml()
1318 self.writeXml(filename)
1319 1319 self.createObjects()
1320 1320 self.connectObjects()
1321 1321 self.run()
Show file before | Show file after | Hide comments
@@ -1,179 +1,179
1 1 import threading
2 2 from Queue import Queue
3 3
4 4 from schainpy.controller import Project
5 5 from schainpy.model.graphics.jroplotter import PlotManager
6 6
7 7 class ControllerThread(threading.Thread, Project):
8
8
9 9 def __init__(self, plotter_queue=None):
10
10
11 11 threading.Thread.__init__(self)
12 12 Project.__init__(self, plotter_queue)
13
13
14 14 self.setDaemon(True)
15
15
16 16 self.lock = threading.Lock()
17 17 self.control = {'stop':False, 'pause':False}
18
18
19 19 def __del__(self):
20
20
21 21 self.control['stop'] = True
22
22
23 23 def stop(self):
24
24
25 25 self.lock.acquire()
26
26
27 27 self.control['stop'] = True
28
28
29 29 self.lock.release()
30
30
31 31 def pause(self):
32
32
33 33 self.lock.acquire()
34
34
35 35 self.control['pause'] = not(self.control['pause'])
36 36 paused = self.control['pause']
37
37
38 38 self.lock.release()
39
39
40 40 return paused
41
41
42 42 def isPaused(self):
43
43
44 44 self.lock.acquire()
45 45 paused = self.control['pause']
46 46 self.lock.release()
47
47
48 48 return paused
49
49
50 50 def isStopped(self):
51
51
52 52 self.lock.acquire()
53 53 stopped = self.control['stop']
54 54 self.lock.release()
55
55
56 56 return stopped
57
57
58 58 def run(self):
59 59 self.control['stop'] = False
60 60 self.control['pause'] = False
61
61
62 62 self.writeXml()
63
63
64 64 self.createObjects()
65 65 self.connectObjects()
66 66 Project.run(self)
67
67
68 68 def isRunning(self):
69
69
70 70 return self.is_alive()
71
71
72 72 def isFinished(self):
73
73
74 74 return not self.is_alive()
75 75
76 76 def setPlotters(self):
77
77
78 78 plotterList = PlotManager.plotterList
79
79
80 80 for thisPUConfObj in self.procUnitConfObjDict.values():
81
81
82 82 inputId = thisPUConfObj.getInputId()
83
83
84 84 if int(inputId) == 0:
85 85 continue
86
86
87 87 for thisOpObj in thisPUConfObj.getOperationObjList():
88
88
89 89 if thisOpObj.type == "self":
90 90 continue
91
91
92 92 if thisOpObj.name in plotterList:
93 93 thisOpObj.type = "plotter"
94 94
95 95 def setPlotterQueue(self, plotter_queue):
96
96
97 97 self.plotterQueue = plotter_queue
98
98
99 99 def getPlotterQueue(self):
100
100
101 101 return self.plotterQueue
102 102
103 103 def useExternalPlotter(self):
104
104
105 105 self.plotterQueue = Queue(10)
106 106 self.setPlotters()
107
107
108 108 plotManagerObj = PlotManager(self.plotterQueue)
109 109 plotManagerObj.setController(self)
110
110
111 111 return plotManagerObj
112
112
113 113 # from PyQt4 import QtCore
114 114 # from PyQt4.QtCore import SIGNAL
115 #
115 #
116 116 # class ControllerQThread(QtCore.QThread, Project):
117 #
117 #
118 118 # def __init__(self, filename):
119 #
119 #
120 120 # QtCore.QThread.__init__(self)
121 121 # Project.__init__(self)
122 #
122 #
123 123 # self.filename = filename
124 #
124 #
125 125 # self.lock = threading.Lock()
126 126 # self.control = {'stop':False, 'pause':False}
127 #
127 #
128 128 # def __del__(self):
129 #
129 #
130 130 # self.control['stop'] = True
131 131 # self.wait()
132 #
132 #
133 133 # def stop(self):
134 #
134 #
135 135 # self.lock.acquire()
136 #
136 #
137 137 # self.control['stop'] = True
138 #
138 #
139 139 # self.lock.release()
140 #
140 #
141 141 # def pause(self):
142 #
142 #
143 143 # self.lock.acquire()
144 #
144 #
145 145 # self.control['pause'] = not(self.control['pause'])
146 146 # paused = self.control['pause']
147 #
147 #
148 148 # self.lock.release()
149 #
149 #
150 150 # return paused
151 #
151 #
152 152 # def isPaused(self):
153 #
153 #
154 154 # self.lock.acquire()
155 155 # paused = self.control['pause']
156 156 # self.lock.release()
157 #
157 #
158 158 # return paused
159 #
159 #
160 160 # def isStopped(self):
161 #
161 #
162 162 # self.lock.acquire()
163 163 # stopped = self.control['stop']
164 164 # self.lock.release()
165 #
165 #
166 166 # return stopped
167 #
167 #
168 168 # def run(self):
169 #
169 #
170 170 # self.control['stop'] = False
171 171 # self.control['pause'] = False
172 #
172 #
173 173 # self.readXml(self.filename)
174 174 # self.createObjects()
175 175 # self.connectObjects()
176 176 # self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
177 177 # Project.run(self)
178 178 # self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
179 # No newline at end of file
179 #
Show file before | Show file after | Hide comments
@@ -1,1750 +1,1794
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 import inspect
12 13 import time, datetime
13 14 #import h5py
14 15 import traceback
15 16
16 17 try:
17 18 from gevent import sleep
18 19 except:
19 20 from time import sleep
20 21
21 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 24
24 25 LOCALTIME = True
25 26
26 27 def isNumber(cad):
27 28 """
28 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 30
30 31 Excepciones:
31 32 Si un determinado string no puede ser convertido a numero
32 33 Input:
33 34 str, string al cual se le analiza para determinar si convertible a un numero o no
34 35
35 36 Return:
36 37 True : si el string es uno numerico
37 38 False : no es un string numerico
38 39 """
39 40 try:
40 41 float( cad )
41 42 return True
42 43 except:
43 44 return False
44 45
45 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 47 """
47 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 49
49 50 Inputs:
50 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 52
52 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 54 segundos contados desde 01/01/1970.
54 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 56 segundos contados desde 01/01/1970.
56 57
57 58 Return:
58 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 60 fecha especificado, de lo contrario retorna False.
60 61
61 62 Excepciones:
62 63 Si el archivo no existe o no puede ser abierto
63 64 Si la cabecera no puede ser leida.
64 65
65 66 """
66 67 basicHeaderObj = BasicHeader(LOCALTIME)
67 68
68 69 try:
69 70 fp = open(filename,'rb')
70 71 except IOError:
71 72 print "The file %s can't be opened" %(filename)
72 73 return 0
73 74
74 75 sts = basicHeaderObj.read(fp)
75 76 fp.close()
76 77
77 78 if not(sts):
78 79 print "Skipping the file %s because it has not a valid header" %(filename)
79 80 return 0
80 81
81 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 83 return 0
83 84
84 85 return 1
85 86
86 87 def isTimeInRange(thisTime, startTime, endTime):
87 88
88 89 if endTime >= startTime:
89 90 if (thisTime < startTime) or (thisTime > endTime):
90 91 return 0
91 92
92 93 return 1
93 94 else:
94 95 if (thisTime < startTime) and (thisTime > endTime):
95 96 return 0
96 97
97 98 return 1
98 99
99 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 101 """
101 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 103
103 104 Inputs:
104 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 106
106 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 108
108 109 endDate : fecha final del rango seleccionado en formato datetime.date
109 110
110 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 112
112 113 endTime : tiempo final del rango seleccionado en formato datetime.time
113 114
114 115 Return:
115 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 117 fecha especificado, de lo contrario retorna False.
117 118
118 119 Excepciones:
119 120 Si el archivo no existe o no puede ser abierto
120 121 Si la cabecera no puede ser leida.
121 122
122 123 """
123 124
124 125
125 126 try:
126 127 fp = open(filename,'rb')
127 128 except IOError:
128 129 print "The file %s can't be opened" %(filename)
129 130 return None
130 131
131 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 133 systemHeaderObj = SystemHeader()
133 134 radarControllerHeaderObj = RadarControllerHeader()
134 135 processingHeaderObj = ProcessingHeader()
135 136
136 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 138
138 139 sts = firstBasicHeaderObj.read(fp)
139 140
140 141 if not(sts):
141 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 143 return None
143 144
144 145 if not systemHeaderObj.read(fp):
145 146 return None
146 147
147 148 if not radarControllerHeaderObj.read(fp):
148 149 return None
149 150
150 151 if not processingHeaderObj.read(fp):
151 152 return None
152 153
153 154 filesize = os.path.getsize(filename)
154 155
155 156 offset = processingHeaderObj.blockSize + 24 #header size
156 157
157 158 if filesize <= offset:
158 159 print "[Reading] %s: This file has not enough data" %filename
159 160 return None
160 161
161 162 fp.seek(-offset, 2)
162 163
163 164 sts = lastBasicHeaderObj.read(fp)
164 165
165 166 fp.close()
166 167
167 168 thisDatetime = lastBasicHeaderObj.datatime
168 169 thisTime_last_block = thisDatetime.time()
169 170
170 171 thisDatetime = firstBasicHeaderObj.datatime
171 172 thisDate = thisDatetime.date()
172 173 thisTime_first_block = thisDatetime.time()
173 174
174 175 #General case
175 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 177 #-----------o----------------------------o-----------
177 178 # startTime endTime
178 179
179 180 if endTime >= startTime:
180 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 182 return None
182 183
183 184 return thisDatetime
184 185
185 186 #If endTime < startTime then endTime belongs to the next day
186 187
187 188
188 189 #<<<<<<<<<<<o o>>>>>>>>>>>
189 190 #-----------o----------------------------o-----------
190 191 # endTime startTime
191 192
192 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 194 return None
194 195
195 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 197 return None
197 198
198 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 200 return None
200 201
201 202 return thisDatetime
202 203
203 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 205 """
205 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 207
207 208 Inputs:
208 209 folder : nombre completo del directorio.
209 210 Su formato deberia ser "/path_root/?YYYYDDD"
210 211
211 212 siendo:
212 213 YYYY : Anio (ejemplo 2015)
213 214 DDD : Dia del anio (ejemplo 305)
214 215
215 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 217
217 218 endDate : fecha final del rango seleccionado en formato datetime.date
218 219
219 220 Return:
220 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 222 fecha especificado, de lo contrario retorna False.
222 223 Excepciones:
223 224 Si el directorio no tiene el formato adecuado
224 225 """
225 226
226 227 basename = os.path.basename(folder)
227 228
228 229 if not isRadarFolder(basename):
229 230 print "The folder %s has not the rigth format" %folder
230 231 return 0
231 232
232 233 if startDate and endDate:
233 234 thisDate = getDateFromRadarFolder(basename)
234 235
235 236 if thisDate < startDate:
236 237 return 0
237 238
238 239 if thisDate > endDate:
239 240 return 0
240 241
241 242 return 1
242 243
243 244 def isFileInDateRange(filename, startDate=None, endDate=None):
244 245 """
245 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 247
247 248 Inputs:
248 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 250
250 251 Su formato deberia ser "?YYYYDDDsss"
251 252
252 253 siendo:
253 254 YYYY : Anio (ejemplo 2015)
254 255 DDD : Dia del anio (ejemplo 305)
255 256 sss : set
256 257
257 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 259
259 260 endDate : fecha final del rango seleccionado en formato datetime.date
260 261
261 262 Return:
262 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 264 fecha especificado, de lo contrario retorna False.
264 265 Excepciones:
265 266 Si el archivo no tiene el formato adecuado
266 267 """
267 268
268 269 basename = os.path.basename(filename)
269 270
270 271 if not isRadarFile(basename):
271 272 print "The filename %s has not the rigth format" %filename
272 273 return 0
273 274
274 275 if startDate and endDate:
275 276 thisDate = getDateFromRadarFile(basename)
276 277
277 278 if thisDate < startDate:
278 279 return 0
279 280
280 281 if thisDate > endDate:
281 282 return 0
282 283
283 284 return 1
284 285
285 286 def getFileFromSet(path, ext, set):
286 287 validFilelist = []
287 288 fileList = os.listdir(path)
288 289
289 290 # 0 1234 567 89A BCDE
290 291 # H YYYY DDD SSS .ext
291 292
292 293 for thisFile in fileList:
293 294 try:
294 295 year = int(thisFile[1:5])
295 296 doy = int(thisFile[5:8])
296 297 except:
297 298 continue
298 299
299 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 301 continue
301 302
302 303 validFilelist.append(thisFile)
303 304
304 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 306
306 307 if len(myfile)!= 0:
307 308 return myfile[0]
308 309 else:
309 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 311 print 'the filename %s does not exist'%filename
311 312 print '...going to the last file: '
312 313
313 314 if validFilelist:
314 315 validFilelist = sorted( validFilelist, key=str.lower )
315 316 return validFilelist[-1]
316 317
317 318 return None
318 319
319 320 def getlastFileFromPath(path, ext):
320 321 """
321 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 324
324 325 Input:
325 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 327 ext : extension de los files contenidos en una carpeta
327 328
328 329 Return:
329 330 El ultimo file de una determinada carpeta, no se considera el path.
330 331 """
331 332 validFilelist = []
332 333 fileList = os.listdir(path)
333 334
334 335 # 0 1234 567 89A BCDE
335 336 # H YYYY DDD SSS .ext
336 337
337 338 for thisFile in fileList:
338 339
339 340 year = thisFile[1:5]
340 341 if not isNumber(year):
341 342 continue
342 343
343 344 doy = thisFile[5:8]
344 345 if not isNumber(doy):
345 346 continue
346 347
347 348 year = int(year)
348 349 doy = int(doy)
349 350
350 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 352 continue
352 353
353 354 validFilelist.append(thisFile)
354 355
355 356 if validFilelist:
356 357 validFilelist = sorted( validFilelist, key=str.lower )
357 358 return validFilelist[-1]
358 359
359 360 return None
360 361
361 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 363 """
363 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 366 el path exacto de un determinado file.
366 367
367 368 Example :
368 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 370
370 371 Entonces la funcion prueba con las siguientes combinaciones
371 372 .../.../y2009307367.ext
372 373 .../.../Y2009307367.ext
373 374 .../.../x2009307/y2009307367.ext
374 375 .../.../x2009307/Y2009307367.ext
375 376 .../.../X2009307/y2009307367.ext
376 377 .../.../X2009307/Y2009307367.ext
377 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 379
379 380 Return:
380 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 383 para el filename
383 384 """
384 385 fullfilename = None
385 386 find_flag = False
386 387 filename = None
387 388
388 389 prefixDirList = [None,'d','D']
389 390 if ext.lower() == ".r": #voltage
390 391 prefixFileList = ['d','D']
391 392 elif ext.lower() == ".pdata": #spectra
392 393 prefixFileList = ['p','P']
393 394 else:
394 395 return None, filename
395 396
396 397 #barrido por las combinaciones posibles
397 398 for prefixDir in prefixDirList:
398 399 thispath = path
399 400 if prefixDir != None:
400 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 402 if foldercounter == 0:
402 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 404 else:
404 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 409
409 410 if os.path.exists( fullfilename ): #verifico que exista
410 411 find_flag = True
411 412 break
412 413 if find_flag:
413 414 break
414 415
415 416 if not(find_flag):
416 417 return None, filename
417 418
418 419 return fullfilename, filename
419 420
420 421 def isRadarFolder(folder):
421 422 try:
422 423 year = int(folder[1:5])
423 424 doy = int(folder[5:8])
424 425 except:
425 426 return 0
426 427
427 428 return 1
428 429
429 430 def isRadarFile(file):
430 431 try:
431 432 year = int(file[1:5])
432 433 doy = int(file[5:8])
433 434 set = int(file[8:11])
434 435 except:
435 436 return 0
436 437
437 438 return 1
438 439
439 440 def getDateFromRadarFile(file):
440 441 try:
441 442 year = int(file[1:5])
442 443 doy = int(file[5:8])
443 444 set = int(file[8:11])
444 445 except:
445 446 return None
446 447
447 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 449 return thisDate
449 450
450 451 def getDateFromRadarFolder(folder):
451 452 try:
452 453 year = int(folder[1:5])
453 454 doy = int(folder[5:8])
454 455 except:
455 456 return None
456 457
457 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 459 return thisDate
459 460
460 461 class JRODataIO:
461 462
462 463 c = 3E8
463 464
464 465 isConfig = False
465 466
466 467 basicHeaderObj = None
467 468
468 469 systemHeaderObj = None
469 470
470 471 radarControllerHeaderObj = None
471 472
472 473 processingHeaderObj = None
473 474
474 475 dtype = None
475 476
476 477 pathList = []
477 478
478 479 filenameList = []
479 480
480 481 filename = None
481 482
482 483 ext = None
483 484
484 485 flagIsNewFile = 1
485 486
486 487 flagDiscontinuousBlock = 0
487 488
488 489 flagIsNewBlock = 0
489 490
490 491 fp = None
491 492
492 493 firstHeaderSize = 0
493 494
494 495 basicHeaderSize = 24
495 496
496 497 versionFile = 1103
497 498
498 499 fileSize = None
499 500
500 501 # ippSeconds = None
501 502
502 503 fileSizeByHeader = None
503 504
504 505 fileIndex = None
505 506
506 507 profileIndex = None
507 508
508 509 blockIndex = None
509 510
510 511 nTotalBlocks = None
511 512
512 513 maxTimeStep = 30
513 514
514 515 lastUTTime = None
515 516
516 517 datablock = None
517 518
518 519 dataOut = None
519 520
520 521 blocksize = None
521 522
522 523 getByBlock = False
523 524
524 525 def __init__(self):
525 526
526 527 raise NotImplementedError
527 528
528 529 def run(self):
529 530
530 531 raise NotImplementedError
531 532
532 533 def getDtypeWidth(self):
533 534
534 535 dtype_index = get_dtype_index(self.dtype)
535 536 dtype_width = get_dtype_width(dtype_index)
536 537
537 538 return dtype_width
538 539
540 def getAllowedArgs(self):
541 return inspect.getargspec(self.run).args
542
539 543 class JRODataReader(JRODataIO):
540 544
541 545
542 546 online = 0
543 547
544 548 realtime = 0
545 549
546 550 nReadBlocks = 0
547 551
548 552 delay = 10 #number of seconds waiting a new file
549 553
550 554 nTries = 3 #quantity tries
551 555
552 556 nFiles = 3 #number of files for searching
553 557
554 558 path = None
555 559
556 560 foldercounter = 0
557 561
558 562 flagNoMoreFiles = 0
559 563
560 564 datetimeList = []
561 565
562 566 __isFirstTimeOnline = 1
563 567
564 568 __printInfo = True
565 569
566 570 profileIndex = None
567 571
568 572 nTxs = 1
569 573
570 574 txIndex = None
571 575
572 576 #Added--------------------
573 577
574 578 selBlocksize = None
575 579
576 580 selBlocktime = None
577 581
578 582
579 583 def __init__(self):
580 584
581 585 """
582 586 This class is used to find data files
583 587
584 588 Example:
585 589 reader = JRODataReader()
586 590 fileList = reader.findDataFiles()
587 591
588 592 """
589 593 pass
590 594
591 595
592 596 def createObjByDefault(self):
593 597 """
594 598
595 599 """
596 600 raise NotImplementedError
597 601
598 602 def getBlockDimension(self):
599 603
600 604 raise NotImplementedError
601 605
602 606 def __searchFilesOffLine(self,
603 607 path,
604 608 startDate=None,
605 609 endDate=None,
606 610 startTime=datetime.time(0,0,0),
607 611 endTime=datetime.time(23,59,59),
608 612 set=None,
609 613 expLabel='',
610 614 ext='.r',
611 615 queue=None,
612 616 cursor=None,
613 617 skip=None,
614 618 walk=True):
615 619
616 620 self.filenameList = []
617 621 self.datetimeList = []
618 622
619 623 pathList = []
620 624
621 625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
622 626
623 627 if dateList == []:
624 628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
625 629 return None, None
626 630
627 631 if len(dateList) > 1:
628 632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
629 633 else:
630 634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
631 635
632 636 filenameList = []
633 637 datetimeList = []
634 638
635 639 for thisPath in pathList:
636 640 # thisPath = pathList[pathDict[file]]
637 641
638 642 fileList = glob.glob1(thisPath, "*%s" %ext)
639 643 fileList.sort()
640 644
641 645 skippedFileList = []
642 646
643 647 if cursor is not None and skip is not None:
644 648 # if cursor*skip > len(fileList):
645 649 if skip == 0:
646 650 if queue is not None:
647 651 queue.put(len(fileList))
648 652 skippedFileList = []
649 653 else:
650 654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
651 655
652 656 else:
653 657 skippedFileList = fileList
654 658
655 659 for file in skippedFileList:
656 660
657 661 filename = os.path.join(thisPath,file)
658 662
659 663 if not isFileInDateRange(filename, startDate, endDate):
660 664 continue
661 665
662 666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
663 667
664 668 if not(thisDatetime):
665 669 continue
666 670
667 671 filenameList.append(filename)
668 672 datetimeList.append(thisDatetime)
669 673
670 674 if not(filenameList):
671 675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
672 676 return None, None
673 677
674 678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
675 679 print
676 680
677 681 for i in range(len(filenameList)):
678 682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
679 683
680 684 self.filenameList = filenameList
681 685 self.datetimeList = datetimeList
682 686
683 687 return pathList, filenameList
684 688
685 689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
686 690
687 691 """
688 692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
689 693 devuelve el archivo encontrado ademas de otros datos.
690 694
691 695 Input:
692 696 path : carpeta donde estan contenidos los files que contiene data
693 697
694 698 expLabel : Nombre del subexperimento (subfolder)
695 699
696 700 ext : extension de los files
697 701
698 702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
699 703
700 704 Return:
701 705 directory : eL directorio donde esta el file encontrado
702 706 filename : el ultimo file de una determinada carpeta
703 707 year : el anho
704 708 doy : el numero de dia del anho
705 709 set : el set del archivo
706 710
707 711
708 712 """
709 713 if not os.path.isdir(path):
710 714 return None, None, None, None, None, None
711 715
712 716 dirList = []
713 717
714 718 if not walk:
715 719 fullpath = path
716 720 foldercounter = 0
717 721 else:
718 722 #Filtra solo los directorios
719 723 for thisPath in os.listdir(path):
720 724 if not os.path.isdir(os.path.join(path,thisPath)):
721 725 continue
722 726 if not isRadarFolder(thisPath):
723 727 continue
724 728
725 729 dirList.append(thisPath)
726 730
727 731 if not(dirList):
728 732 return None, None, None, None, None, None
729 733
730 734 dirList = sorted( dirList, key=str.lower )
731 735
732 736 doypath = dirList[-1]
733 737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
734 738 fullpath = os.path.join(path, doypath, expLabel)
735 739
736 740
737 741 print "[Reading] %s folder was found: " %(fullpath )
738 742
739 743 if set == None:
740 744 filename = getlastFileFromPath(fullpath, ext)
741 745 else:
742 746 filename = getFileFromSet(fullpath, ext, set)
743 747
744 748 if not(filename):
745 749 return None, None, None, None, None, None
746 750
747 751 print "[Reading] %s file was found" %(filename)
748 752
749 753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
750 754 return None, None, None, None, None, None
751 755
752 756 year = int( filename[1:5] )
753 757 doy = int( filename[5:8] )
754 758 set = int( filename[8:11] )
755 759
756 760 return fullpath, foldercounter, filename, year, doy, set
757 761
758 762 def __setNextFileOffline(self):
759 763
760 764 idFile = self.fileIndex
761 765
762 766 while (True):
763 767 idFile += 1
764 768 if not(idFile < len(self.filenameList)):
765 769 self.flagNoMoreFiles = 1
766 770 # print "[Reading] No more Files"
767 771 return 0
768 772
769 773 filename = self.filenameList[idFile]
770 774
771 775 if not(self.__verifyFile(filename)):
772 776 continue
773 777
774 778 fileSize = os.path.getsize(filename)
775 779 fp = open(filename,'rb')
776 780 break
777 781
778 782 self.flagIsNewFile = 1
779 783 self.fileIndex = idFile
780 784 self.filename = filename
781 785 self.fileSize = fileSize
782 786 self.fp = fp
783 787
784 788 # print "[Reading] Setting the file: %s"%self.filename
785 789
786 790 return 1
787 791
788 792 def __setNextFileOnline(self):
789 793 """
790 794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
791 795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
792 796 siguientes.
793 797
794 798 Affected:
795 799 self.flagIsNewFile
796 800 self.filename
797 801 self.fileSize
798 802 self.fp
799 803 self.set
800 804 self.flagNoMoreFiles
801 805
802 806 Return:
803 807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
804 808 1 : si el file fue abierto con exito y esta listo a ser leido
805 809
806 810 Excepciones:
807 811 Si un determinado file no puede ser abierto
808 812 """
809 813 nFiles = 0
810 814 fileOk_flag = False
811 815 firstTime_flag = True
812 816
813 817 self.set += 1
814 818
815 819 if self.set > 999:
816 820 self.set = 0
817 821 self.foldercounter += 1
818 822
819 823 #busca el 1er file disponible
820 824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
821 825 if fullfilename:
822 826 if self.__verifyFile(fullfilename, False):
823 827 fileOk_flag = True
824 828
825 829 #si no encuentra un file entonces espera y vuelve a buscar
826 830 if not(fileOk_flag):
827 831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
828 832
829 833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
830 834 tries = self.nTries
831 835 else:
832 836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
833 837
834 838 for nTries in range( tries ):
835 839 if firstTime_flag:
836 840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
837 841 sleep( self.delay )
838 842 else:
839 843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
840 844
841 845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
842 846 if fullfilename:
843 847 if self.__verifyFile(fullfilename):
844 848 fileOk_flag = True
845 849 break
846 850
847 851 if fileOk_flag:
848 852 break
849 853
850 854 firstTime_flag = False
851 855
852 856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
853 857 self.set += 1
854 858
855 859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
856 860 self.set = 0
857 861 self.doy += 1
858 862 self.foldercounter = 0
859 863
860 864 if fileOk_flag:
861 865 self.fileSize = os.path.getsize( fullfilename )
862 866 self.filename = fullfilename
863 867 self.flagIsNewFile = 1
864 868 if self.fp != None: self.fp.close()
865 869 self.fp = open(fullfilename, 'rb')
866 870 self.flagNoMoreFiles = 0
867 871 # print '[Reading] Setting the file: %s' % fullfilename
868 872 else:
869 873 self.fileSize = 0
870 874 self.filename = None
871 875 self.flagIsNewFile = 0
872 876 self.fp = None
873 877 self.flagNoMoreFiles = 1
874 878 # print '[Reading] No more files to read'
875 879
876 880 return fileOk_flag
877 881
878 882 def setNextFile(self):
879 883 if self.fp != None:
880 884 self.fp.close()
881 885
882 886 if self.online:
883 887 newFile = self.__setNextFileOnline()
884 888 else:
885 889 newFile = self.__setNextFileOffline()
886 890
887 891 if not(newFile):
888 892 print '[Reading] No more files to read'
889 893 return 0
890 894
891 895 if self.verbose:
892 896 print '[Reading] Setting the file: %s' % self.filename
893 897
894 898 self.__readFirstHeader()
895 899 self.nReadBlocks = 0
896 900 return 1
897 901
898 902 def __waitNewBlock(self):
899 903 """
900 904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
901 905
902 906 Si el modo de lectura es OffLine siempre retorn 0
903 907 """
904 908 if not self.online:
905 909 return 0
906 910
907 911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
908 912 return 0
909 913
910 914 currentPointer = self.fp.tell()
911 915
912 916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
913 917
914 918 for nTries in range( self.nTries ):
915 919
916 920 self.fp.close()
917 921 self.fp = open( self.filename, 'rb' )
918 922 self.fp.seek( currentPointer )
919 923
920 924 self.fileSize = os.path.getsize( self.filename )
921 925 currentSize = self.fileSize - currentPointer
922 926
923 927 if ( currentSize >= neededSize ):
924 928 self.basicHeaderObj.read(self.fp)
925 929 return 1
926 930
927 931 if self.fileSize == self.fileSizeByHeader:
928 932 # self.flagEoF = True
929 933 return 0
930 934
931 935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
932 936 sleep( self.delay )
933 937
934 938
935 939 return 0
936 940
937 941 def waitDataBlock(self,pointer_location):
938 942
939 943 currentPointer = pointer_location
940 944
941 945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
942 946
943 947 for nTries in range( self.nTries ):
944 948 self.fp.close()
945 949 self.fp = open( self.filename, 'rb' )
946 950 self.fp.seek( currentPointer )
947 951
948 952 self.fileSize = os.path.getsize( self.filename )
949 953 currentSize = self.fileSize - currentPointer
950 954
951 955 if ( currentSize >= neededSize ):
952 956 return 1
953 957
954 958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
955 959 sleep( self.delay )
956 960
957 961 return 0
958 962
959 963 def __jumpToLastBlock(self):
960 964
961 965 if not(self.__isFirstTimeOnline):
962 966 return
963 967
964 968 csize = self.fileSize - self.fp.tell()
965 969 blocksize = self.processingHeaderObj.blockSize
966 970
967 971 #salta el primer bloque de datos
968 972 if csize > self.processingHeaderObj.blockSize:
969 973 self.fp.seek(self.fp.tell() + blocksize)
970 974 else:
971 975 return
972 976
973 977 csize = self.fileSize - self.fp.tell()
974 978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
975 979 while True:
976 980
977 981 if self.fp.tell()<self.fileSize:
978 982 self.fp.seek(self.fp.tell() + neededsize)
979 983 else:
980 984 self.fp.seek(self.fp.tell() - neededsize)
981 985 break
982 986
983 987 # csize = self.fileSize - self.fp.tell()
984 988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
985 989 # factor = int(csize/neededsize)
986 990 # if factor > 0:
987 991 # self.fp.seek(self.fp.tell() + factor*neededsize)
988 992
989 993 self.flagIsNewFile = 0
990 994 self.__isFirstTimeOnline = 0
991 995
992 996 def __setNewBlock(self):
993 997
994 998 if self.fp == None:
995 999 return 0
996 1000
997 1001 # if self.online:
998 1002 # self.__jumpToLastBlock()
999 1003
1000 1004 if self.flagIsNewFile:
1001 1005 self.lastUTTime = self.basicHeaderObj.utc
1002 1006 return 1
1003 1007
1004 1008 if self.realtime:
1005 1009 self.flagDiscontinuousBlock = 1
1006 1010 if not(self.setNextFile()):
1007 1011 return 0
1008 1012 else:
1009 1013 return 1
1010 1014
1011 1015 currentSize = self.fileSize - self.fp.tell()
1012 1016 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1013 1017
1014 1018 if (currentSize >= neededSize):
1015 1019 self.basicHeaderObj.read(self.fp)
1016 1020 self.lastUTTime = self.basicHeaderObj.utc
1017 1021 return 1
1018 1022
1019 1023 if self.__waitNewBlock():
1020 1024 self.lastUTTime = self.basicHeaderObj.utc
1021 1025 return 1
1022 1026
1023 1027 if not(self.setNextFile()):
1024 1028 return 0
1025 1029
1026 1030 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1027 1031 self.lastUTTime = self.basicHeaderObj.utc
1028 1032
1029 1033 self.flagDiscontinuousBlock = 0
1030 1034
1031 1035 if deltaTime > self.maxTimeStep:
1032 1036 self.flagDiscontinuousBlock = 1
1033 1037
1034 1038 return 1
1035 1039
1036 1040 def readNextBlock(self):
1037 1041
1038 1042 #Skip block out of startTime and endTime
1039 1043 while True:
1040 1044 if not(self.__setNewBlock()):
1041 1045 return 0
1042 1046
1043 1047 if not(self.readBlock()):
1044 1048 return 0
1045 1049
1046 1050 self.getBasicHeader()
1047 1051
1048 1052 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1049 1053
1050 1054 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1051 1055 self.processingHeaderObj.dataBlocksPerFile,
1052 1056 self.dataOut.datatime.ctime())
1053 1057 continue
1054 1058
1055 1059 break
1056 1060
1057 1061 if self.verbose:
1058 1062 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1059 1063 self.processingHeaderObj.dataBlocksPerFile,
1060 1064 self.dataOut.datatime.ctime())
1061 1065 return 1
1062 1066
1063 1067 def __readFirstHeader(self):
1064 1068
1065 1069 self.basicHeaderObj.read(self.fp)
1066 1070 self.systemHeaderObj.read(self.fp)
1067 1071 self.radarControllerHeaderObj.read(self.fp)
1068 1072 self.processingHeaderObj.read(self.fp)
1069 1073
1070 1074 self.firstHeaderSize = self.basicHeaderObj.size
1071 1075
1072 1076 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1073 1077 if datatype == 0:
1074 1078 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1075 1079 elif datatype == 1:
1076 1080 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1077 1081 elif datatype == 2:
1078 1082 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1079 1083 elif datatype == 3:
1080 1084 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1081 1085 elif datatype == 4:
1082 1086 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1083 1087 elif datatype == 5:
1084 1088 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1085 1089 else:
1086 1090 raise ValueError, 'Data type was not defined'
1087 1091
1088 1092 self.dtype = datatype_str
1089 1093 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1090 1094 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1091 1095 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1092 1096 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1093 1097 self.getBlockDimension()
1094 1098
1095 1099 def __verifyFile(self, filename, msgFlag=True):
1096 1100
1097 1101 msg = None
1098 1102
1099 1103 try:
1100 1104 fp = open(filename, 'rb')
1101 1105 except IOError:
1102 1106
1103 1107 if msgFlag:
1104 1108 print "[Reading] File %s can't be opened" % (filename)
1105 1109
1106 1110 return False
1107 1111
1108 1112 currentPosition = fp.tell()
1109 1113 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1110 1114
1111 1115 if neededSize == 0:
1112 1116 basicHeaderObj = BasicHeader(LOCALTIME)
1113 1117 systemHeaderObj = SystemHeader()
1114 1118 radarControllerHeaderObj = RadarControllerHeader()
1115 1119 processingHeaderObj = ProcessingHeader()
1116 1120
1117 1121 if not( basicHeaderObj.read(fp) ):
1118 1122 fp.close()
1119 1123 return False
1120 1124
1121 1125 if not( systemHeaderObj.read(fp) ):
1122 1126 fp.close()
1123 1127 return False
1124 1128
1125 1129 if not( radarControllerHeaderObj.read(fp) ):
1126 1130 fp.close()
1127 1131 return False
1128 1132
1129 1133 if not( processingHeaderObj.read(fp) ):
1130 1134 fp.close()
1131 1135 return False
1132 1136
1133 1137 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1134 1138 else:
1135 1139 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1136 1140
1137 1141 fp.close()
1138 1142
1139 1143 fileSize = os.path.getsize(filename)
1140 1144 currentSize = fileSize - currentPosition
1141 1145
1142 1146 if currentSize < neededSize:
1143 1147 if msgFlag and (msg != None):
1144 1148 print msg
1145 1149 return False
1146 1150
1147 1151 return True
1148 1152
1149 1153 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1150 1154
1151 1155 path_empty = True
1152 1156
1153 1157 dateList = []
1154 1158 pathList = []
1155 1159
1156 1160 multi_path = path.split(',')
1157 1161
1158 1162 if not walk:
1159 1163
1160 1164 for single_path in multi_path:
1161 1165
1162 1166 if not os.path.isdir(single_path):
1163 1167 continue
1164 1168
1165 1169 fileList = glob.glob1(single_path, "*"+ext)
1166 1170
1167 1171 if not fileList:
1168 1172 continue
1169 1173
1170 1174 path_empty = False
1171 1175
1172 1176 fileList.sort()
1173 1177
1174 1178 for thisFile in fileList:
1175 1179
1176 1180 if not os.path.isfile(os.path.join(single_path, thisFile)):
1177 1181 continue
1178 1182
1179 1183 if not isRadarFile(thisFile):
1180 1184 continue
1181 1185
1182 1186 if not isFileInDateRange(thisFile, startDate, endDate):
1183 1187 continue
1184 1188
1185 1189 thisDate = getDateFromRadarFile(thisFile)
1186 1190
1187 1191 if thisDate in dateList:
1188 1192 continue
1189 1193
1190 1194 dateList.append(thisDate)
1191 1195 pathList.append(single_path)
1192 1196
1193 1197 else:
1194 1198 for single_path in multi_path:
1195 1199
1196 1200 if not os.path.isdir(single_path):
1197 1201 continue
1198 1202
1199 1203 dirList = []
1200 1204
1201 1205 for thisPath in os.listdir(single_path):
1202 1206
1203 1207 if not os.path.isdir(os.path.join(single_path,thisPath)):
1204 1208 continue
1205 1209
1206 1210 if not isRadarFolder(thisPath):
1207 1211 continue
1208 1212
1209 1213 if not isFolderInDateRange(thisPath, startDate, endDate):
1210 1214 continue
1211 1215
1212 1216 dirList.append(thisPath)
1213 1217
1214 1218 if not dirList:
1215 1219 continue
1216 1220
1217 1221 dirList.sort()
1218 1222
1219 1223 for thisDir in dirList:
1220 1224
1221 1225 datapath = os.path.join(single_path, thisDir, expLabel)
1222 1226 fileList = glob.glob1(datapath, "*"+ext)
1223 1227
1224 1228 if not fileList:
1225 1229 continue
1226 1230
1227 1231 path_empty = False
1228 1232
1229 1233 thisDate = getDateFromRadarFolder(thisDir)
1230 1234
1231 1235 pathList.append(datapath)
1232 1236 dateList.append(thisDate)
1233 1237
1234 1238 dateList.sort()
1235 1239
1236 1240 if walk:
1237 1241 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1238 1242 else:
1239 1243 pattern_path = multi_path[0]
1240 1244
1241 1245 if path_empty:
1242 1246 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1243 1247 else:
1244 1248 if not dateList:
1245 1249 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1246 1250
1247 1251 if include_path:
1248 1252 return dateList, pathList
1249 1253
1250 1254 return dateList
1251 1255
1252 1256 def setup(self,
1253 1257 path=None,
1254 1258 startDate=None,
1255 1259 endDate=None,
1256 1260 startTime=datetime.time(0,0,0),
1257 1261 endTime=datetime.time(23,59,59),
1258 1262 set=None,
1259 1263 expLabel = "",
1260 1264 ext = None,
1261 1265 online = False,
1262 1266 delay = 60,
1263 1267 walk = True,
1264 1268 getblock = False,
1265 1269 nTxs = 1,
1266 1270 realtime=False,
1267 1271 blocksize=None,
1268 1272 blocktime=None,
1269 1273 queue=None,
1270 1274 skip=None,
1271 1275 cursor=None,
1272 1276 warnings=True,
1273 1277 verbose=True):
1274 1278
1275 1279 if path == None:
1276 1280 raise ValueError, "[Reading] The path is not valid"
1277 1281
1278 1282 if ext == None:
1279 1283 ext = self.ext
1280 1284
1281 1285 if online:
1282 1286 print "[Reading] Searching files in online mode..."
1283 1287
1284 1288 for nTries in range( self.nTries ):
1285 1289 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1286 1290
1287 1291 if fullpath:
1288 1292 break
1289 1293
1290 1294 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1291 1295 sleep( self.delay )
1292 1296
1293 1297 if not(fullpath):
1294 1298 print "[Reading] There 'isn't any valid file in %s" % path
1295 1299 return
1296 1300
1297 1301 self.year = year
1298 1302 self.doy = doy
1299 1303 self.set = set - 1
1300 1304 self.path = path
1301 1305 self.foldercounter = foldercounter
1302 1306 last_set = None
1303 1307
1304 1308 else:
1305 1309 print "[Reading] Searching files in offline mode ..."
1306 1310 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1307 1311 startTime=startTime, endTime=endTime,
1308 1312 set=set, expLabel=expLabel, ext=ext,
1309 1313 walk=walk, cursor=cursor,
1310 1314 skip=skip, queue=queue)
1311 1315
1312 1316 if not(pathList):
1313 1317 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1314 1318 # datetime.datetime.combine(startDate,startTime).ctime(),
1315 1319 # datetime.datetime.combine(endDate,endTime).ctime())
1316 1320
1317 1321 # sys.exit(-1)
1318 1322
1319 1323 self.fileIndex = -1
1320 1324 self.pathList = []
1321 1325 self.filenameList = []
1322 1326 return
1323 1327
1324 1328 self.fileIndex = -1
1325 1329 self.pathList = pathList
1326 1330 self.filenameList = filenameList
1327 1331 file_name = os.path.basename(filenameList[-1])
1328 1332 basename, ext = os.path.splitext(file_name)
1329 1333 last_set = int(basename[-3:])
1330 1334
1331 1335 self.online = online
1332 1336 self.realtime = realtime
1333 1337 self.delay = delay
1334 1338 ext = ext.lower()
1335 1339 self.ext = ext
1336 1340 self.getByBlock = getblock
1337 1341 self.nTxs = nTxs
1338 1342 self.startTime = startTime
1339 1343 self.endTime = endTime
1340 1344
1341 1345 #Added-----------------
1342 1346 self.selBlocksize = blocksize
1343 1347 self.selBlocktime = blocktime
1344 1348
1345 1349 # Verbose-----------
1346 1350 self.verbose = verbose
1347 1351 self.warnings = warnings
1348 1352
1349 1353 if not(self.setNextFile()):
1350 1354 if (startDate!=None) and (endDate!=None):
1351 1355 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1352 1356 elif startDate != None:
1353 1357 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1354 1358 else:
1355 1359 print "[Reading] No files"
1356 1360
1357 1361 self.fileIndex = -1
1358 1362 self.pathList = []
1359 1363 self.filenameList = []
1360 1364 return
1361 1365
1362 1366 # self.getBasicHeader()
1363 1367
1364 1368 if last_set != None:
1365 1369 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1366 1370 return
1367 1371
1368 1372 def getBasicHeader(self):
1369 1373
1370 1374 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1371 1375
1372 1376 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1373 1377
1374 1378 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1375 1379
1376 1380 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1377 1381
1378 1382 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1379 1383
1380 1384 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1381 1385
1382 1386 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1383 1387
1384 1388 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1385 1389
1386 1390
1387 1391 def getFirstHeader(self):
1388 1392
1389 1393 raise NotImplementedError
1390 1394
1391 1395 def getData(self):
1392 1396
1393 1397 raise NotImplementedError
1394 1398
1395 1399 def hasNotDataInBuffer(self):
1396 1400
1397 1401 raise NotImplementedError
1398 1402
1399 1403 def readBlock(self):
1400 1404
1401 1405 raise NotImplementedError
1402 1406
1403 1407 def isEndProcess(self):
1404 1408
1405 1409 return self.flagNoMoreFiles
1406 1410
1407 1411 def printReadBlocks(self):
1408 1412
1409 1413 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1410 1414
1411 1415 def printTotalBlocks(self):
1412 1416
1413 1417 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1414 1418
1415 1419 def printNumberOfBlock(self):
1416 1420
1417 1421 if self.flagIsNewBlock:
1418 1422 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1419 1423 self.processingHeaderObj.dataBlocksPerFile,
1420 1424 self.dataOut.datatime.ctime())
1421 1425
1422 1426 def printInfo(self):
1423 1427
1424 1428 if self.__printInfo == False:
1425 1429 return
1426 1430
1427 1431 self.basicHeaderObj.printInfo()
1428 1432 self.systemHeaderObj.printInfo()
1429 1433 self.radarControllerHeaderObj.printInfo()
1430 1434 self.processingHeaderObj.printInfo()
1431 1435
1432 1436 self.__printInfo = False
1433 1437
1434 1438
1435 def run(self, **kwargs):
1439 def run(self,
1440 path=None,
1441 startDate=None,
1442 endDate=None,
1443 startTime=datetime.time(0,0,0),
1444 endTime=datetime.time(23,59,59),
1445 set=None,
1446 expLabel = "",
1447 ext = None,
1448 online = False,
1449 delay = 60,
1450 walk = True,
1451 getblock = False,
1452 nTxs = 1,
1453 realtime=False,
1454 blocksize=None,
1455 blocktime=None,
1456 queue=None,
1457 skip=None,
1458 cursor=None,
1459 warnings=True,
1460 verbose=True, **kwargs):
1436 1461
1437 1462 if not(self.isConfig):
1438
1439 1463 # self.dataOut = dataOut
1440 self.setup(**kwargs)
1464 self.setup( path=path,
1465 startDate=startDate,
1466 endDate=endDate,
1467 startTime=startTime,
1468 endTime=endTime,
1469 set=set,
1470 expLabel=expLabel,
1471 ext=ext,
1472 online=online,
1473 delay=delay,
1474 walk=walk,
1475 getblock=getblock,
1476 nTxs=nTxs,
1477 realtime=realtime,
1478 blocksize=blocksize,
1479 blocktime=blocktime,
1480 queue=queue,
1481 skip=skip,
1482 cursor=cursor,
1483 warnings=warnings,
1484 verbose=verbose)
1441 1485 self.isConfig = True
1442 1486
1443 1487 self.getData()
1444 1488
1445 1489 class JRODataWriter(JRODataIO):
1446 1490
1447 1491 """
1448 1492 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1449 1493 de los datos siempre se realiza por bloques.
1450 1494 """
1451 1495
1452 1496 blockIndex = 0
1453 1497
1454 1498 path = None
1455 1499
1456 1500 setFile = None
1457 1501
1458 1502 profilesPerBlock = None
1459 1503
1460 1504 blocksPerFile = None
1461 1505
1462 1506 nWriteBlocks = 0
1463 1507
1464 1508 fileDate = None
1465 1509
1466 1510 def __init__(self, dataOut=None):
1467 1511 raise NotImplementedError
1468 1512
1469 1513
1470 1514 def hasAllDataInBuffer(self):
1471 1515 raise NotImplementedError
1472 1516
1473 1517
1474 1518 def setBlockDimension(self):
1475 1519 raise NotImplementedError
1476 1520
1477 1521
1478 1522 def writeBlock(self):
1479 1523 raise NotImplementedError
1480 1524
1481 1525
1482 1526 def putData(self):
1483 1527 raise NotImplementedError
1484 1528
1485 1529
1486 1530 def getProcessFlags(self):
1487 1531
1488 1532 processFlags = 0
1489 1533
1490 1534 dtype_index = get_dtype_index(self.dtype)
1491 1535 procflag_dtype = get_procflag_dtype(dtype_index)
1492 1536
1493 1537 processFlags += procflag_dtype
1494 1538
1495 1539 if self.dataOut.flagDecodeData:
1496 1540 processFlags += PROCFLAG.DECODE_DATA
1497 1541
1498 1542 if self.dataOut.flagDeflipData:
1499 1543 processFlags += PROCFLAG.DEFLIP_DATA
1500 1544
1501 1545 if self.dataOut.code is not None:
1502 1546 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1503 1547
1504 1548 if self.dataOut.nCohInt > 1:
1505 1549 processFlags += PROCFLAG.COHERENT_INTEGRATION
1506 1550
1507 1551 if self.dataOut.type == "Spectra":
1508 1552 if self.dataOut.nIncohInt > 1:
1509 1553 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1510 1554
1511 1555 if self.dataOut.data_dc is not None:
1512 1556 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1513 1557
1514 1558 if self.dataOut.flagShiftFFT:
1515 1559 processFlags += PROCFLAG.SHIFT_FFT_DATA
1516 1560
1517 1561 return processFlags
1518 1562
1519 1563 def setBasicHeader(self):
1520 1564
1521 1565 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1522 1566 self.basicHeaderObj.version = self.versionFile
1523 1567 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1524 1568
1525 1569 utc = numpy.floor(self.dataOut.utctime)
1526 1570 milisecond = (self.dataOut.utctime - utc)* 1000.0
1527 1571
1528 1572 self.basicHeaderObj.utc = utc
1529 1573 self.basicHeaderObj.miliSecond = milisecond
1530 1574 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1531 1575 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1532 1576 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1533 1577
1534 1578 def setFirstHeader(self):
1535 1579 """
1536 1580 Obtiene una copia del First Header
1537 1581
1538 1582 Affected:
1539 1583
1540 1584 self.basicHeaderObj
1541 1585 self.systemHeaderObj
1542 1586 self.radarControllerHeaderObj
1543 1587 self.processingHeaderObj self.
1544 1588
1545 1589 Return:
1546 1590 None
1547 1591 """
1548 1592
1549 1593 raise NotImplementedError
1550 1594
1551 1595 def __writeFirstHeader(self):
1552 1596 """
1553 1597 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1554 1598
1555 1599 Affected:
1556 1600 __dataType
1557 1601
1558 1602 Return:
1559 1603 None
1560 1604 """
1561 1605
1562 1606 # CALCULAR PARAMETROS
1563 1607
1564 1608 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1565 1609 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1566 1610
1567 1611 self.basicHeaderObj.write(self.fp)
1568 1612 self.systemHeaderObj.write(self.fp)
1569 1613 self.radarControllerHeaderObj.write(self.fp)
1570 1614 self.processingHeaderObj.write(self.fp)
1571 1615
1572 1616 def __setNewBlock(self):
1573 1617 """
1574 1618 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1575 1619
1576 1620 Return:
1577 1621 0 : si no pudo escribir nada
1578 1622 1 : Si escribio el Basic el First Header
1579 1623 """
1580 1624 if self.fp == None:
1581 1625 self.setNextFile()
1582 1626
1583 1627 if self.flagIsNewFile:
1584 1628 return 1
1585 1629
1586 1630 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1587 1631 self.basicHeaderObj.write(self.fp)
1588 1632 return 1
1589 1633
1590 1634 if not( self.setNextFile() ):
1591 1635 return 0
1592 1636
1593 1637 return 1
1594 1638
1595 1639
1596 1640 def writeNextBlock(self):
1597 1641 """
1598 1642 Selecciona el bloque siguiente de datos y los escribe en un file
1599 1643
1600 1644 Return:
1601 1645 0 : Si no hizo pudo escribir el bloque de datos
1602 1646 1 : Si no pudo escribir el bloque de datos
1603 1647 """
1604 1648 if not( self.__setNewBlock() ):
1605 1649 return 0
1606 1650
1607 1651 self.writeBlock()
1608 1652
1609 1653 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1610 1654 self.processingHeaderObj.dataBlocksPerFile)
1611 1655
1612 1656 return 1
1613 1657
1614 1658 def setNextFile(self):
1615 1659 """
1616 1660 Determina el siguiente file que sera escrito
1617 1661
1618 1662 Affected:
1619 1663 self.filename
1620 1664 self.subfolder
1621 1665 self.fp
1622 1666 self.setFile
1623 1667 self.flagIsNewFile
1624 1668
1625 1669 Return:
1626 1670 0 : Si el archivo no puede ser escrito
1627 1671 1 : Si el archivo esta listo para ser escrito
1628 1672 """
1629 1673 ext = self.ext
1630 1674 path = self.path
1631 1675
1632 1676 if self.fp != None:
1633 1677 self.fp.close()
1634 1678
1635 1679 timeTuple = time.localtime( self.dataOut.utctime)
1636 1680 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1637 1681
1638 1682 fullpath = os.path.join( path, subfolder )
1639 1683 setFile = self.setFile
1640 1684
1641 1685 if not( os.path.exists(fullpath) ):
1642 1686 os.mkdir(fullpath)
1643 1687 setFile = -1 #inicializo mi contador de seteo
1644 1688 else:
1645 1689 filesList = os.listdir( fullpath )
1646 1690 if len( filesList ) > 0:
1647 1691 filesList = sorted( filesList, key=str.lower )
1648 1692 filen = filesList[-1]
1649 1693 # el filename debera tener el siguiente formato
1650 1694 # 0 1234 567 89A BCDE (hex)
1651 1695 # x YYYY DDD SSS .ext
1652 1696 if isNumber( filen[8:11] ):
1653 1697 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1654 1698 else:
1655 1699 setFile = -1
1656 1700 else:
1657 1701 setFile = -1 #inicializo mi contador de seteo
1658 1702
1659 1703 setFile += 1
1660 1704
1661 1705 #If this is a new day it resets some values
1662 1706 if self.dataOut.datatime.date() > self.fileDate:
1663 1707 setFile = 0
1664 1708 self.nTotalBlocks = 0
1665 1709
1666 1710 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1667 1711
1668 1712 filename = os.path.join( path, subfolder, filen )
1669 1713
1670 1714 fp = open( filename,'wb' )
1671 1715
1672 1716 self.blockIndex = 0
1673 1717
1674 1718 #guardando atributos
1675 1719 self.filename = filename
1676 1720 self.subfolder = subfolder
1677 1721 self.fp = fp
1678 1722 self.setFile = setFile
1679 1723 self.flagIsNewFile = 1
1680 1724 self.fileDate = self.dataOut.datatime.date()
1681 1725
1682 1726 self.setFirstHeader()
1683 1727
1684 1728 print '[Writing] Opening file: %s'%self.filename
1685 1729
1686 1730 self.__writeFirstHeader()
1687 1731
1688 1732 return 1
1689 1733
1690 1734 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1691 1735 """
1692 1736 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1693 1737
1694 1738 Inputs:
1695 1739 path : directory where data will be saved
1696 1740 profilesPerBlock : number of profiles per block
1697 1741 set : initial file set
1698 1742 datatype : An integer number that defines data type:
1699 1743 0 : int8 (1 byte)
1700 1744 1 : int16 (2 bytes)
1701 1745 2 : int32 (4 bytes)
1702 1746 3 : int64 (8 bytes)
1703 1747 4 : float32 (4 bytes)
1704 1748 5 : double64 (8 bytes)
1705 1749
1706 1750 Return:
1707 1751 0 : Si no realizo un buen seteo
1708 1752 1 : Si realizo un buen seteo
1709 1753 """
1710 1754
1711 1755 if ext == None:
1712 1756 ext = self.ext
1713 1757
1714 1758 self.ext = ext.lower()
1715 1759
1716 1760 self.path = path
1717 1761
1718 1762 if set is None:
1719 1763 self.setFile = -1
1720 1764 else:
1721 1765 self.setFile = set - 1
1722 1766
1723 1767 self.blocksPerFile = blocksPerFile
1724 1768
1725 1769 self.profilesPerBlock = profilesPerBlock
1726 1770
1727 1771 self.dataOut = dataOut
1728 1772 self.fileDate = self.dataOut.datatime.date()
1729 1773 #By default
1730 1774 self.dtype = self.dataOut.dtype
1731 1775
1732 1776 if datatype is not None:
1733 1777 self.dtype = get_numpy_dtype(datatype)
1734 1778
1735 1779 if not(self.setNextFile()):
1736 1780 print "[Writing] There isn't a next file"
1737 1781 return 0
1738 1782
1739 1783 self.setBlockDimension()
1740 1784
1741 1785 return 1
1742 1786
1743 def run(self, dataOut, **kwargs):
1787 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1744 1788
1745 1789 if not(self.isConfig):
1746 1790
1747 self.setup(dataOut, **kwargs)
1791 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1748 1792 self.isConfig = True
1749 1793
1750 1794 self.putData()
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="SpectraReader" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/sp1_f0" /><Parameter format="date" id="191113" name="startDate" value="2017/01/28" /><Parameter format="date" id="191114" name="endDate" value="2017/01/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="5" /><Parameter format="int" id="191119" name="skip" value="177" /><Parameter format="int" id="191120" name="walk" value="1" /><Parameter format="int" id="191121" name="verbose" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="ParametersProc" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="PublishData" priority="3" type="other"><Parameter format="int" id="191331" name="zeromq" value="1" /><Parameter format="bool" id="191332" name="verbose" value="0" /><Parameter format="int" id="191333" name="delay" value="0" /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file
1 <Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="Spectra" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/sp1_f0" /><Parameter format="date" id="191113" name="startDate" value="2017/01/28" /><Parameter format="date" id="191114" name="endDate" value="2017/01/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="5" /><Parameter format="int" id="191119" name="skip" value="177" /><Parameter format="int" id="191120" name="walk" value="1" /><Parameter format="int" id="191121" name="verbose" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Parameters" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="PublishData" priority="3" type="other"><Parameter format="int" id="191331" name="zeromq" value="1" /><Parameter format="bool" id="191332" name="verbose" value="0" /><Parameter format="int" id="191333" name="delay" value="0" /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,49 +1,57
1 '''
1 """.
2
2 3 Created on Jul 16, 2014
3 4
4 5 @author: Miguel Urco
5 '''
6 """
6 7
7 8 from schainpy import __version__
8 9 from setuptools import setup, Extension
9 10
10 11 setup(name="schainpy",
11 version=__version__,
12 description="Python tools to read, write and process Jicamarca data",
13 author="Miguel Urco",
14 author_email="miguel.urco@jro.igp.gob.pe",
15 url="http://jro.igp.gob.pe",
16 packages = {'schainpy',
17 'schainpy.model',
18 'schainpy.model.data',
19 'schainpy.model.graphics',
20 'schainpy.model.io',
21 'schainpy.model.proc',
22 'schainpy.model.serializer',
23 'schainpy.model.utils',
24 'schainpy.gui',
25 'schainpy.gui.figures',
26 'schainpy.gui.viewcontroller',
27 'schainpy.gui.viewer',
28 'schainpy.gui.viewer.windows'},
29 ext_package='schainpy',
30 py_modules=[''],
31 package_data={'': ['schain.conf.template'],
32 'schainpy.gui.figures': ['*.png','*.jpg'],
33 },
34 include_package_data=False,
35 scripts =['schainpy/gui/schainGUI',
36 'schainpy/scripts/schain'],
37 ext_modules=[Extension("cSchain", ["schainpy/model/proc/extensions.c"])],
38 install_requires=[
39 "scipy >= 0.14.0",
40 "h5py >= 2.2.1",
41 "matplotlib >= 1.4.2",
42 "pyfits >= 3.4",
43 "numpy >= 1.11.2",
44 "paramiko >= 2.1.2",
45 "paho-mqtt >= 1.2",
46 "zmq",
47 "fuzzywuzzy"
48 ],
12 version=__version__,
13 description="Python tools to read, write and process Jicamarca data",
14 author="Miguel Urco",
15 author_email="miguel.urco@jro.igp.gob.pe",
16 url="http://jro.igp.gob.pe",
17 packages={'schainpy',
18 'schainpy.model',
19 'schainpy.model.data',
20 'schainpy.model.graphics',
21 'schainpy.model.io',
22 'schainpy.model.proc',
23 'schainpy.model.serializer',
24 'schainpy.model.utils',
25 'schainpy.gui',
26 'schainpy.gui.figures',
27 'schainpy.gui.viewcontroller',
28 'schainpy.gui.viewer',
29 'schainpy.gui.viewer.windows'},
30 ext_package='schainpy',
31 py_modules=[''],
32 package_data={'': ['schain.conf.template'],
33 'schainpy.gui.figures': ['*.png', '*.jpg'],
34 },
35 include_package_data=False,
36 entry_points={
37 'console_scripts': [
38 'schain = schaincli.cli:main',
39 ],
40 },
41 scripts=['schainpy/gui/schainGUI'],
42 ext_modules=[Extension("cSchain", ["schainpy/model/proc/extensions.c"])],
43 install_requires=[
44 "scipy >= 0.14.0",
45 "h5py >= 2.2.1",
46 "matplotlib >= 1.4.2",
47 "pyfits >= 3.4",
48 "numpy >= 1.11.2",
49 "paramiko >= 2.1.2",
50 "paho-mqtt >= 1.2",
51 "zmq",
52 "fuzzywuzzy",
53 "click",
54 "colorama",
55 "python-Levenshtein"
56 ],
49 57 )
General Comments 0
You need to be logged in to leave comments. Login now