##// END OF EJS Templates
schain
RSS
merge con schain_mp
Jose Chavez -
r977:3514ee80f098 merge
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,4
1 {
2 "python.linting.pylintEnabled": true,
3 "git.ignoreLimitWarning": true
4 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,88
1 import argparse
2
3 from schainpy.controller import Project, multiSchain
4
5 desc = "HF_EXAMPLE"
6
7 controllerObj = Project()
8
9 controllerObj.setup(id='191', name='test01', description=desc)
10
11 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
12 path='/home/nanosat/data/sp1_f0',
13 startDate="2017/01/26",
14 endDate="2017/01/26",
15 startTime="00:00:00",
16 endTime="23:59:59",
17 online=0,
18 #set=1426485881,
19 walk=1,
20 verbose=1
21 #timezone=-5*3600
22 )
23
24 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
25 #
26 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
27 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
28
29 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
30 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
31
32 #
33 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
34 # opObj11.addParameter(name='id', value='1000', format='int')
35 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
36 # opObj11.addParameter(name='channelList', value='0', format='intlist')
37 # opObj11.addParameter(name='zmin', value='-120', format='float')
38 # opObj11.addParameter(name='zmax', value='-70', format='float')
39 # opObj11.addParameter(name='save', value='1', format='int')
40 # opObj11.addParameter(name='figpath', value=figpath, format='str')
41
42 opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
43 # opObj11.addParameter(name='channelList', value='0', format='intList')
44
45 opObj11.addParameter(name='id', value='2000', format='int')
46 # # opObj11.addParameter(name='colormap', value='0', format='bool')
47 opObj11.addParameter(name='onlySNR', value='1', format='bool')
48 opObj11.addParameter(name='DOP', value='0', format='bool')
49 opObj11.addParameter(name='SNR', value='1', format='bool')
50 opObj11.addParameter(name='SNRthresh', value='0', format='int')
51 opObj11.addParameter(name='SNRmin', value='-10', format='int')
52 opObj11.addParameter(name='SNRmax', value='30', format='int')
53 opObj11.addParameter(name='xmin', value='0', format='int')
54 opObj11.addParameter(name='xmax', value='24', format='int')
55
56 # opObj12 = procUnitConfObj3.addOperation(name='ParametersPlot', optype='other')
57 # #opObj11.addParameter(name='channelList', value='0', format='intlist')
58 # opObj12.addParameter(name='id', value='301', format='int')
59 # opObj12.addParameter(name='xmin', value='0', format='float')
60 # opObj12.addParameter(name='xmax', value='24', format='float')
61
62 # opObj11.addParameter(name='zmin', value='-110', format='float')
63 # opObj11.addParameter(name='zmax', value='-70', format='float')
64 # opObj11.addParameter(name='save', value='0', format='int')
65 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
66 #
67 # opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
68 # opObj12.addParameter(name='zeromq', value=1, format='int')
69 # opObj12.addParameter(name='verbose', value=0, format='bool')
70
71
72 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
73 # opObj13.addParameter(name='zeromq', value=1, format='int')
74 # opObj13.addParameter(name='server', value="juanca", format='str')
75
76 # opObj12.addParameter(name='delay', value=0, format='int')
77
78
79 # print "Escribiendo el archivo XML"
80 # controllerObj.writeXml(filename)
81 # print "Leyendo el archivo XML"
82 # controllerObj.readXml(filename)
83
84
85 # timeit.timeit('controllerObj.run()', number=2)
86
87 controllerObj.start()
88
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 You should install "digital_rf_hdf5" module if you want to read USRP data
Show file before | Show file after | Hide comments
@@ -1,102 +1,108
1 # Byte-compiled / optimized / DLL files
1 # Byte-compiled / optimized / DLL files
2 __pycache__/
2 __pycache__/
3 *.py[cod]
3 *.py[cod]
4 *$py.class
4 *$py.class
5
5
6 # C extensions
6 # C extensions
7 *.so
7 *.so
8
8
9 # Distribution / packaging
9 # Distribution / packaging
10 .Python
10 .Python
11 env/
11 env/
12 build/
12 build/
13 develop-eggs/
13 develop-eggs/
14 dist/
14 dist/
15 downloads/
15 downloads/
16 eggs/
16 eggs/
17 .eggs/
17 .eggs/
18 lib/
18 lib/
19 lib64/
19 lib64/
20 parts/
20 parts/
21 sdist/
21 sdist/
22 var/
22 var/
23 wheels/
23 wheels/
24 *.egg-info/
24 *.egg-info/
25 .installed.cfg
25 .installed.cfg
26 *.egg
26 *.egg
27
27
28 # PyInstaller
28 # PyInstaller
29 # Usually these files are written by a python script from a template
29 # Usually these files are written by a python script from a template
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 *.manifest
31 *.manifest
32 *.spec
32 *.spec
33
33
34 # Installer logs
34 # Installer logs
35 pip-log.txt
35 pip-log.txt
36 pip-delete-this-directory.txt
36 pip-delete-this-directory.txt
37
37
38 # Unit test / coverage reports
38 # Unit test / coverage reports
39 htmlcov/
39 htmlcov/
40 .tox/
40 .tox/
41 .coverage
41 .coverage
42 .coverage.*
42 .coverage.*
43 .cache
43 .cache
44 nosetests.xml
44 nosetests.xml
45 coverage.xml
45 coverage.xml
46 *,cover
46 *,cover
47 .hypothesis/
47 .hypothesis/
48
48
49 # Translations
49 # Translations
50 *.mo
50 *.mo
51 *.pot
51 *.pot
52
52
53 # Django stuff:
53 # Django stuff:
54 *.log
54 *.log
55 local_settings.py
55 local_settings.py
56
56
57 # Flask stuff:
57 # Flask stuff:
58 instance/
58 instance/
59 .webassets-cache
59 .webassets-cache
60
60
61 # Scrapy stuff:
61 # Scrapy stuff:
62 .scrapy
62 .scrapy
63
63
64 # Sphinx documentation
64 # Sphinx documentation
65 docs/_build/
65 docs/_build/
66
66
67 # PyBuilder
67 # PyBuilder
68 target/
68 target/
69
69
70 # Jupyter Notebook
70 # Jupyter Notebook
71 .ipynb_checkpoints
71 .ipynb_checkpoints
72
72
73 # pyenv
73 # pyenv
74 .python-version
74 .python-version
75
75
76 # celery beat schedule file
76 # celery beat schedule file
77 celerybeat-schedule
77 celerybeat-schedule
78
78
79 # SageMath parsed files
79 # SageMath parsed files
80 *.sage.py
80 *.sage.py
81
81
82 # dotenv
82 # dotenv
83 .env
83 .env
84
84
85 # virtualenv
85 # virtualenv
86 .venv
86 .venv
87 venv/
87 venv/
88 ENV/
88 ENV/
89
89
90 # Spyder project settings
90 # Spyder project settings
91 .spyderproject
91 .spyderproject
92 .spyproject
92 .spyproject
93
93
94 # Rope project settings
94 # Rope project settings
95 .ropeproject
95 .ropeproject
96
96
97 # mkdocs documentation
97 # mkdocs documentation
98 /site
98 /site
99
99
100 # eclipse
100 # eclipse
101 .project
101 .project
102 .pydevproject
102 .pydevproject
103
104 # vscode
105
106 .vscode
107
108 schainpy/scripts/ No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="Voltage" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="plotter"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="other"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,395 +1,395
1 """The admin module contains all administrative classes relating to the schain python api.
1 """The admin module contains all administrative classes relating to the schain python api.
2
2
3 The main role of this module is to send some reports. It contains a
3 The main role of this module is to send some reports. It contains a
4 notification class and a standard error handing class.
4 notification class and a standard error handing class.
5
5
6 $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
6 $Id: admin.py 3966 2015-12-01 14:32:29Z miguel.urco $
7 """
7 """
8 import os, sys
8 import os, sys
9 import traceback
9 import traceback
10 import smtplib
10 import smtplib
11 import ConfigParser
11 import ConfigParser
12 import StringIO
12 import StringIO
13
13
14 from email.mime.text import MIMEText
14 from email.mime.text import MIMEText
15 from email.mime.application import MIMEApplication
15 from email.mime.application import MIMEApplication
16 from email.mime.multipart import MIMEMultipart
16 from email.mime.multipart import MIMEMultipart
17
17
18 class SchainConfigure():
18 class SchainConfigure():
19
19
20 __DEFAULT_ADMINISTRATOR_EMAIL = ""
20 __DEFAULT_ADMINISTRATOR_EMAIL = ""
21 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
21 __DEFAULT_EMAIL_SERVER = "jro-zimbra.igp.gob.pe"
22 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
22 __DEFAULT_SENDER_EMAIL = "notifier-schain@jro.igp.gob.pe"
23 __DEFAULT_SENDER_PASS = ""
23 __DEFAULT_SENDER_PASS = ""
24
24
25 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
25 __SCHAIN_ADMINISTRATOR_EMAIL = "CONTACT"
26 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
26 __SCHAIN_EMAIL_SERVER = "MAILSERVER"
27 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
27 __SCHAIN_SENDER_EMAIL = "MAILSERVER_ACCOUNT"
28 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
28 __SCHAIN_SENDER_PASS = "MAILSERVER_PASSWORD"
29
29
30 def __init__(self, initFile = None):
30 def __init__(self, initFile = None):
31
31
32 # Set configuration file
32 # Set configuration file
33 if (initFile == None):
33 if (initFile == None):
34 self.__confFilePath = "/etc/schain.conf"
34 self.__confFilePath = "/etc/schain.conf"
35 else:
35 else:
36 self.__confFilePath = initFile
36 self.__confFilePath = initFile
37
37
38 # open configuration file
38 # open configuration file
39 try:
39 try:
40 self.__confFile = open(self.__confFilePath, "r")
40 self.__confFile = open(self.__confFilePath, "r")
41 except IOError:
41 except IOError:
42 # can't read from file - use all hard-coded values
42 # can't read from file - use all hard-coded values
43 self.__initFromHardCode()
43 self.__initFromHardCode()
44 return
44 return
45
45
46 # create Parser using standard module ConfigParser
46 # create Parser using standard module ConfigParser
47 self.__parser = ConfigParser.ConfigParser()
47 self.__parser = ConfigParser.ConfigParser()
48
48
49 # read conf file into a StringIO with "[madrigal]\n" section heading prepended
49 # read conf file into a StringIO with "[madrigal]\n" section heading prepended
50 strConfFile = StringIO.StringIO("[schain]\n" + self.__confFile.read())
50 strConfFile = StringIO.StringIO("[schain]\n" + self.__confFile.read())
51
51
52 # parse StringIO configuration file
52 # parse StringIO configuration file
53 self.__parser.readfp(strConfFile)
53 self.__parser.readfp(strConfFile)
54
54
55 # read information from configuration file
55 # read information from configuration file
56 self.__readConfFile()
56 self.__readConfFile()
57
57
58 # close conf file
58 # close conf file
59 self.__confFile.close()
59 self.__confFile.close()
60
60
61
61
62 def __initFromHardCode(self):
62 def __initFromHardCode(self):
63
63
64 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
64 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
65 self.__sender_pass = self.__DEFAULT_SENDER_PASS
65 self.__sender_pass = self.__DEFAULT_SENDER_PASS
66 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
66 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
67 self.__email_server = self.__DEFAULT_EMAIL_SERVER
67 self.__email_server = self.__DEFAULT_EMAIL_SERVER
68
68
69 def __readConfFile(self):
69 def __readConfFile(self):
70 """__readConfFile is a private helper function that reads information from the parsed config file.
70 """__readConfFile is a private helper function that reads information from the parsed config file.
71
71
72 Inputs: None
72 Inputs: None
73
73
74 Returns: Void.
74 Returns: Void.
75
75
76 Affects: Initializes class member variables that are found in the config file.
76 Affects: Initializes class member variables that are found in the config file.
77
77
78 Exceptions: MadrigalError thrown if any key not found.
78 Exceptions: MadrigalError thrown if any key not found.
79 """
79 """
80
80
81 # get the sender email
81 # get the sender email
82 try:
82 try:
83 self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
83 self.__sender_email = self.__parser.get("schain", self.__SCHAIN_SENDER_EMAIL)
84 except:
84 except:
85 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
85 self.__sender_email = self.__DEFAULT_SENDER_EMAIL
86
86
87 # get the sender password
87 # get the sender password
88 try:
88 try:
89 self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
89 self.__sender_pass = self.__parser.get("schain", self.__SCHAIN_SENDER_PASS)
90 except:
90 except:
91 self.__sender_pass = self.__DEFAULT_SENDER_PASS
91 self.__sender_pass = self.__DEFAULT_SENDER_PASS
92
92
93 # get the administrator email
93 # get the administrator email
94 try:
94 try:
95 self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
95 self.__admin_email = self.__parser.get("schain", self.__SCHAIN_ADMINISTRATOR_EMAIL)
96 except:
96 except:
97 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
97 self.__admin_email = self.__DEFAULT_ADMINISTRATOR_EMAIL
98
98
99 # get the server email
99 # get the server email
100 try:
100 try:
101 self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
101 self.__email_server = self.__parser.get("schain", self.__SCHAIN_EMAIL_SERVER)
102 except:
102 except:
103 self.__email_server = self.__DEFAULT_EMAIL_SERVER
103 self.__email_server = self.__DEFAULT_EMAIL_SERVER
104
104
105 def getEmailServer(self):
105 def getEmailServer(self):
106
106
107 return self.__email_server
107 return self.__email_server
108
108
109 def getSenderEmail(self):
109 def getSenderEmail(self):
110
110
111 return self.__sender_email
111 return self.__sender_email
112
112
113 def getSenderPass(self):
113 def getSenderPass(self):
114
114
115 return self.__sender_pass
115 return self.__sender_pass
116
116
117 def getAdminEmail(self):
117 def getAdminEmail(self):
118
118
119 return self.__admin_email
119 return self.__admin_email
120
120
121 class SchainNotify:
121 class SchainNotify:
122 """SchainNotify is an object used to send messages to an administrator about a Schain software.
122 """SchainNotify is an object used to send messages to an administrator about a Schain software.
123
123
124 This object provides functions needed to send messages to an administrator about a Schain , for now
124 This object provides functions needed to send messages to an administrator about a Schain , for now
125 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
125 only sendAlert, which sends an email to the site administrator found is ADMIN_EMAIL
126
126
127 Usage example:
127 Usage example:
128
128
129 import schainpy.admin
129 import schainpy.admin
130
130
131 try:
131 try:
132
132
133 adminObj = schainpy.admin.SchainNotify()
133 adminObj = schainpy.admin.SchainNotify()
134 adminObj.sendAlert('This is important!', 'Important Message')
134 adminObj.sendAlert('This is important!', 'Important Message')
135
135
136 except schainpy.admin.SchainError, e:
136 except schainpy.admin.SchainError, e:
137
137
138 print e.getExceptionStr()
138 print e.getExceptionStr()
139
139
140
140
141 Non-standard Python modules used:
141 Non-standard Python modules used:
142 None
142 None
143
143
144 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
144 Exceptions thrown: None - Note that SchainNotify tries every trick it knows to avoid
145 throwing exceptions, since this is the class that will generally be called when there is a problem.
145 throwing exceptions, since this is the class that will generally be called when there is a problem.
146
146
147 Change history:
147 Change history:
148
148
149 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Dec. 1, 2015
149 Written by "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Dec. 1, 2015
150 """
150 """
151
151
152 #constants
152 #constants
153
153
154 def __init__(self):
154 def __init__(self):
155 """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
155 """__init__ initializes SchainNotify by getting some basic information from SchainDB and SchainSite.
156
156
157 Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
157 Note that SchainNotify tries every trick it knows to avoid throwing exceptions, since
158 this is the class that will generally be called when there is a problem.
158 this is the class that will generally be called when there is a problem.
159
159
160 Inputs: Existing SchainDB object, by default = None.
160 Inputs: Existing SchainDB object, by default = None.
161
161
162 Returns: void
162 Returns: void
163
163
164 Affects: Initializes self.__binDir.
164 Affects: Initializes self.__binDir.
165
165
166 Exceptions: None.
166 Exceptions: None.
167 """
167 """
168
168
169 # note that the main configuration file is unavailable
169 # note that the main configuration file is unavailable
170 # the best that can be done is send an email to root using localhost mailserver
170 # the best that can be done is send an email to root using localhost mailserver
171 confObj = SchainConfigure()
171 confObj = SchainConfigure()
172
172
173 self.__emailFromAddress = confObj.getSenderEmail()
173 self.__emailFromAddress = confObj.getSenderEmail()
174 self.__emailPass = confObj.getSenderPass()
174 self.__emailPass = confObj.getSenderPass()
175 self.__emailToAddress = confObj.getAdminEmail()
175 self.__emailToAddress = confObj.getAdminEmail()
176 self.__emailServer = confObj.getEmailServer()
176 self.__emailServer = confObj.getEmailServer()
177
177
178 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
178 def sendEmail(self, email_from, email_to, subject='Error running ...', message="", subtitle="", filename="", html_format=True):
179
179
180 if not email_to:
180 if not email_to:
181 return 0
181 return 0
182
182
183 if not self.__emailServer:
183 if not self.__emailServer:
184 return 0
184 return 0
185
185
186 msg = MIMEMultipart()
186 msg = MIMEMultipart()
187 msg['Subject'] = subject
187 msg['Subject'] = subject
188 msg['From'] = "(Python SChain API): " + email_from
188 msg['From'] = "(Python SChain API): " + email_from
189 msg['Reply-to'] = email_from
189 msg['Reply-to'] = email_from
190 msg['To'] = email_to
190 msg['To'] = email_to
191
191
192 # That is what u see if dont have an email reader:
192 # That is what u see if dont have an email reader:
193 msg.preamble = 'SChainPy'
193 msg.preamble = 'SChainPy'
194
194
195 if html_format:
195 if html_format:
196 message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
196 message = "<h1> %s </h1>" %subject + "<h3>" + subtitle.replace("\n", "</h3><h3>\n") + "</h3>" + message.replace("\n", "<br>\n")
197 message = "<html>\n" + message + '</html>'
197 message = "<html>\n" + message + '</html>'
198
198
199 # This is the textual part:
199 # This is the textual part:
200 part = MIMEText(message, "html")
200 part = MIMEText(message, "html")
201 else:
201 else:
202 message = subject + "\n" + subtitle + "\n" + message
202 message = subject + "\n" + subtitle + "\n" + message
203 part = MIMEText(message)
203 part = MIMEText(message)
204
204
205 msg.attach(part)
205 msg.attach(part)
206
206
207 if os.path.isfile(filename):
207 if os.path.isfile(filename):
208 # This is the binary part(The Attachment):
208 # This is the binary part(The Attachment):
209 part = MIMEApplication(open(filename,"rb").read())
209 part = MIMEApplication(open(filename,"rb").read())
210 part.add_header('Content-Disposition',
210 part.add_header('Content-Disposition',
211 'attachment',
211 'attachment',
212 filename=os.path.basename(filename))
212 filename=os.path.basename(filename))
213 msg.attach(part)
213 msg.attach(part)
214
214
215 # Create an instance in SMTP server
215 # Create an instance in SMTP server
216 try:
216 try:
217 smtp = smtplib.SMTP(self.__emailServer)
217 smtp = smtplib.SMTP(self.__emailServer)
218 except:
218 except:
219 print "***** Could not connect to server %s *****" %self.__emailServer
219 print "***** Could not connect to server %s *****" %self.__emailServer
220 return 0
220 return 0
221
221
222 # Start the server:
222 # Start the server:
223 # smtp.ehlo()
223 # smtp.ehlo()
224 if self.__emailPass:
224 if self.__emailPass:
225 smtp.login(self.__emailFromAddress, self.__emailPass)
225 smtp.login(self.__emailFromAddress, self.__emailPass)
226
226
227 # Send the email
227 # Send the email
228 try:
228 try:
229 smtp.sendmail(msg['From'], msg['To'], msg.as_string())
229 smtp.sendmail(msg['From'], msg['To'], msg.as_string())
230 except:
230 except:
231 print "***** Could not send the email to %s *****" %msg['To']
231 print "***** Could not send the email to %s *****" %msg['To']
232 smtp.quit()
232 smtp.quit()
233 return 0
233 return 0
234
234
235 smtp.quit()
235 smtp.quit()
236
236
237 return 1
237 return 1
238
238
239 def sendAlert(self, message, subject = "", subtitle="", filename=""):
239 def sendAlert(self, message, subject = "", subtitle="", filename=""):
240 """sendAlert sends an email with the given message and optional title.
240 """sendAlert sends an email with the given message and optional title.
241
241
242 Inputs: message (string), and optional title (string)
242 Inputs: message (string), and optional title (string)
243
243
244 Returns: void
244 Returns: void
245
245
246 Affects: none
246 Affects: none
247
247
248 Exceptions: None.
248 Exceptions: None.
249 """
249 """
250
250
251 if not self.__emailToAddress:
251 if not self.__emailToAddress:
252 return 0
252 return 0
253
253
254 print "***** Sending alert to %s *****" %self.__emailToAddress
254 print "***** Sending alert to %s *****" %self.__emailToAddress
255 # set up message
255 # set up message
256
256
257 sent=self.sendEmail(email_from=self.__emailFromAddress,
257 sent=self.sendEmail(email_from=self.__emailFromAddress,
258 email_to=self.__emailToAddress,
258 email_to=self.__emailToAddress,
259 subject=subject,
259 subject=subject,
260 message=message,
260 message=message,
261 subtitle=subtitle,
261 subtitle=subtitle,
262 filename=filename)
262 filename=filename)
263
263
264 if not sent:
264 if not sent:
265 return 0
265 return 0
266
266
267 print "***** Your system administrator has been notified *****"
267 print "***** Your system administrator has been notified *****"
268
268
269 return 1
269 return 1
270
270
271 def notify(self, email, message, subject = "", subtitle="", filename=""):
271 def notify(self, email, message, subject = "", subtitle="", filename=""):
272 """notify sends an email with the given message and title to email.
272 """notify sends an email with the given message and title to email.
273
273
274 Inputs: email (string), message (string), and subject (string)
274 Inputs: email (string), message (string), and subject (string)
275
275
276 Returns: void
276 Returns: void
277
277
278 Affects: none
278 Affects: none
279
279
280 Exceptions: None.
280 Exceptions: None.
281 """
281 """
282
282
283 print "Notifying to %s ..." %email
283 print "Notifying to %s ..." %email
284
284
285 self.sendEmail(email_from=self.__emailFromAddress,
285 self.sendEmail(email_from=self.__emailFromAddress,
286 email_to=email,
286 email_to=email,
287 subject=subject,
287 subject=subject,
288 message=message,
288 message=message,
289 subtitle=subtitle,
289 subtitle=subtitle,
290 filename=filename)
290 filename=filename)
291
291
292 print "***** Your system administrator has been notified *****"
292 print "***** Your system administrator has been notified *****"
293
293
294 class SchainError(Exception):
294 class SchainError(Exception):
295 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
295 """SchainError is an exception class that is thrown for all known errors using Schain Py lib.
296
296
297 Usage example:
297 Usage example:
298
298
299 import sys, traceback
299 import sys, traceback
300 import schainpy.admin
300 import schainpy.admin
301
301
302 try:
302 try:
303
303
304 test = open('ImportantFile.txt', 'r')
304 test = open('ImportantFile.txt', 'r')
305
305
306 except:
306 except:
307
307
308 raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
308 raise schainpy.admin.SchainError('ImportantFile.txt not opened!',
309 traceback.format_exception(sys.exc_info()[0],
309 traceback.format_exception(sys.exc_info()[0],
310 sys.exc_info()[1],
310 sys.exc_info()[1],
311 sys.exc_info()[2]))
311 sys.exc_info()[2]))
312 """
312 """
313
313
314
314
315 def __init__(self, strInterpretation, exceptionList=None):
315 def __init__(self, strInterpretation, exceptionList=None):
316 """ __init__ gathers the interpretation string along with all information from sys.exc_info().
316 """ __init__ gathers the interpretation string along with all information from sys.exc_info().
317
317
318 Inputs:
318 Inputs:
319 strIntepretation - A string representing the programmer's interpretation of
319 strIntepretation - A string representing the programmer's interpretation of
320 why the exception occurred
320 why the exception occurred
321
321
322 exceptionList - a list of strings completely describing the exception.
322 exceptionList - a list of strings completely describing the exception.
323 Generated by traceback.format_exception(sys.exc_info()[0],
323 Generated by traceback.format_exception(sys.exc_info()[0],
324 sys.exc_info()[1],
324 sys.exc_info()[1],
325 sys.exc_info()[2])
325 sys.exc_info()[2])
326
326
327 Returns: Void.
327 Returns: Void.
328
328
329 Affects: Initializes class member variables _strInterp, _strExcList.
329 Affects: Initializes class member variables _strInterp, _strExcList.
330
330
331 Exceptions: None.
331 Exceptions: None.
332 """
332 """
333
333
334 if not exceptionList:
334 if not exceptionList:
335 exceptionList = traceback.format_exception(sys.exc_info()[0],
335 exceptionList = traceback.format_exception(sys.exc_info()[0],
336 sys.exc_info()[1],
336 sys.exc_info()[1],
337 sys.exc_info()[2])
337 sys.exc_info()[2])
338
338
339 self._strInterp = strInterpretation
339 self._strInterp = strInterpretation
340 self._strExcList = exceptionList
340 self._strExcList = exceptionList
341
341
342
342
343 def getExceptionStr(self):
343 def getExceptionStr(self):
344 """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
344 """ getExceptionStr returns a formatted string ready for printing completely describing the exception.
345
345
346 Inputs: None
346 Inputs: None
347
347
348 Returns: A formatted string ready for printing completely describing the exception.
348 Returns: A formatted string ready for printing completely describing the exception.
349
349
350 Affects: None
350 Affects: None
351
351
352 Exceptions: None.
352 Exceptions: None.
353 """
353 """
354 excStr = ''
354 excStr = ''
355 excStr = excStr + self._strInterp + '\n\n'
355 excStr = excStr + self._strInterp + '\n\n'
356
356
357 if self._strExcList != None:
357 if self._strExcList != None:
358 for item in self._strExcList:
358 for item in self._strExcList:
359 excStr = excStr + str(item) + '\n'
359 excStr = excStr + str(item) + '\n'
360
360
361 return excStr
361 return excStr
362
362
363 def __str__(self):
363 def __str__(self):
364
364
365 return(self.getExceptionStr())
365 return(self.getExceptionStr())
366
366
367
367
368 def getExceptionHtml(self):
368 def getExceptionHtml(self):
369 """ getExceptionHtml returns an Html formatted string completely describing the exception.
369 """ getExceptionHtml returns an Html formatted string completely describing the exception.
370
370
371 Inputs: None
371 Inputs: None
372
372
373 Returns: A formatted string ready for printing completely describing the exception.
373 Returns: A formatted string ready for printing completely describing the exception.
374
374
375 Affects: None
375 Affects: None
376
376
377 Exceptions: None.
377 Exceptions: None.
378 """
378 """
379
379
380 excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
380 excStr = '<BR>The following Schain Python exception has occurred:\n<BR>'
381 excStr = excStr + self._strInterp + '\n<BR>\n'
381 excStr = excStr + self._strInterp + '\n<BR>\n'
382
382
383 if self._strExcList != None:
383 if self._strExcList != None:
384 for item in self._strExcList:
384 for item in self._strExcList:
385 excStr = excStr + str(item) + '\n<BR>'
385 excStr = excStr + str(item) + '\n<BR>'
386
386
387 return excStr
387 return excStr
388
388
389 if __name__ == '__main__':
389 if __name__ == '__main__':
390
390
391 test = SchainNotify()
391 test = SchainNotify()
392
392
393 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
393 test.sendAlert('This is a message from the python module SchainNotify', 'Test from SchainNotify')
394
394
395 print 'Hopefully message sent - check.'
395 print 'Hopefully message sent - check.'
Show file before | Show file after | Hide comments
@@ -1,1321 +1,1324
1 '''
1 '''
2 Created on September , 2012
2 Created on September , 2012
3 @author:
3 @author:
4 '''
4 '''
5
5
6 import sys
6 import sys
7 import ast
7 import ast
8 import datetime
8 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, Queue, cpu_count
13
13
14 import schainpy
14 import schainpy
15 import schainpy.admin
15 import schainpy.admin
16
16
17 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
17 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
18 from xml.dom import minidom
18 from xml.dom import minidom
19
19
20 from schainpy.model import *
20 from schainpy.model import *
21 from time import sleep
21 from time import sleep
22
22
23 def prettify(elem):
23 def prettify(elem):
24 """Return a pretty-printed XML string for the Element.
24 """Return a pretty-printed XML string for the Element.
25 """
25 """
26 rough_string = tostring(elem, 'utf-8')
26 rough_string = tostring(elem, 'utf-8')
27 reparsed = minidom.parseString(rough_string)
27 reparsed = minidom.parseString(rough_string)
28 return reparsed.toprettyxml(indent=" ")
28 return reparsed.toprettyxml(indent=" ")
29
29
30 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
30 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
31 skip = 0
31 skip = 0
32 cursor = 0
32 cursor = 0
33 nFiles = None
33 nFiles = None
34 processes = []
34 processes = []
35 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
35 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
36 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
36 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
37 days = (dt2 - dt1).days
37 days = (dt2 - dt1).days
38
38
39 for day in range(days+1):
39 for day in range(days+1):
40 skip = 0
40 skip = 0
41 cursor = 0
41 cursor = 0
42 q = Queue()
42 q = Queue()
43 processes = []
43 processes = []
44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
44 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
45 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
46 firstProcess.start()
46 firstProcess.start()
47 if by_day:
47 if by_day:
48 continue
48 continue
49 nFiles = q.get()
49 nFiles = q.get()
50 firstProcess.terminate()
50 firstProcess.terminate()
51 skip = int(math.ceil(nFiles/nProcess))
51 skip = int(math.ceil(nFiles/nProcess))
52 while True:
52 while True:
53 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
53 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
54 processes[cursor].start()
54 processes[cursor].start()
55 if nFiles < cursor*skip:
55 if nFiles < cursor*skip:
56 break
56 break
57 cursor += 1
57 cursor += 1
58
58
59 def beforeExit(exctype, value, trace):
59 def beforeExit(exctype, value, trace):
60 for process in processes:
60 for process in processes:
61 process.terminate()
61 process.terminate()
62 process.join()
62 process.join()
63 print traceback.print_tb(trace)
63 print traceback.print_tb(trace)
64
64
65 sys.excepthook = beforeExit
65 sys.excepthook = beforeExit
66
66
67 for process in processes:
67 for process in processes:
68 process.join()
68 process.join()
69 process.terminate()
69 process.terminate()
70 time.sleep(3)
70 time.sleep(3)
71
71
72 class ParameterConf():
72 class ParameterConf():
73
73
74 id = None
74 id = None
75 name = None
75 name = None
76 value = None
76 value = None
77 format = None
77 format = None
78
78
79 __formated_value = None
79 __formated_value = None
80
80
81 ELEMENTNAME = 'Parameter'
81 ELEMENTNAME = 'Parameter'
82
82
83 def __init__(self):
83 def __init__(self):
84
84
85 self.format = 'str'
85 self.format = 'str'
86
86
87 def getElementName(self):
87 def getElementName(self):
88
88
89 return self.ELEMENTNAME
89 return self.ELEMENTNAME
90
90
91 def getValue(self):
91 def getValue(self):
92
92
93 value = self.value
93 value = self.value
94 format = self.format
94 format = self.format
95
95
96 if self.__formated_value != None:
96 if self.__formated_value != None:
97
97
98 return self.__formated_value
98 return self.__formated_value
99
99
100 if format == 'obj':
100 if format == 'obj':
101 return value
101 return value
102
102
103 if format == 'str':
103 if format == 'str':
104 self.__formated_value = str(value)
104 self.__formated_value = str(value)
105 return self.__formated_value
105 return self.__formated_value
106
106
107 if value == '':
107 if value == '':
108 raise ValueError, "%s: This parameter value is empty" %self.name
108 raise ValueError, "%s: This parameter value is empty" %self.name
109
109
110 if format == 'list':
110 if format == 'list':
111 strList = value.split(',')
111 strList = value.split(',')
112
112
113 self.__formated_value = strList
113 self.__formated_value = strList
114
114
115 return self.__formated_value
115 return self.__formated_value
116
116
117 if format == 'intlist':
117 if format == 'intlist':
118 """
118 """
119 Example:
119 Example:
120 value = (0,1,2)
120 value = (0,1,2)
121 """
121 """
122
122
123 new_value = ast.literal_eval(value)
123 new_value = ast.literal_eval(value)
124
124
125 if type(new_value) not in (tuple, list):
125 if type(new_value) not in (tuple, list):
126 new_value = [int(new_value)]
126 new_value = [int(new_value)]
127
127
128 self.__formated_value = new_value
128 self.__formated_value = new_value
129
129
130 return self.__formated_value
130 return self.__formated_value
131
131
132 if format == 'floatlist':
132 if format == 'floatlist':
133 """
133 """
134 Example:
134 Example:
135 value = (0.5, 1.4, 2.7)
135 value = (0.5, 1.4, 2.7)
136 """
136 """
137
137
138 new_value = ast.literal_eval(value)
138 new_value = ast.literal_eval(value)
139
139
140 if type(new_value) not in (tuple, list):
140 if type(new_value) not in (tuple, list):
141 new_value = [float(new_value)]
141 new_value = [float(new_value)]
142
142
143 self.__formated_value = new_value
143 self.__formated_value = new_value
144
144
145 return self.__formated_value
145 return self.__formated_value
146
146
147 if format == 'date':
147 if format == 'date':
148 strList = value.split('/')
148 strList = value.split('/')
149 intList = [int(x) for x in strList]
149 intList = [int(x) for x in strList]
150 date = datetime.date(intList[0], intList[1], intList[2])
150 date = datetime.date(intList[0], intList[1], intList[2])
151
151
152 self.__formated_value = date
152 self.__formated_value = date
153
153
154 return self.__formated_value
154 return self.__formated_value
155
155
156 if format == 'time':
156 if format == 'time':
157 strList = value.split(':')
157 strList = value.split(':')
158 intList = [int(x) for x in strList]
158 intList = [int(x) for x in strList]
159 time = datetime.time(intList[0], intList[1], intList[2])
159 time = datetime.time(intList[0], intList[1], intList[2])
160
160
161 self.__formated_value = time
161 self.__formated_value = time
162
162
163 return self.__formated_value
163 return self.__formated_value
164
164
165 if format == 'pairslist':
165 if format == 'pairslist':
166 """
166 """
167 Example:
167 Example:
168 value = (0,1),(1,2)
168 value = (0,1),(1,2)
169 """
169 """
170
170
171 new_value = ast.literal_eval(value)
171 new_value = ast.literal_eval(value)
172
172
173 if type(new_value) not in (tuple, list):
173 if type(new_value) not in (tuple, list):
174 raise ValueError, "%s has to be a tuple or list of pairs" %value
174 raise ValueError, "%s has to be a tuple or list of pairs" %value
175
175
176 if type(new_value[0]) not in (tuple, list):
176 if type(new_value[0]) not in (tuple, list):
177 if len(new_value) != 2:
177 if len(new_value) != 2:
178 raise ValueError, "%s has to be a tuple or list of pairs" %value
178 raise ValueError, "%s has to be a tuple or list of pairs" %value
179 new_value = [new_value]
179 new_value = [new_value]
180
180
181 for thisPair in new_value:
181 for thisPair in new_value:
182 if len(thisPair) != 2:
182 if len(thisPair) != 2:
183 raise ValueError, "%s has to be a tuple or list of pairs" %value
183 raise ValueError, "%s has to be a tuple or list of pairs" %value
184
184
185 self.__formated_value = new_value
185 self.__formated_value = new_value
186
186
187 return self.__formated_value
187 return self.__formated_value
188
188
189 if format == 'multilist':
189 if format == 'multilist':
190 """
190 """
191 Example:
191 Example:
192 value = (0,1,2),(3,4,5)
192 value = (0,1,2),(3,4,5)
193 """
193 """
194 multiList = ast.literal_eval(value)
194 multiList = ast.literal_eval(value)
195
195
196 if type(multiList[0]) == int:
196 if type(multiList[0]) == int:
197 multiList = ast.literal_eval("(" + value + ")")
197 multiList = ast.literal_eval("(" + value + ")")
198
198
199 self.__formated_value = multiList
199 self.__formated_value = multiList
200
200
201 return self.__formated_value
201 return self.__formated_value
202
202
203 if format == 'bool':
203 if format == 'bool':
204 value = int(value)
204 value = int(value)
205
205
206 if format == 'int':
206 if format == 'int':
207 value = float(value)
207 value = float(value)
208
208
209 format_func = eval(format)
209 format_func = eval(format)
210
210
211 self.__formated_value = format_func(value)
211 self.__formated_value = format_func(value)
212
212
213 return self.__formated_value
213 return self.__formated_value
214
214
215 def updateId(self, new_id):
215 def updateId(self, new_id):
216
216
217 self.id = str(new_id)
217 self.id = str(new_id)
218
218
219 def setup(self, id, name, value, format='str'):
219 def setup(self, id, name, value, format='str'):
220
221 self.id = str(id)
220 self.id = str(id)
222 self.name = name
221 self.name = name
223 if format == 'obj':
222 if format == 'obj':
224 self.value = value
223 self.value = value
225 else:
224 else:
226 self.value = str(value)
225 self.value = str(value)
227 self.format = str.lower(format)
226 self.format = str.lower(format)
228
227
229 self.getValue()
228 self.getValue()
230
229
231 return 1
230 return 1
232
231
233 def update(self, name, value, format='str'):
232 def update(self, name, value, format='str'):
234
233
235 self.name = name
234 self.name = name
236 self.value = str(value)
235 self.value = str(value)
237 self.format = format
236 self.format = format
238
237
239 def makeXml(self, opElement):
238 def makeXml(self, opElement):
240 if self.name not in ('queue',):
239 if self.name not in ('queue',):
241 parmElement = SubElement(opElement, self.ELEMENTNAME)
240 parmElement = SubElement(opElement, self.ELEMENTNAME)
242 parmElement.set('id', str(self.id))
241 parmElement.set('id', str(self.id))
243 parmElement.set('name', self.name)
242 parmElement.set('name', self.name)
244 parmElement.set('value', self.value)
243 parmElement.set('value', self.value)
245 parmElement.set('format', self.format)
244 parmElement.set('format', self.format)
246
245
247 def readXml(self, parmElement):
246 def readXml(self, parmElement):
248
247
249 self.id = parmElement.get('id')
248 self.id = parmElement.get('id')
250 self.name = parmElement.get('name')
249 self.name = parmElement.get('name')
251 self.value = parmElement.get('value')
250 self.value = parmElement.get('value')
252 self.format = str.lower(parmElement.get('format'))
251 self.format = str.lower(parmElement.get('format'))
253
252
254 #Compatible with old signal chain version
253 #Compatible with old signal chain version
255 if self.format == 'int' and self.name == 'idfigure':
254 if self.format == 'int' and self.name == 'idfigure':
256 self.name = 'id'
255 self.name = 'id'
257
256
258 def printattr(self):
257 def printattr(self):
259
258
260 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
259 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
261
260
262 class OperationConf():
261 class OperationConf():
263
262
264 id = None
263 id = None
265 name = None
264 name = None
266 priority = None
265 priority = None
267 type = None
266 type = None
268
267
269 parmConfObjList = []
268 parmConfObjList = []
270
269
271 ELEMENTNAME = 'Operation'
270 ELEMENTNAME = 'Operation'
272
271
273 def __init__(self):
272 def __init__(self):
274
273
275 self.id = '0'
274 self.id = '0'
276 self.name = None
275 self.name = None
277 self.priority = None
276 self.priority = None
278 self.type = 'self'
277 self.type = 'self'
279
278
280
279
281 def __getNewId(self):
280 def __getNewId(self):
282
281
283 return int(self.id)*10 + len(self.parmConfObjList) + 1
282 return int(self.id)*10 + len(self.parmConfObjList) + 1
284
283
285 def updateId(self, new_id):
284 def updateId(self, new_id):
286
285
287 self.id = str(new_id)
286 self.id = str(new_id)
288
287
289 n = 1
288 n = 1
290 for parmObj in self.parmConfObjList:
289 for parmObj in self.parmConfObjList:
291
290
292 idParm = str(int(new_id)*10 + n)
291 idParm = str(int(new_id)*10 + n)
293 parmObj.updateId(idParm)
292 parmObj.updateId(idParm)
294
293
295 n += 1
294 n += 1
296
295
297 def getElementName(self):
296 def getElementName(self):
298
297
299 return self.ELEMENTNAME
298 return self.ELEMENTNAME
300
299
301 def getParameterObjList(self):
300 def getParameterObjList(self):
302
301
303 return self.parmConfObjList
302 return self.parmConfObjList
304
303
305 def getParameterObj(self, parameterName):
304 def getParameterObj(self, parameterName):
306
305
307 for parmConfObj in self.parmConfObjList:
306 for parmConfObj in self.parmConfObjList:
308
307
309 if parmConfObj.name != parameterName:
308 if parmConfObj.name != parameterName:
310 continue
309 continue
311
310
312 return parmConfObj
311 return parmConfObj
313
312
314 return None
313 return None
315
314
316 def getParameterObjfromValue(self, parameterValue):
315 def getParameterObjfromValue(self, parameterValue):
317
316
318 for parmConfObj in self.parmConfObjList:
317 for parmConfObj in self.parmConfObjList:
319
318
320 if parmConfObj.getValue() != parameterValue:
319 if parmConfObj.getValue() != parameterValue:
321 continue
320 continue
322
321
323 return parmConfObj.getValue()
322 return parmConfObj.getValue()
324
323
325 return None
324 return None
326
325
327 def getParameterValue(self, parameterName):
326 def getParameterValue(self, parameterName):
328
327
329 parameterObj = self.getParameterObj(parameterName)
328 parameterObj = self.getParameterObj(parameterName)
330
329
331 # if not parameterObj:
330 # if not parameterObj:
332 # return None
331 # return None
333
332
334 value = parameterObj.getValue()
333 value = parameterObj.getValue()
335
334
336 return value
335 return value
337
336
338
337
339 def getKwargs(self):
338 def getKwargs(self):
340
339
341 kwargs = {}
340 kwargs = {}
342
341
343 for parmConfObj in self.parmConfObjList:
342 for parmConfObj in self.parmConfObjList:
344 if self.name == 'run' and parmConfObj.name == 'datatype':
343 if self.name == 'run' and parmConfObj.name == 'datatype':
345 continue
344 continue
346
345
347 kwargs[parmConfObj.name] = parmConfObj.getValue()
346 kwargs[parmConfObj.name] = parmConfObj.getValue()
348
347
349 return kwargs
348 return kwargs
350
349
351 def setup(self, id, name, priority, type):
350 def setup(self, id, name, priority, type):
352
351
353 self.id = str(id)
352 self.id = str(id)
354 self.name = name
353 self.name = name
355 self.type = type
354 self.type = type
356 self.priority = priority
355 self.priority = priority
357
356
358 self.parmConfObjList = []
357 self.parmConfObjList = []
359
358
360 def removeParameters(self):
359 def removeParameters(self):
361
360
362 for obj in self.parmConfObjList:
361 for obj in self.parmConfObjList:
363 del obj
362 del obj
364
363
365 self.parmConfObjList = []
364 self.parmConfObjList = []
366
365
367 def addParameter(self, name, value, format='str'):
366 def addParameter(self, name, value, format='str'):
368
367
369 id = self.__getNewId()
368 id = self.__getNewId()
370
369
371 parmConfObj = ParameterConf()
370 parmConfObj = ParameterConf()
372 if not parmConfObj.setup(id, name, value, format):
371 if not parmConfObj.setup(id, name, value, format):
373 return None
372 return None
374
373
375 self.parmConfObjList.append(parmConfObj)
374 self.parmConfObjList.append(parmConfObj)
376
375
377 return parmConfObj
376 return parmConfObj
378
377
379 def changeParameter(self, name, value, format='str'):
378 def changeParameter(self, name, value, format='str'):
380
379
381 parmConfObj = self.getParameterObj(name)
380 parmConfObj = self.getParameterObj(name)
382 parmConfObj.update(name, value, format)
381 parmConfObj.update(name, value, format)
383
382
384 return parmConfObj
383 return parmConfObj
385
384
386 def makeXml(self, procUnitElement):
385 def makeXml(self, procUnitElement):
387
386
388 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
387 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
389 opElement.set('id', str(self.id))
388 opElement.set('id', str(self.id))
390 opElement.set('name', self.name)
389 opElement.set('name', self.name)
391 opElement.set('type', self.type)
390 opElement.set('type', self.type)
392 opElement.set('priority', str(self.priority))
391 opElement.set('priority', str(self.priority))
393
392
394 for parmConfObj in self.parmConfObjList:
393 for parmConfObj in self.parmConfObjList:
395 parmConfObj.makeXml(opElement)
394 parmConfObj.makeXml(opElement)
396
395
397 def readXml(self, opElement):
396 def readXml(self, opElement):
398
397
399 self.id = opElement.get('id')
398 self.id = opElement.get('id')
400 self.name = opElement.get('name')
399 self.name = opElement.get('name')
401 self.type = opElement.get('type')
400 self.type = opElement.get('type')
402 self.priority = opElement.get('priority')
401 self.priority = opElement.get('priority')
403
402
404 #Compatible with old signal chain version
403 #Compatible with old signal chain version
405 #Use of 'run' method instead 'init'
404 #Use of 'run' method instead 'init'
406 if self.type == 'self' and self.name == 'init':
405 if self.type == 'self' and self.name == 'init':
407 self.name = 'run'
406 self.name = 'run'
408
407
409 self.parmConfObjList = []
408 self.parmConfObjList = []
410
409
411 parmElementList = opElement.iter(ParameterConf().getElementName())
410 parmElementList = opElement.iter(ParameterConf().getElementName())
412
411
413 for parmElement in parmElementList:
412 for parmElement in parmElementList:
414 parmConfObj = ParameterConf()
413 parmConfObj = ParameterConf()
415 parmConfObj.readXml(parmElement)
414 parmConfObj.readXml(parmElement)
416
415
417 #Compatible with old signal chain version
416 #Compatible with old signal chain version
418 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
417 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
419 if self.type != 'self' and self.name == 'Plot':
418 if self.type != 'self' and self.name == 'Plot':
420 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
419 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
421 self.name = parmConfObj.value
420 self.name = parmConfObj.value
422 continue
421 continue
423
422
424 self.parmConfObjList.append(parmConfObj)
423 self.parmConfObjList.append(parmConfObj)
425
424
426 def printattr(self):
425 def printattr(self):
427
426
428 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
427 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
429 self.id,
428 self.id,
430 self.name,
429 self.name,
431 self.type,
430 self.type,
432 self.priority)
431 self.priority)
433
432
434 for parmConfObj in self.parmConfObjList:
433 for parmConfObj in self.parmConfObjList:
435 parmConfObj.printattr()
434 parmConfObj.printattr()
436
435
437 def createObject(self, plotter_queue=None):
436 def createObject(self, plotter_queue=None):
438
437
439
438
440 if self.type == 'self':
439 if self.type == 'self':
441 raise ValueError, "This operation type cannot be created"
440 raise ValueError, "This operation type cannot be created"
442
441
443 if self.type == 'plotter':
442 if self.type == 'plotter':
444 #Plotter(plotter_name)
443 #Plotter(plotter_name)
445 if not plotter_queue:
444 if not plotter_queue:
446 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
445 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
447
446
448 opObj = Plotter(self.name, plotter_queue)
447 opObj = Plotter(self.name, plotter_queue)
449
448
450 if self.type == 'external' or self.type == 'other':
449 if self.type == 'external' or self.type == 'other':
451
450
452 className = eval(self.name)
451 className = eval(self.name)
453 kwargs = self.getKwargs()
452 kwargs = self.getKwargs()
454
453
455 opObj = className(**kwargs)
454 opObj = className(**kwargs)
456
455
457 return opObj
456 return opObj
458
457
459
458
460 class ProcUnitConf():
459 class ProcUnitConf():
461
460
462 id = None
461 id = None
463 name = None
462 name = None
464 datatype = None
463 datatype = None
465 inputId = None
464 inputId = None
466 parentId = None
465 parentId = None
467
466
468 opConfObjList = []
467 opConfObjList = []
469
468
470 procUnitObj = None
469 procUnitObj = None
471 opObjList = []
470 opObjList = []
472
471
473 ELEMENTNAME = 'ProcUnit'
472 ELEMENTNAME = 'ProcUnit'
474
473
475 def __init__(self):
474 def __init__(self):
476
475
477 self.id = None
476 self.id = None
478 self.datatype = None
477 self.datatype = None
479 self.name = None
478 self.name = None
480 self.inputId = None
479 self.inputId = None
481
480
482 self.opConfObjList = []
481 self.opConfObjList = []
483
482
484 self.procUnitObj = None
483 self.procUnitObj = None
485 self.opObjDict = {}
484 self.opObjDict = {}
486
485
487 def __getPriority(self):
486 def __getPriority(self):
488
487
489 return len(self.opConfObjList)+1
488 return len(self.opConfObjList)+1
490
489
491 def __getNewId(self):
490 def __getNewId(self):
492
491
493 return int(self.id)*10 + len(self.opConfObjList) + 1
492 return int(self.id)*10 + len(self.opConfObjList) + 1
494
493
495 def getElementName(self):
494 def getElementName(self):
496
495
497 return self.ELEMENTNAME
496 return self.ELEMENTNAME
498
497
499 def getId(self):
498 def getId(self):
500
499
501 return self.id
500 return self.id
502
501
503 def updateId(self, new_id, parentId=parentId):
502 def updateId(self, new_id, parentId=parentId):
504
503
505
504
506 new_id = int(parentId)*10 + (int(self.id) % 10)
505 new_id = int(parentId)*10 + (int(self.id) % 10)
507 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
506 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
508
507
509 #If this proc unit has not inputs
508 #If this proc unit has not inputs
510 if self.inputId == '0':
509 if self.inputId == '0':
511 new_inputId = 0
510 new_inputId = 0
512
511
513 n = 1
512 n = 1
514 for opConfObj in self.opConfObjList:
513 for opConfObj in self.opConfObjList:
515
514
516 idOp = str(int(new_id)*10 + n)
515 idOp = str(int(new_id)*10 + n)
517 opConfObj.updateId(idOp)
516 opConfObj.updateId(idOp)
518
517
519 n += 1
518 n += 1
520
519
521 self.parentId = str(parentId)
520 self.parentId = str(parentId)
522 self.id = str(new_id)
521 self.id = str(new_id)
523 self.inputId = str(new_inputId)
522 self.inputId = str(new_inputId)
524
523
525
524
526 def getInputId(self):
525 def getInputId(self):
527
526
528 return self.inputId
527 return self.inputId
529
528
530 def getOperationObjList(self):
529 def getOperationObjList(self):
531
530
532 return self.opConfObjList
531 return self.opConfObjList
533
532
534 def getOperationObj(self, name=None):
533 def getOperationObj(self, name=None):
535
534
536 for opConfObj in self.opConfObjList:
535 for opConfObj in self.opConfObjList:
537
536
538 if opConfObj.name != name:
537 if opConfObj.name != name:
539 continue
538 continue
540
539
541 return opConfObj
540 return opConfObj
542
541
543 return None
542 return None
544
543
545 def getOpObjfromParamValue(self, value=None):
544 def getOpObjfromParamValue(self, value=None):
546
545
547 for opConfObj in self.opConfObjList:
546 for opConfObj in self.opConfObjList:
548 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
547 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
549 continue
548 continue
550 return opConfObj
549 return opConfObj
551 return None
550 return None
552
551
553 def getProcUnitObj(self):
552 def getProcUnitObj(self):
554
553
555 return self.procUnitObj
554 return self.procUnitObj
556
555
557 def setup(self, id, name, datatype, inputId, parentId=None):
556 def setup(self, id, name, datatype, inputId, parentId=None):
558
557
559 #Compatible with old signal chain version
558 #Compatible with old signal chain version
560 if datatype==None and name==None:
559 if datatype==None and name==None:
561 raise ValueError, "datatype or name should be defined"
560 raise ValueError, "datatype or name should be defined"
562
561
563 if name==None:
562 if name==None:
564 if 'Proc' in datatype:
563 if 'Proc' in datatype:
565 name = datatype
564 name = datatype
566 else:
565 else:
567 name = '%sProc' %(datatype)
566 name = '%sProc' %(datatype)
568
567
569 if datatype==None:
568 if datatype==None:
570 datatype = name.replace('Proc','')
569 datatype = name.replace('Proc','')
571
570
572 self.id = str(id)
571 self.id = str(id)
573 self.name = name
572 self.name = name
574 self.datatype = datatype
573 self.datatype = datatype
575 self.inputId = inputId
574 self.inputId = inputId
576 self.parentId = parentId
575 self.parentId = parentId
577
576
578 self.opConfObjList = []
577 self.opConfObjList = []
579
578
580 self.addOperation(name='run', optype='self')
579 self.addOperation(name='run', optype='self')
581
580
582 def removeOperations(self):
581 def removeOperations(self):
583
582
584 for obj in self.opConfObjList:
583 for obj in self.opConfObjList:
585 del obj
584 del obj
586
585
587 self.opConfObjList = []
586 self.opConfObjList = []
588 self.addOperation(name='run')
587 self.addOperation(name='run')
589
588
590 def addParameter(self, **kwargs):
589 def addParameter(self, **kwargs):
591 '''
590 '''
592 Add parameters to "run" operation
591 Add parameters to "run" operation
593 '''
592 '''
594 opObj = self.opConfObjList[0]
593 opObj = self.opConfObjList[0]
595
594
596 opObj.addParameter(**kwargs)
595 opObj.addParameter(**kwargs)
597
596
598 return opObj
597 return opObj
599
598
600 def addOperation(self, name, optype='self'):
599 def addOperation(self, name, optype='self'):
601
600
602 id = self.__getNewId()
601 id = self.__getNewId()
603 priority = self.__getPriority()
602 priority = self.__getPriority()
604
603
605 opConfObj = OperationConf()
604 opConfObj = OperationConf()
606 opConfObj.setup(id, name=name, priority=priority, type=optype)
605 opConfObj.setup(id, name=name, priority=priority, type=optype)
607
606
608 self.opConfObjList.append(opConfObj)
607 self.opConfObjList.append(opConfObj)
609
608
610 return opConfObj
609 return opConfObj
611
610
612 def makeXml(self, projectElement):
611 def makeXml(self, projectElement):
613
612
614 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
613 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
615 procUnitElement.set('id', str(self.id))
614 procUnitElement.set('id', str(self.id))
616 procUnitElement.set('name', self.name)
615 procUnitElement.set('name', self.name)
617 procUnitElement.set('datatype', self.datatype)
616 procUnitElement.set('datatype', self.datatype)
618 procUnitElement.set('inputId', str(self.inputId))
617 procUnitElement.set('inputId', str(self.inputId))
619
618
620 for opConfObj in self.opConfObjList:
619 for opConfObj in self.opConfObjList:
621 opConfObj.makeXml(procUnitElement)
620 opConfObj.makeXml(procUnitElement)
622
621
623 def readXml(self, upElement):
622 def readXml(self, upElement):
624
623
625 self.id = upElement.get('id')
624 self.id = upElement.get('id')
626 self.name = upElement.get('name')
625 self.name = upElement.get('name')
627 self.datatype = upElement.get('datatype')
626 self.datatype = upElement.get('datatype')
628 self.inputId = upElement.get('inputId')
627 self.inputId = upElement.get('inputId')
629
628
630 if self.ELEMENTNAME == "ReadUnit":
629 if self.ELEMENTNAME == "ReadUnit":
631 self.datatype = self.datatype.replace("Reader", "")
630 self.datatype = self.datatype.replace("Reader", "")
632
631
633 if self.ELEMENTNAME == "ProcUnit":
632 if self.ELEMENTNAME == "ProcUnit":
634 self.datatype = self.datatype.replace("Proc", "")
633 self.datatype = self.datatype.replace("Proc", "")
635
634
636 if self.inputId == 'None':
635 if self.inputId == 'None':
637 self.inputId = '0'
636 self.inputId = '0'
638
637
639 self.opConfObjList = []
638 self.opConfObjList = []
640
639
641 opElementList = upElement.iter(OperationConf().getElementName())
640 opElementList = upElement.iter(OperationConf().getElementName())
642
641
643 for opElement in opElementList:
642 for opElement in opElementList:
644 opConfObj = OperationConf()
643 opConfObj = OperationConf()
645 opConfObj.readXml(opElement)
644 opConfObj.readXml(opElement)
646 self.opConfObjList.append(opConfObj)
645 self.opConfObjList.append(opConfObj)
647
646
648 def printattr(self):
647 def printattr(self):
649
648
650 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
649 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
651 self.id,
650 self.id,
652 self.name,
651 self.name,
653 self.datatype,
652 self.datatype,
654 self.inputId)
653 self.inputId)
655
654
656 for opConfObj in self.opConfObjList:
655 for opConfObj in self.opConfObjList:
657 opConfObj.printattr()
656 opConfObj.printattr()
658
657
659
658
660 def getKwargs(self):
659 def getKwargs(self):
661
660
662 opObj = self.opConfObjList[0]
661 opObj = self.opConfObjList[0]
663 kwargs = opObj.getKwargs()
662 kwargs = opObj.getKwargs()
664
663
665 return kwargs
664 return kwargs
666
665
667 def createObjects(self, plotter_queue=None):
666 def createObjects(self, plotter_queue=None):
668
667
669 className = eval(self.name)
668 className = eval(self.name)
670 kwargs = self.getKwargs()
669 kwargs = self.getKwargs()
671 procUnitObj = className(**kwargs)
670 procUnitObj = className(**kwargs)
672
671
673 for opConfObj in self.opConfObjList:
672 for opConfObj in self.opConfObjList:
674
673
675 if opConfObj.type=='self' and self.name=='run':
674 if opConfObj.type=='self' and self.name=='run':
676 continue
675 continue
677 elif opConfObj.type=='self':
676 elif opConfObj.type=='self':
678 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
677 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
679 continue
678 continue
680
679
681 opObj = opConfObj.createObject(plotter_queue)
680 opObj = opConfObj.createObject(plotter_queue)
682
681
683 self.opObjDict[opConfObj.id] = opObj
682 self.opObjDict[opConfObj.id] = opObj
684
683
685 procUnitObj.addOperation(opObj, opConfObj.id)
684 procUnitObj.addOperation(opObj, opConfObj.id)
686
685
687 self.procUnitObj = procUnitObj
686 self.procUnitObj = procUnitObj
688
687
689 return procUnitObj
688 return procUnitObj
690
689
691 def run(self):
690 def run(self):
692
691
693 is_ok = False
692 is_ok = False
694
693
695 for opConfObj in self.opConfObjList:
694 for opConfObj in self.opConfObjList:
696
695
697 kwargs = {}
696 kwargs = {}
698 for parmConfObj in opConfObj.getParameterObjList():
697 for parmConfObj in opConfObj.getParameterObjList():
699 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
698 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
700 continue
699 continue
701
700
702 kwargs[parmConfObj.name] = parmConfObj.getValue()
701 kwargs[parmConfObj.name] = parmConfObj.getValue()
703
702
704 #ini = time.time()
703 #ini = time.time()
705
704
706 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
705 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
707 sts = self.procUnitObj.call(opType = opConfObj.type,
706 sts = self.procUnitObj.call(opType = opConfObj.type,
708 opName = opConfObj.name,
707 opName = opConfObj.name,
709 opId = opConfObj.id,
708 opId = opConfObj.id,
710 )
709 )
711
710
712 # total_time = time.time() - ini
711 # total_time = time.time() - ini
713 #
712 #
714 # if total_time > 0.002:
713 # if total_time > 0.002:
715 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
714 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
716
715
717 is_ok = is_ok or sts
716 is_ok = is_ok or sts
718
717
719 return is_ok
718 return is_ok
720
719
721 def close(self):
720 def close(self):
722
721
723 for opConfObj in self.opConfObjList:
722 for opConfObj in self.opConfObjList:
724 if opConfObj.type == 'self':
723 if opConfObj.type == 'self':
725 continue
724 continue
726
725
727 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
726 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
728 opObj.close()
727 opObj.close()
729
728
730 self.procUnitObj.close()
729 self.procUnitObj.close()
731
730
732 return
731 return
733
732
734 class ReadUnitConf(ProcUnitConf):
733 class ReadUnitConf(ProcUnitConf):
735
734
736 path = None
735 path = None
737 startDate = None
736 startDate = None
738 endDate = None
737 endDate = None
739 startTime = None
738 startTime = None
740 endTime = None
739 endTime = None
741
740
742 ELEMENTNAME = 'ReadUnit'
741 ELEMENTNAME = 'ReadUnit'
743
742
744 def __init__(self):
743 def __init__(self):
745
744
746 self.id = None
745 self.id = None
747 self.datatype = None
746 self.datatype = None
748 self.name = None
747 self.name = None
749 self.inputId = None
748 self.inputId = None
750
749
751 self.parentId = None
750 self.parentId = None
752
751
753 self.opConfObjList = []
752 self.opConfObjList = []
754 self.opObjList = []
753 self.opObjList = []
755
754
756 def getElementName(self):
755 def getElementName(self):
757
756
758 return self.ELEMENTNAME
757 return self.ELEMENTNAME
759
758
760 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
759 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
760 endTime="", parentId=None, queue=None, server=None, **kwargs):
761
761
762 #Compatible with old signal chain version
762 #Compatible with old signal chain version
763 if datatype==None and name==None:
763 if datatype==None and name==None:
764 raise ValueError, "datatype or name should be defined"
764 raise ValueError, "datatype or name should be defined"
765
765
766 if name==None:
766 if name==None:
767 if 'Reader' in datatype:
767 if 'Reader' in datatype:
768 name = datatype
768 name = datatype
769 else:
769 else:
770 name = '%sReader' %(datatype)
770 name = '%sReader' %(datatype)
771
772 if datatype==None:
771 if datatype==None:
773 datatype = name.replace('Reader','')
772 datatype = name.replace('Reader','')
774
773
775 self.id = id
774 self.id = id
776 self.name = name
775 self.name = name
777 self.datatype = datatype
776 self.datatype = datatype
778
777 if path != '':
779 self.path = os.path.abspath(path)
778 self.path = os.path.abspath(path)
780 self.startDate = startDate
779 self.startDate = startDate
781 self.endDate = endDate
780 self.endDate = endDate
782 self.startTime = startTime
781 self.startTime = startTime
783 self.endTime = endTime
782 self.endTime = endTime
784
783
785 self.inputId = '0'
784 self.inputId = '0'
786 self.parentId = parentId
785 self.parentId = parentId
787 self.queue = queue
786 self.queue = queue
787 self.server = server
788 self.addRunOperation(**kwargs)
788 self.addRunOperation(**kwargs)
789
789
790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
791
791
792 #Compatible with old signal chain version
792 #Compatible with old signal chain version
793 if datatype==None and name==None:
793 if datatype==None and name==None:
794 raise ValueError, "datatype or name should be defined"
794 raise ValueError, "datatype or name should be defined"
795
795
796 if name==None:
796 if name==None:
797 if 'Reader' in datatype:
797 if 'Reader' in datatype:
798 name = datatype
798 name = datatype
799 else:
799 else:
800 name = '%sReader' %(datatype)
800 name = '%sReader' %(datatype)
801
801
802 if datatype==None:
802 if datatype==None:
803 datatype = name.replace('Reader','')
803 datatype = name.replace('Reader','')
804
804
805 self.datatype = datatype
805 self.datatype = datatype
806 self.name = name
806 self.name = name
807 self.path = path
807 self.path = path
808 self.startDate = startDate
808 self.startDate = startDate
809 self.endDate = endDate
809 self.endDate = endDate
810 self.startTime = startTime
810 self.startTime = startTime
811 self.endTime = endTime
811 self.endTime = endTime
812
812
813 self.inputId = '0'
813 self.inputId = '0'
814 self.parentId = parentId
814 self.parentId = parentId
815
815
816 self.updateRunOperation(**kwargs)
816 self.updateRunOperation(**kwargs)
817
817
818 def removeOperations(self):
818 def removeOperations(self):
819
819
820 for obj in self.opConfObjList:
820 for obj in self.opConfObjList:
821 del obj
821 del obj
822
822
823 self.opConfObjList = []
823 self.opConfObjList = []
824
824
825 def addRunOperation(self, **kwargs):
825 def addRunOperation(self, **kwargs):
826
826
827 opObj = self.addOperation(name = 'run', optype = 'self')
827 opObj = self.addOperation(name = 'run', optype = 'self')
828
828
829 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
829 if self.server is None:
830 opObj.addParameter(name='path' , value=self.path, format='str')
830 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
831 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
831 opObj.addParameter(name='path' , value=self.path, format='str')
832 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
832 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
833 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
833 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
834 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
834 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
835 opObj.addParameter(name='queue' , value=self.queue, format='obj')
835 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
836
836 opObj.addParameter(name='queue' , value=self.queue, format='obj')
837 for key, value in kwargs.items():
837 for key, value in kwargs.items():
838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
839 else:
840 opObj.addParameter(name='server' , value=self.server, format='str')
841
839
842
840 return opObj
843 return opObj
841
844
842 def updateRunOperation(self, **kwargs):
845 def updateRunOperation(self, **kwargs):
843
846
844 opObj = self.getOperationObj(name = 'run')
847 opObj = self.getOperationObj(name = 'run')
845 opObj.removeParameters()
848 opObj.removeParameters()
846
849
847 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
850 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
848 opObj.addParameter(name='path' , value=self.path, format='str')
851 opObj.addParameter(name='path' , value=self.path, format='str')
849 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
852 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
850 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
853 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
851 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
854 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
852 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
855 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
853
856
854 for key, value in kwargs.items():
857 for key, value in kwargs.items():
855 opObj.addParameter(name=key, value=value, format=type(value).__name__)
858 opObj.addParameter(name=key, value=value, format=type(value).__name__)
856
859
857 return opObj
860 return opObj
858
861
859 # def makeXml(self, projectElement):
862 # def makeXml(self, projectElement):
860 #
863 #
861 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
864 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
862 # procUnitElement.set('id', str(self.id))
865 # procUnitElement.set('id', str(self.id))
863 # procUnitElement.set('name', self.name)
866 # procUnitElement.set('name', self.name)
864 # procUnitElement.set('datatype', self.datatype)
867 # procUnitElement.set('datatype', self.datatype)
865 # procUnitElement.set('inputId', str(self.inputId))
868 # procUnitElement.set('inputId', str(self.inputId))
866 #
869 #
867 # for opConfObj in self.opConfObjList:
870 # for opConfObj in self.opConfObjList:
868 # opConfObj.makeXml(procUnitElement)
871 # opConfObj.makeXml(procUnitElement)
869
872
870 def readXml(self, upElement):
873 def readXml(self, upElement):
871
874
872 self.id = upElement.get('id')
875 self.id = upElement.get('id')
873 self.name = upElement.get('name')
876 self.name = upElement.get('name')
874 self.datatype = upElement.get('datatype')
877 self.datatype = upElement.get('datatype')
875 self.inputId = upElement.get('inputId')
878 self.inputId = upElement.get('inputId')
876
879
877 if self.ELEMENTNAME == "ReadUnit":
880 if self.ELEMENTNAME == "ReadUnit":
878 self.datatype = self.datatype.replace("Reader", "")
881 self.datatype = self.datatype.replace("Reader", "")
879
882
880 if self.inputId == 'None':
883 if self.inputId == 'None':
881 self.inputId = '0'
884 self.inputId = '0'
882
885
883 self.opConfObjList = []
886 self.opConfObjList = []
884
887
885 opElementList = upElement.iter(OperationConf().getElementName())
888 opElementList = upElement.iter(OperationConf().getElementName())
886
889
887 for opElement in opElementList:
890 for opElement in opElementList:
888 opConfObj = OperationConf()
891 opConfObj = OperationConf()
889 opConfObj.readXml(opElement)
892 opConfObj.readXml(opElement)
890 self.opConfObjList.append(opConfObj)
893 self.opConfObjList.append(opConfObj)
891
894
892 if opConfObj.name == 'run':
895 if opConfObj.name == 'run':
893 self.path = opConfObj.getParameterValue('path')
896 self.path = opConfObj.getParameterValue('path')
894 self.startDate = opConfObj.getParameterValue('startDate')
897 self.startDate = opConfObj.getParameterValue('startDate')
895 self.endDate = opConfObj.getParameterValue('endDate')
898 self.endDate = opConfObj.getParameterValue('endDate')
896 self.startTime = opConfObj.getParameterValue('startTime')
899 self.startTime = opConfObj.getParameterValue('startTime')
897 self.endTime = opConfObj.getParameterValue('endTime')
900 self.endTime = opConfObj.getParameterValue('endTime')
898
901
899 class Project():
902 class Project():
900
903
901 id = None
904 id = None
902 name = None
905 name = None
903 description = None
906 description = None
904 filename = None
907 filename = None
905
908
906 procUnitConfObjDict = None
909 procUnitConfObjDict = None
907
910
908 ELEMENTNAME = 'Project'
911 ELEMENTNAME = 'Project'
909
912
910 plotterQueue = None
913 plotterQueue = None
911
914
912 def __init__(self, plotter_queue=None):
915 def __init__(self, plotter_queue=None):
913
916
914 self.id = None
917 self.id = None
915 self.name = None
918 self.name = None
916 self.description = None
919 self.description = None
917
920
918 self.plotterQueue = plotter_queue
921 self.plotterQueue = plotter_queue
919
922
920 self.procUnitConfObjDict = {}
923 self.procUnitConfObjDict = {}
921
924
922 def __getNewId(self):
925 def __getNewId(self):
923
926
924 idList = self.procUnitConfObjDict.keys()
927 idList = self.procUnitConfObjDict.keys()
925
928
926 id = int(self.id)*10
929 id = int(self.id)*10
927
930
928 while True:
931 while True:
929 id += 1
932 id += 1
930
933
931 if str(id) in idList:
934 if str(id) in idList:
932 continue
935 continue
933
936
934 break
937 break
935
938
936 return str(id)
939 return str(id)
937
940
938 def getElementName(self):
941 def getElementName(self):
939
942
940 return self.ELEMENTNAME
943 return self.ELEMENTNAME
941
944
942 def getId(self):
945 def getId(self):
943
946
944 return self.id
947 return self.id
945
948
946 def updateId(self, new_id):
949 def updateId(self, new_id):
947
950
948 self.id = str(new_id)
951 self.id = str(new_id)
949
952
950 keyList = self.procUnitConfObjDict.keys()
953 keyList = self.procUnitConfObjDict.keys()
951 keyList.sort()
954 keyList.sort()
952
955
953 n = 1
956 n = 1
954 newProcUnitConfObjDict = {}
957 newProcUnitConfObjDict = {}
955
958
956 for procKey in keyList:
959 for procKey in keyList:
957
960
958 procUnitConfObj = self.procUnitConfObjDict[procKey]
961 procUnitConfObj = self.procUnitConfObjDict[procKey]
959 idProcUnit = str(int(self.id)*10 + n)
962 idProcUnit = str(int(self.id)*10 + n)
960 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
963 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
961
964
962 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
965 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
963 n += 1
966 n += 1
964
967
965 self.procUnitConfObjDict = newProcUnitConfObjDict
968 self.procUnitConfObjDict = newProcUnitConfObjDict
966
969
967 def setup(self, id, name, description):
970 def setup(self, id, name, description):
968
971
969 self.id = str(id)
972 self.id = str(id)
970 self.name = name
973 self.name = name
971 self.description = description
974 self.description = description
972
975
973 def update(self, name, description):
976 def update(self, name, description):
974
977
975 self.name = name
978 self.name = name
976 self.description = description
979 self.description = description
977
980
978 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
981 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
979
982
980 if id is None:
983 if id is None:
981 idReadUnit = self.__getNewId()
984 idReadUnit = self.__getNewId()
982 else:
985 else:
983 idReadUnit = str(id)
986 idReadUnit = str(id)
984
987
985 readUnitConfObj = ReadUnitConf()
988 readUnitConfObj = ReadUnitConf()
986 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
989 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
987
990
988 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
991 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
989
992
990 return readUnitConfObj
993 return readUnitConfObj
991
994
992 def addProcUnit(self, inputId='0', datatype=None, name=None):
995 def addProcUnit(self, inputId='0', datatype=None, name=None):
993
996
994 idProcUnit = self.__getNewId()
997 idProcUnit = self.__getNewId()
995
998
996 procUnitConfObj = ProcUnitConf()
999 procUnitConfObj = ProcUnitConf()
997 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1000 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
998
1001
999 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1002 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1000
1003
1001 return procUnitConfObj
1004 return procUnitConfObj
1002
1005
1003 def removeProcUnit(self, id):
1006 def removeProcUnit(self, id):
1004
1007
1005 if id in self.procUnitConfObjDict.keys():
1008 if id in self.procUnitConfObjDict.keys():
1006 self.procUnitConfObjDict.pop(id)
1009 self.procUnitConfObjDict.pop(id)
1007
1010
1008 def getReadUnitId(self):
1011 def getReadUnitId(self):
1009
1012
1010 readUnitConfObj = self.getReadUnitObj()
1013 readUnitConfObj = self.getReadUnitObj()
1011
1014
1012 return readUnitConfObj.id
1015 return readUnitConfObj.id
1013
1016
1014 def getReadUnitObj(self):
1017 def getReadUnitObj(self):
1015
1018
1016 for obj in self.procUnitConfObjDict.values():
1019 for obj in self.procUnitConfObjDict.values():
1017 if obj.getElementName() == "ReadUnit":
1020 if obj.getElementName() == "ReadUnit":
1018 return obj
1021 return obj
1019
1022
1020 return None
1023 return None
1021
1024
1022 def getProcUnitObj(self, id=None, name=None):
1025 def getProcUnitObj(self, id=None, name=None):
1023
1026
1024 if id != None:
1027 if id != None:
1025 return self.procUnitConfObjDict[id]
1028 return self.procUnitConfObjDict[id]
1026
1029
1027 if name != None:
1030 if name != None:
1028 return self.getProcUnitObjByName(name)
1031 return self.getProcUnitObjByName(name)
1029
1032
1030 return None
1033 return None
1031
1034
1032 def getProcUnitObjByName(self, name):
1035 def getProcUnitObjByName(self, name):
1033
1036
1034 for obj in self.procUnitConfObjDict.values():
1037 for obj in self.procUnitConfObjDict.values():
1035 if obj.name == name:
1038 if obj.name == name:
1036 return obj
1039 return obj
1037
1040
1038 return None
1041 return None
1039
1042
1040 def procUnitItems(self):
1043 def procUnitItems(self):
1041
1044
1042 return self.procUnitConfObjDict.items()
1045 return self.procUnitConfObjDict.items()
1043
1046
1044 def makeXml(self):
1047 def makeXml(self):
1045
1048
1046 projectElement = Element('Project')
1049 projectElement = Element('Project')
1047 projectElement.set('id', str(self.id))
1050 projectElement.set('id', str(self.id))
1048 projectElement.set('name', self.name)
1051 projectElement.set('name', self.name)
1049 projectElement.set('description', self.description)
1052 projectElement.set('description', self.description)
1050
1053
1051 for procUnitConfObj in self.procUnitConfObjDict.values():
1054 for procUnitConfObj in self.procUnitConfObjDict.values():
1052 procUnitConfObj.makeXml(projectElement)
1055 procUnitConfObj.makeXml(projectElement)
1053
1056
1054 self.projectElement = projectElement
1057 self.projectElement = projectElement
1055
1058
1056 def writeXml(self, filename=None):
1059 def writeXml(self, filename=None):
1057
1060
1058 if filename == None:
1061 if filename == None:
1059 if self.filename:
1062 if self.filename:
1060 filename = self.filename
1063 filename = self.filename
1061 else:
1064 else:
1062 filename = "schain.xml"
1065 filename = "schain.xml"
1063
1066
1064 if not filename:
1067 if not filename:
1065 print "filename has not been defined. Use setFilename(filename) for do it."
1068 print "filename has not been defined. Use setFilename(filename) for do it."
1066 return 0
1069 return 0
1067
1070
1068 abs_file = os.path.abspath(filename)
1071 abs_file = os.path.abspath(filename)
1069
1072
1070 if not os.access(os.path.dirname(abs_file), os.W_OK):
1073 if not os.access(os.path.dirname(abs_file), os.W_OK):
1071 print "No write permission on %s" %os.path.dirname(abs_file)
1074 print "No write permission on %s" %os.path.dirname(abs_file)
1072 return 0
1075 return 0
1073
1076
1074 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1077 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1075 print "File %s already exists and it could not be overwriten" %abs_file
1078 print "File %s already exists and it could not be overwriten" %abs_file
1076 return 0
1079 return 0
1077
1080
1078 self.makeXml()
1081 self.makeXml()
1079
1082
1080 ElementTree(self.projectElement).write(abs_file, method='xml')
1083 ElementTree(self.projectElement).write(abs_file, method='xml')
1081
1084
1082 self.filename = abs_file
1085 self.filename = abs_file
1083
1086
1084 return 1
1087 return 1
1085
1088
1086 def readXml(self, filename = None):
1089 def readXml(self, filename = None):
1087
1090
1088 if not filename:
1091 if not filename:
1089 print "filename is not defined"
1092 print "filename is not defined"
1090 return 0
1093 return 0
1091
1094
1092 abs_file = os.path.abspath(filename)
1095 abs_file = os.path.abspath(filename)
1093
1096
1094 if not os.path.isfile(abs_file):
1097 if not os.path.isfile(abs_file):
1095 print "%s file does not exist" %abs_file
1098 print "%s file does not exist" %abs_file
1096 return 0
1099 return 0
1097
1100
1098 self.projectElement = None
1101 self.projectElement = None
1099 self.procUnitConfObjDict = {}
1102 self.procUnitConfObjDict = {}
1100
1103
1101 try:
1104 try:
1102 self.projectElement = ElementTree().parse(abs_file)
1105 self.projectElement = ElementTree().parse(abs_file)
1103 except:
1106 except:
1104 print "Error reading %s, verify file format" %filename
1107 print "Error reading %s, verify file format" %filename
1105 return 0
1108 return 0
1106
1109
1107 self.project = self.projectElement.tag
1110 self.project = self.projectElement.tag
1108
1111
1109 self.id = self.projectElement.get('id')
1112 self.id = self.projectElement.get('id')
1110 self.name = self.projectElement.get('name')
1113 self.name = self.projectElement.get('name')
1111 self.description = self.projectElement.get('description')
1114 self.description = self.projectElement.get('description')
1112
1115
1113 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1116 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1114
1117
1115 for readUnitElement in readUnitElementList:
1118 for readUnitElement in readUnitElementList:
1116 readUnitConfObj = ReadUnitConf()
1119 readUnitConfObj = ReadUnitConf()
1117 readUnitConfObj.readXml(readUnitElement)
1120 readUnitConfObj.readXml(readUnitElement)
1118
1121
1119 if readUnitConfObj.parentId == None:
1122 if readUnitConfObj.parentId == None:
1120 readUnitConfObj.parentId = self.id
1123 readUnitConfObj.parentId = self.id
1121
1124
1122 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1125 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1123
1126
1124 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1127 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1125
1128
1126 for procUnitElement in procUnitElementList:
1129 for procUnitElement in procUnitElementList:
1127 procUnitConfObj = ProcUnitConf()
1130 procUnitConfObj = ProcUnitConf()
1128 procUnitConfObj.readXml(procUnitElement)
1131 procUnitConfObj.readXml(procUnitElement)
1129
1132
1130 if procUnitConfObj.parentId == None:
1133 if procUnitConfObj.parentId == None:
1131 procUnitConfObj.parentId = self.id
1134 procUnitConfObj.parentId = self.id
1132
1135
1133 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1136 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1134
1137
1135 self.filename = abs_file
1138 self.filename = abs_file
1136
1139
1137 return 1
1140 return 1
1138
1141
1139 def printattr(self):
1142 def printattr(self):
1140
1143
1141 print "Project[%s]: name = %s, description = %s" %(self.id,
1144 print "Project[%s]: name = %s, description = %s" %(self.id,
1142 self.name,
1145 self.name,
1143 self.description)
1146 self.description)
1144
1147
1145 for procUnitConfObj in self.procUnitConfObjDict.values():
1148 for procUnitConfObj in self.procUnitConfObjDict.values():
1146 procUnitConfObj.printattr()
1149 procUnitConfObj.printattr()
1147
1150
1148 def createObjects(self):
1151 def createObjects(self):
1149
1152
1150 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 for procUnitConfObj in self.procUnitConfObjDict.values():
1151 procUnitConfObj.createObjects(self.plotterQueue)
1154 procUnitConfObj.createObjects(self.plotterQueue)
1152
1155
1153 def __connect(self, objIN, thisObj):
1156 def __connect(self, objIN, thisObj):
1154
1157
1155 thisObj.setInput(objIN.getOutputObj())
1158 thisObj.setInput(objIN.getOutputObj())
1156
1159
1157 def connectObjects(self):
1160 def connectObjects(self):
1158
1161
1159 for thisPUConfObj in self.procUnitConfObjDict.values():
1162 for thisPUConfObj in self.procUnitConfObjDict.values():
1160
1163
1161 inputId = thisPUConfObj.getInputId()
1164 inputId = thisPUConfObj.getInputId()
1162
1165
1163 if int(inputId) == 0:
1166 if int(inputId) == 0:
1164 continue
1167 continue
1165
1168
1166 #Get input object
1169 #Get input object
1167 puConfINObj = self.procUnitConfObjDict[inputId]
1170 puConfINObj = self.procUnitConfObjDict[inputId]
1168 puObjIN = puConfINObj.getProcUnitObj()
1171 puObjIN = puConfINObj.getProcUnitObj()
1169
1172
1170 #Get current object
1173 #Get current object
1171 thisPUObj = thisPUConfObj.getProcUnitObj()
1174 thisPUObj = thisPUConfObj.getProcUnitObj()
1172
1175
1173 self.__connect(puObjIN, thisPUObj)
1176 self.__connect(puObjIN, thisPUObj)
1174
1177
1175 def __handleError(self, procUnitConfObj, send_email=True):
1178 def __handleError(self, procUnitConfObj, send_email=True):
1176
1179
1177 import socket
1180 import socket
1178
1181
1179 err = traceback.format_exception(sys.exc_info()[0],
1182 err = traceback.format_exception(sys.exc_info()[0],
1180 sys.exc_info()[1],
1183 sys.exc_info()[1],
1181 sys.exc_info()[2])
1184 sys.exc_info()[2])
1182
1185
1183 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1186 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1184 print "***** %s" %err[-1]
1187 print "***** %s" %err[-1]
1185
1188
1186 message = "".join(err)
1189 message = "".join(err)
1187
1190
1188 sys.stderr.write(message)
1191 sys.stderr.write(message)
1189
1192
1190 if not send_email:
1193 if not send_email:
1191 return
1194 return
1192
1195
1193 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1196 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1194
1197
1195 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1198 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1196 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1199 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1197 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1200 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1198 subtitle += "Configuration file: %s\n" %self.filename
1201 subtitle += "Configuration file: %s\n" %self.filename
1199 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1202 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1200
1203
1201 readUnitConfObj = self.getReadUnitObj()
1204 readUnitConfObj = self.getReadUnitObj()
1202 if readUnitConfObj:
1205 if readUnitConfObj:
1203 subtitle += "\nInput parameters:\n"
1206 subtitle += "\nInput parameters:\n"
1204 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1207 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1205 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1208 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1206 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1209 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1207 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1210 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1208 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1211 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1209 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1212 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1210
1213
1211 adminObj = schainpy.admin.SchainNotify()
1214 adminObj = schainpy.admin.SchainNotify()
1212 adminObj.sendAlert(message=message,
1215 adminObj.sendAlert(message=message,
1213 subject=subject,
1216 subject=subject,
1214 subtitle=subtitle,
1217 subtitle=subtitle,
1215 filename=self.filename)
1218 filename=self.filename)
1216
1219
1217 def isPaused(self):
1220 def isPaused(self):
1218 return 0
1221 return 0
1219
1222
1220 def isStopped(self):
1223 def isStopped(self):
1221 return 0
1224 return 0
1222
1225
1223 def runController(self):
1226 def runController(self):
1224 """
1227 """
1225 returns 0 when this process has been stopped, 1 otherwise
1228 returns 0 when this process has been stopped, 1 otherwise
1226 """
1229 """
1227
1230
1228 if self.isPaused():
1231 if self.isPaused():
1229 print "Process suspended"
1232 print "Process suspended"
1230
1233
1231 while True:
1234 while True:
1232 sleep(0.1)
1235 sleep(0.1)
1233
1236
1234 if not self.isPaused():
1237 if not self.isPaused():
1235 break
1238 break
1236
1239
1237 if self.isStopped():
1240 if self.isStopped():
1238 break
1241 break
1239
1242
1240 print "Process reinitialized"
1243 print "Process reinitialized"
1241
1244
1242 if self.isStopped():
1245 if self.isStopped():
1243 print "Process stopped"
1246 print "Process stopped"
1244 return 0
1247 return 0
1245
1248
1246 return 1
1249 return 1
1247
1250
1248 def setFilename(self, filename):
1251 def setFilename(self, filename):
1249
1252
1250 self.filename = filename
1253 self.filename = filename
1251
1254
1252 def setPlotterQueue(self, plotter_queue):
1255 def setPlotterQueue(self, plotter_queue):
1253
1256
1254 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1257 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1255
1258
1256 def getPlotterQueue(self):
1259 def getPlotterQueue(self):
1257
1260
1258 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1259
1262
1260 def useExternalPlotter(self):
1263 def useExternalPlotter(self):
1261
1264
1262 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1263
1266
1264 def run(self):
1267 def run(self):
1265
1268
1266 print
1269 print
1267 print "*"*60
1270 print "*"*60
1268 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1271 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1269 print "*"*60
1272 print "*"*60
1270 print
1273 print
1271
1274
1272 keyList = self.procUnitConfObjDict.keys()
1275 keyList = self.procUnitConfObjDict.keys()
1273 keyList.sort()
1276 keyList.sort()
1274
1277
1275 while(True):
1278 while(True):
1276
1279
1277 is_ok = False
1280 is_ok = False
1278
1281
1279 for procKey in keyList:
1282 for procKey in keyList:
1280 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1283 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1281
1284
1282 procUnitConfObj = self.procUnitConfObjDict[procKey]
1285 procUnitConfObj = self.procUnitConfObjDict[procKey]
1283
1286
1284 try:
1287 try:
1285 sts = procUnitConfObj.run()
1288 sts = procUnitConfObj.run()
1286 is_ok = is_ok or sts
1289 is_ok = is_ok or sts
1287 except KeyboardInterrupt:
1290 except KeyboardInterrupt:
1288 is_ok = False
1291 is_ok = False
1289 break
1292 break
1290 except ValueError, e:
1293 except ValueError, e:
1291 sleep(0.5)
1294 sleep(0.5)
1292 self.__handleError(procUnitConfObj, send_email=True)
1295 self.__handleError(procUnitConfObj, send_email=True)
1293 is_ok = False
1296 is_ok = False
1294 break
1297 break
1295 except:
1298 except:
1296 sleep(0.5)
1299 sleep(0.5)
1297 self.__handleError(procUnitConfObj)
1300 self.__handleError(procUnitConfObj)
1298 is_ok = False
1301 is_ok = False
1299 break
1302 break
1300
1303
1301 #If every process unit finished so end process
1304 #If every process unit finished so end process
1302 if not(is_ok):
1305 if not(is_ok):
1303 # print "Every process unit have finished"
1306 # print "Every process unit have finished"
1304 break
1307 break
1305
1308
1306 if not self.runController():
1309 if not self.runController():
1307 break
1310 break
1308
1311
1309 #Closing every process
1312 #Closing every process
1310 for procKey in keyList:
1313 for procKey in keyList:
1311 procUnitConfObj = self.procUnitConfObjDict[procKey]
1314 procUnitConfObj = self.procUnitConfObjDict[procKey]
1312 procUnitConfObj.close()
1315 procUnitConfObj.close()
1313
1316
1314 print "Process finished"
1317 print "Process finished"
1315
1318
1316 def start(self, filename=None):
1319 def start(self, filename=None):
1317
1320
1318 self.writeXml(filename)
1321 self.writeXml(filename)
1319 self.createObjects()
1322 self.createObjects()
1320 self.connectObjects()
1323 self.connectObjects()
1321 self.run()
1324 self.run()
Show file before | Show file after | Hide comments
@@ -1,761 +1,851
1 '''
1 '''
2
2
3 $Author: murco $
3 $Author: murco $
4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
4 $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $
5 '''
5 '''
6 import sys
6 import sys
7 import numpy
7 import numpy
8 import copy
8 import copy
9 import datetime
9 import datetime
10
10
11 SPEED_OF_LIGHT = 299792458
11 SPEED_OF_LIGHT = 299792458
12 SPEED_OF_LIGHT = 3e8
12 SPEED_OF_LIGHT = 3e8
13
13
14 BASIC_STRUCTURE = numpy.dtype([
14 BASIC_STRUCTURE = numpy.dtype([
15 ('nSize','<u4'),
15 ('nSize','<u4'),
16 ('nVersion','<u2'),
16 ('nVersion','<u2'),
17 ('nDataBlockId','<u4'),
17 ('nDataBlockId','<u4'),
18 ('nUtime','<u4'),
18 ('nUtime','<u4'),
19 ('nMilsec','<u2'),
19 ('nMilsec','<u2'),
20 ('nTimezone','<i2'),
20 ('nTimezone','<i2'),
21 ('nDstflag','<i2'),
21 ('nDstflag','<i2'),
22 ('nErrorCount','<u4')
22 ('nErrorCount','<u4')
23 ])
23 ])
24
24
25 SYSTEM_STRUCTURE = numpy.dtype([
25 SYSTEM_STRUCTURE = numpy.dtype([
26 ('nSize','<u4'),
26 ('nSize','<u4'),
27 ('nNumSamples','<u4'),
27 ('nNumSamples','<u4'),
28 ('nNumProfiles','<u4'),
28 ('nNumProfiles','<u4'),
29 ('nNumChannels','<u4'),
29 ('nNumChannels','<u4'),
30 ('nADCResolution','<u4'),
30 ('nADCResolution','<u4'),
31 ('nPCDIOBusWidth','<u4'),
31 ('nPCDIOBusWidth','<u4'),
32 ])
32 ])
33
33
34 RADAR_STRUCTURE = numpy.dtype([
34 RADAR_STRUCTURE = numpy.dtype([
35 ('nSize','<u4'),
35 ('nSize','<u4'),
36 ('nExpType','<u4'),
36 ('nExpType','<u4'),
37 ('nNTx','<u4'),
37 ('nNTx','<u4'),
38 ('fIpp','<f4'),
38 ('fIpp','<f4'),
39 ('fTxA','<f4'),
39 ('fTxA','<f4'),
40 ('fTxB','<f4'),
40 ('fTxB','<f4'),
41 ('nNumWindows','<u4'),
41 ('nNumWindows','<u4'),
42 ('nNumTaus','<u4'),
42 ('nNumTaus','<u4'),
43 ('nCodeType','<u4'),
43 ('nCodeType','<u4'),
44 ('nLine6Function','<u4'),
44 ('nLine6Function','<u4'),
45 ('nLine5Function','<u4'),
45 ('nLine5Function','<u4'),
46 ('fClock','<f4'),
46 ('fClock','<f4'),
47 ('nPrePulseBefore','<u4'),
47 ('nPrePulseBefore','<u4'),
48 ('nPrePulseAfter','<u4'),
48 ('nPrePulseAfter','<u4'),
49 ('sRangeIPP','<a20'),
49 ('sRangeIPP','<a20'),
50 ('sRangeTxA','<a20'),
50 ('sRangeTxA','<a20'),
51 ('sRangeTxB','<a20'),
51 ('sRangeTxB','<a20'),
52 ])
52 ])
53
53
54 SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
54 SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')])
55
55
56
56
57 PROCESSING_STRUCTURE = numpy.dtype([
57 PROCESSING_STRUCTURE = numpy.dtype([
58 ('nSize','<u4'),
58 ('nSize','<u4'),
59 ('nDataType','<u4'),
59 ('nDataType','<u4'),
60 ('nSizeOfDataBlock','<u4'),
60 ('nSizeOfDataBlock','<u4'),
61 ('nProfilesperBlock','<u4'),
61 ('nProfilesperBlock','<u4'),
62 ('nDataBlocksperFile','<u4'),
62 ('nDataBlocksperFile','<u4'),
63 ('nNumWindows','<u4'),
63 ('nNumWindows','<u4'),
64 ('nProcessFlags','<u4'),
64 ('nProcessFlags','<u4'),
65 ('nCoherentIntegrations','<u4'),
65 ('nCoherentIntegrations','<u4'),
66 ('nIncoherentIntegrations','<u4'),
66 ('nIncoherentIntegrations','<u4'),
67 ('nTotalSpectra','<u4')
67 ('nTotalSpectra','<u4')
68 ])
68 ])
69
69
70 class Header(object):
70 class Header(object):
71
71
72 def __init__(self):
72 def __init__(self):
73 raise NotImplementedError
73 raise NotImplementedError
74
74
75 def copy(self):
75 def copy(self):
76 return copy.deepcopy(self)
76 return copy.deepcopy(self)
77
77
78 def read(self):
78 def read(self):
79
79
80 raise NotImplementedError
80 raise NotImplementedError
81
81
82 def write(self):
82 def write(self):
83
83
84 raise NotImplementedError
84 raise NotImplementedError
85
85
86 def printInfo(self):
86 def printInfo(self):
87
87
88 message = "#"*50 + "\n"
88 message = "#"*50 + "\n"
89 message += self.__class__.__name__.upper() + "\n"
89 message += self.__class__.__name__.upper() + "\n"
90 message += "#"*50 + "\n"
90 message += "#"*50 + "\n"
91
91
92 keyList = self.__dict__.keys()
92 keyList = self.__dict__.keys()
93 keyList.sort()
93 keyList.sort()
94
94
95 for key in keyList:
95 for key in keyList:
96 message += "%s = %s" %(key, self.__dict__[key]) + "\n"
96 message += "%s = %s" %(key, self.__dict__[key]) + "\n"
97
97
98 if "size" not in keyList:
98 if "size" not in keyList:
99 attr = getattr(self, "size")
99 attr = getattr(self, "size")
100
100
101 if attr:
101 if attr:
102 message += "%s = %s" %("size", attr) + "\n"
102 message += "%s = %s" %("size", attr) + "\n"
103
103
104 print message
104 print message
105
105
106 class BasicHeader(Header):
106 class BasicHeader(Header):
107
107
108 size = None
108 size = None
109 version = None
109 version = None
110 dataBlock = None
110 dataBlock = None
111 utc = None
111 utc = None
112 ltc = None
112 ltc = None
113 miliSecond = None
113 miliSecond = None
114 timeZone = None
114 timeZone = None
115 dstFlag = None
115 dstFlag = None
116 errorCount = None
116 errorCount = None
117 datatime = None
117 datatime = None
118
119 __LOCALTIME = None
118 __LOCALTIME = None
120
119
121 def __init__(self, useLocalTime=True):
120 def __init__(self, useLocalTime=True):
122
121
123 self.size = 24
122 self.size = 24
124 self.version = 0
123 self.version = 0
125 self.dataBlock = 0
124 self.dataBlock = 0
126 self.utc = 0
125 self.utc = 0
127 self.miliSecond = 0
126 self.miliSecond = 0
128 self.timeZone = 0
127 self.timeZone = 0
129 self.dstFlag = 0
128 self.dstFlag = 0
130 self.errorCount = 0
129 self.errorCount = 0
131
130
132 self.useLocalTime = useLocalTime
131 self.useLocalTime = useLocalTime
133
132
134 def read(self, fp):
133 def read(self, fp):
135
134
135 self.length = 0
136 try:
136 try:
137 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
137 if hasattr(fp, 'read'):
138
138 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
139 else:
140 header = numpy.fromstring(fp, BASIC_STRUCTURE,1)
139 except Exception, e:
141 except Exception, e:
140 print "BasicHeader: "
142 print "BasicHeader: "
141 print e
143 print e
142 return 0
144 return 0
143
145
144 self.size = int(header['nSize'][0])
146 self.size = int(header['nSize'][0])
145 self.version = int(header['nVersion'][0])
147 self.version = int(header['nVersion'][0])
146 self.dataBlock = int(header['nDataBlockId'][0])
148 self.dataBlock = int(header['nDataBlockId'][0])
147 self.utc = int(header['nUtime'][0])
149 self.utc = int(header['nUtime'][0])
148 self.miliSecond = int(header['nMilsec'][0])
150 self.miliSecond = int(header['nMilsec'][0])
149 self.timeZone = int(header['nTimezone'][0])
151 self.timeZone = int(header['nTimezone'][0])
150 self.dstFlag = int(header['nDstflag'][0])
152 self.dstFlag = int(header['nDstflag'][0])
151 self.errorCount = int(header['nErrorCount'][0])
153 self.errorCount = int(header['nErrorCount'][0])
152
154
153 if self.size < 24:
155 if self.size < 24:
154 return 0
156 return 0
155
157
158 self.length = header.nbytes
156 return 1
159 return 1
157
160
158 def write(self, fp):
161 def write(self, fp):
159
162
160 headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
163 headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount)
161 header = numpy.array(headerTuple, BASIC_STRUCTURE)
164 header = numpy.array(headerTuple, BASIC_STRUCTURE)
162 header.tofile(fp)
165 header.tofile(fp)
163
166
164 return 1
167 return 1
165
168
166 def get_ltc(self):
169 def get_ltc(self):
167
170
168 return self.utc - self.timeZone*60
171 return self.utc - self.timeZone*60
169
172
170 def set_ltc(self, value):
173 def set_ltc(self, value):
171
174
172 self.utc = value + self.timeZone*60
175 self.utc = value + self.timeZone*60
173
176
174 def get_datatime(self):
177 def get_datatime(self):
175
178
176 return datetime.datetime.utcfromtimestamp(self.ltc)
179 return datetime.datetime.utcfromtimestamp(self.ltc)
177
180
178 ltc = property(get_ltc, set_ltc)
181 ltc = property(get_ltc, set_ltc)
179 datatime = property(get_datatime)
182 datatime = property(get_datatime)
180
183
181 class SystemHeader(Header):
184 class SystemHeader(Header):
182
185
183 size = None
186 size = None
184 nSamples = None
187 nSamples = None
185 nProfiles = None
188 nProfiles = None
186 nChannels = None
189 nChannels = None
187 adcResolution = None
190 adcResolution = None
188 pciDioBusWidth = None
191 pciDioBusWidth = None
189
192
190 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWith=0):
193 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWith=0):
191
194
192 self.size = 24
195 self.size = 24
193 self.nSamples = nSamples
196 self.nSamples = nSamples
194 self.nProfiles = nProfiles
197 self.nProfiles = nProfiles
195 self.nChannels = nChannels
198 self.nChannels = nChannels
196 self.adcResolution = adcResolution
199 self.adcResolution = adcResolution
197 self.pciDioBusWidth = pciDioBusWith
200 self.pciDioBusWidth = pciDioBusWith
198
201
199 def read(self, fp):
202 def read(self, fp):
200
203 self.length = 0
201 startFp = fp.tell()
204 try:
202
205 startFp = fp.tell()
206 except Exception, e:
207 startFp = None
208 pass
209
203 try:
210 try:
204 header = numpy.fromfile(fp,SYSTEM_STRUCTURE,1)
211 if hasattr(fp, 'read'):
212 header = numpy.fromfile(fp, SYSTEM_STRUCTURE,1)
213 else:
214 header = numpy.fromstring(fp, SYSTEM_STRUCTURE,1)
205 except Exception, e:
215 except Exception, e:
206 print "System Header: " + e
216 print "System Header: " + str(e)
207 return 0
217 return 0
208
218
209 self.size = header['nSize'][0]
219 self.size = header['nSize'][0]
210 self.nSamples = header['nNumSamples'][0]
220 self.nSamples = header['nNumSamples'][0]
211 self.nProfiles = header['nNumProfiles'][0]
221 self.nProfiles = header['nNumProfiles'][0]
212 self.nChannels = header['nNumChannels'][0]
222 self.nChannels = header['nNumChannels'][0]
213 self.adcResolution = header['nADCResolution'][0]
223 self.adcResolution = header['nADCResolution'][0]
214 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
224 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
215
225
216 endFp = self.size + startFp
217
226
218 if fp.tell() > endFp:
227 if startFp is not None:
219 sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name)
228 endFp = self.size + startFp
220 return 0
221
229
222 if fp.tell() < endFp:
230 if fp.tell() > endFp:
223 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
231 sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name)
224 return 0
232 return 0
233
234 if fp.tell() < endFp:
235 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
236 return 0
225
237
238 self.length = header.nbytes
226 return 1
239 return 1
227
240
228 def write(self, fp):
241 def write(self, fp):
229
242
230 headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
243 headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth)
231 header = numpy.array(headerTuple,SYSTEM_STRUCTURE)
244 header = numpy.array(headerTuple,SYSTEM_STRUCTURE)
232 header.tofile(fp)
245 header.tofile(fp)
233
246
234 return 1
247 return 1
235
248
236 class RadarControllerHeader(Header):
249 class RadarControllerHeader(Header):
237
250
238 expType = None
251 expType = None
239 nTx = None
252 nTx = None
240 ipp = None
253 ipp = None
241 txA = None
254 txA = None
242 txB = None
255 txB = None
243 nWindows = None
256 nWindows = None
244 numTaus = None
257 numTaus = None
245 codeType = None
258 codeType = None
246 line6Function = None
259 line6Function = None
247 line5Function = None
260 line5Function = None
248 fClock = None
261 fClock = None
249 prePulseBefore = None
262 prePulseBefore = None
250 prePulserAfter = None
263 prePulserAfter = None
251 rangeIpp = None
264 rangeIpp = None
252 rangeTxA = None
265 rangeTxA = None
253 rangeTxB = None
266 rangeTxB = None
254
267
255 __size = None
268 __size = None
256
269
257 def __init__(self, expType=2, nTx=1,
270 def __init__(self, expType=2, nTx=1,
258 ippKm=None, txA=0, txB=0,
271 ippKm=None, txA=0, txB=0,
259 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
272 nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
260 numTaus=0, line6Function=0, line5Function=0, fClock=None,
273 numTaus=0, line6Function=0, line5Function=0, fClock=None,
261 prePulseBefore=0, prePulseAfter=0,
274 prePulseBefore=0, prePulseAfter=0,
262 codeType=0, nCode=0, nBaud=0, code=None,
275 codeType=0, nCode=0, nBaud=0, code=None,
263 flip1=0, flip2=0):
276 flip1=0, flip2=0):
264
277
265 # self.size = 116
278 # self.size = 116
266 self.expType = expType
279 self.expType = expType
267 self.nTx = nTx
280 self.nTx = nTx
268 self.ipp = ippKm
281 self.ipp = ippKm
269 self.txA = txA
282 self.txA = txA
270 self.txB = txB
283 self.txB = txB
271 self.rangeIpp = ippKm
284 self.rangeIpp = ippKm
272 self.rangeTxA = txA
285 self.rangeTxA = txA
273 self.rangeTxB = txB
286 self.rangeTxB = txB
274
287
275 self.nWindows = nWindows
288 self.nWindows = nWindows
276 self.numTaus = numTaus
289 self.numTaus = numTaus
277 self.codeType = codeType
290 self.codeType = codeType
278 self.line6Function = line6Function
291 self.line6Function = line6Function
279 self.line5Function = line5Function
292 self.line5Function = line5Function
280 self.fClock = fClock
293 self.fClock = fClock
281 self.prePulseBefore = prePulseBefore
294 self.prePulseBefore = prePulseBefore
282 self.prePulserAfter = prePulseAfter
295 self.prePulserAfter = prePulseAfter
283
296
284 self.nHeights = nHeights
297 self.nHeights = nHeights
285 self.firstHeight = firstHeight
298 self.firstHeight = firstHeight
286 self.deltaHeight = deltaHeight
299 self.deltaHeight = deltaHeight
287 self.samplesWin = nHeights
300 self.samplesWin = nHeights
288
301
289 self.nCode = nCode
302 self.nCode = nCode
290 self.nBaud = nBaud
303 self.nBaud = nBaud
291 self.code = code
304 self.code = code
292 self.flip1 = flip1
305 self.flip1 = flip1
293 self.flip2 = flip2
306 self.flip2 = flip2
294
307
295 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
308 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
296 # self.dynamic = numpy.array([],numpy.dtype('byte'))
309 # self.dynamic = numpy.array([],numpy.dtype('byte'))
297
310
298 if self.fClock is None and self.deltaHeight is not None:
311 if self.fClock is None and self.deltaHeight is not None:
299 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
312 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
300
313
301 def read(self, fp):
314 def read(self, fp):
302
315 self.length = 0
303
316 try:
304 startFp = fp.tell()
317 startFp = fp.tell()
318 except Exception, e:
319 startFp = None
320 pass
321
305 try:
322 try:
306 header = numpy.fromfile(fp,RADAR_STRUCTURE,1)
323 if hasattr(fp, 'read'):
324 header = numpy.fromfile(fp, RADAR_STRUCTURE,1)
325 else:
326 header = numpy.fromstring(fp, RADAR_STRUCTURE,1)
327 self.length += header.nbytes
307 except Exception, e:
328 except Exception, e:
308 print "RadarControllerHeader: " + e
329 print "RadarControllerHeader: " + str(e)
309 return 0
330 return 0
310
331
311 size = int(header['nSize'][0])
332 size = int(header['nSize'][0])
312 self.expType = int(header['nExpType'][0])
333 self.expType = int(header['nExpType'][0])
313 self.nTx = int(header['nNTx'][0])
334 self.nTx = int(header['nNTx'][0])
314 self.ipp = float(header['fIpp'][0])
335 self.ipp = float(header['fIpp'][0])
315 self.txA = float(header['fTxA'][0])
336 self.txA = float(header['fTxA'][0])
316 self.txB = float(header['fTxB'][0])
337 self.txB = float(header['fTxB'][0])
317 self.nWindows = int(header['nNumWindows'][0])
338 self.nWindows = int(header['nNumWindows'][0])
318 self.numTaus = int(header['nNumTaus'][0])
339 self.numTaus = int(header['nNumTaus'][0])
319 self.codeType = int(header['nCodeType'][0])
340 self.codeType = int(header['nCodeType'][0])
320 self.line6Function = int(header['nLine6Function'][0])
341 self.line6Function = int(header['nLine6Function'][0])
321 self.line5Function = int(header['nLine5Function'][0])
342 self.line5Function = int(header['nLine5Function'][0])
322 self.fClock = float(header['fClock'][0])
343 self.fClock = float(header['fClock'][0])
323 self.prePulseBefore = int(header['nPrePulseBefore'][0])
344 self.prePulseBefore = int(header['nPrePulseBefore'][0])
324 self.prePulserAfter = int(header['nPrePulseAfter'][0])
345 self.prePulserAfter = int(header['nPrePulseAfter'][0])
325 self.rangeIpp = header['sRangeIPP'][0]
346 self.rangeIpp = header['sRangeIPP'][0]
326 self.rangeTxA = header['sRangeTxA'][0]
347 self.rangeTxA = header['sRangeTxA'][0]
327 self.rangeTxB = header['sRangeTxB'][0]
348 self.rangeTxB = header['sRangeTxB'][0]
328
349
329 samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
350 try:
330
351 if hasattr(fp, 'read'):
352 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
353 else:
354 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
355 self.length += samplingWindow.nbytes
356 except Exception, e:
357 print "RadarControllerHeader: " + str(e)
358 return 0
331 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
359 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
332 self.firstHeight = samplingWindow['h0']
360 self.firstHeight = samplingWindow['h0']
333 self.deltaHeight = samplingWindow['dh']
361 self.deltaHeight = samplingWindow['dh']
334 self.samplesWin = samplingWindow['nsa']
362 self.samplesWin = samplingWindow['nsa']
363
364
365
366 try:
367 if hasattr(fp, 'read'):
368 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
369 else:
370 self.Taus = numpy.fromstring(fp[self.length:], '<f4', self.numTaus)
371 self.length += self.Taus.nbytes
372 except Exception, e:
373 print "RadarControllerHeader: " + str(e)
374 return 0
375
335
376
336 self.Taus = numpy.fromfile(fp,'<f4',self.numTaus)
337
377
338 self.code_size = 0
378 self.code_size = 0
339 if self.codeType != 0:
379 if self.codeType != 0:
340 self.nCode = int(numpy.fromfile(fp,'<u4',1))
341 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
342
380
381 try:
382 if hasattr(fp, 'read'):
383 self.nCode = numpy.fromfile(fp, '<u4', 1)
384 self.length += self.nCode.nbytes
385 self.nBaud = numpy.fromfile(fp, '<u4', 1)
386 self.length += self.nBaud.nbytes
387 else:
388 self.nCode = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
389 self.length += self.nCode.nbytes
390 self.nBaud = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
391 self.length += self.nBaud.nbytes
392 except Exception, e:
393 print "RadarControllerHeader: " + str(e)
394 return 0
343 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
395 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
396
344 for ic in range(self.nCode):
397 for ic in range(self.nCode):
345 temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.)))
398 try:
399 if hasattr(fp, 'read'):
400 temp = numpy.fromfile(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
401 else:
402 temp = numpy.fromstring(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
403 self.length += temp.nbytes
404 except Exception, e:
405 print "RadarControllerHeader: " + str(e)
406 return 0
407
346 for ib in range(self.nBaud-1,-1,-1):
408 for ib in range(self.nBaud-1,-1,-1):
347 code[ic,ib] = temp[ib/32]%2
409 code[ic,ib] = temp[ib/32]%2
348 temp[ib/32] = temp[ib/32]/2
410 temp[ib/32] = temp[ib/32]/2
349
411
350 self.code = 2.0*code - 1.0
412 self.code = 2.0*code - 1.0
351 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
413 self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4
352
414
353 # if self.line5Function == RCfunction.FLIP:
415 # if self.line5Function == RCfunction.FLIP:
354 # self.flip1 = numpy.fromfile(fp,'<u4',1)
416 # self.flip1 = numpy.fromfile(fp,'<u4',1)
355 #
417 #
356 # if self.line6Function == RCfunction.FLIP:
418 # if self.line6Function == RCfunction.FLIP:
357 # self.flip2 = numpy.fromfile(fp,'<u4',1)
419 # self.flip2 = numpy.fromfile(fp,'<u4',1)
358
420 if startFp is not None:
359 endFp = size + startFp
421 endFp = size + startFp
360
361 if fp.tell() != endFp:
362 # fp.seek(endFp)
363 print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" %(fp.name, fp.tell()-startFp, size)
364 # return 0
365
366 if fp.tell() > endFp:
367 sys.stderr.write("Warning %s: Size value read from Radar Controller header is lower than it has to be\n" %fp.name)
368 # return 0
369
422
370 if fp.tell() < endFp:
423 if fp.tell() != endFp:
371 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
424 # fp.seek(endFp)
425 print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" %(fp.name, fp.tell()-startFp, size)
426 # return 0
427
428 if fp.tell() > endFp:
429 sys.stderr.write("Warning %s: Size value read from Radar Controller header is lower than it has to be\n" %fp.name)
430 # return 0
431
432 if fp.tell() < endFp:
433 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
372
434
435
373 return 1
436 return 1
374
437
375 def write(self, fp):
438 def write(self, fp):
376
439
377 headerTuple = (self.size,
440 headerTuple = (self.size,
378 self.expType,
441 self.expType,
379 self.nTx,
442 self.nTx,
380 self.ipp,
443 self.ipp,
381 self.txA,
444 self.txA,
382 self.txB,
445 self.txB,
383 self.nWindows,
446 self.nWindows,
384 self.numTaus,
447 self.numTaus,
385 self.codeType,
448 self.codeType,
386 self.line6Function,
449 self.line6Function,
387 self.line5Function,
450 self.line5Function,
388 self.fClock,
451 self.fClock,
389 self.prePulseBefore,
452 self.prePulseBefore,
390 self.prePulserAfter,
453 self.prePulserAfter,
391 self.rangeIpp,
454 self.rangeIpp,
392 self.rangeTxA,
455 self.rangeTxA,
393 self.rangeTxB)
456 self.rangeTxB)
394
457
395 header = numpy.array(headerTuple,RADAR_STRUCTURE)
458 header = numpy.array(headerTuple,RADAR_STRUCTURE)
396 header.tofile(fp)
459 header.tofile(fp)
397
460
398 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
461 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
399 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
462 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
400 samplingWindow.tofile(fp)
463 samplingWindow.tofile(fp)
401
464
402 if self.numTaus > 0:
465 if self.numTaus > 0:
403 self.Taus.tofile(fp)
466 self.Taus.tofile(fp)
404
467
405 if self.codeType !=0:
468 if self.codeType !=0:
406 nCode = numpy.array(self.nCode, '<u4')
469 nCode = numpy.array(self.nCode, '<u4')
407 nCode.tofile(fp)
470 nCode.tofile(fp)
408 nBaud = numpy.array(self.nBaud, '<u4')
471 nBaud = numpy.array(self.nBaud, '<u4')
409 nBaud.tofile(fp)
472 nBaud.tofile(fp)
410 code1 = (self.code + 1.0)/2.
473 code1 = (self.code + 1.0)/2.
411
474
412 for ic in range(self.nCode):
475 for ic in range(self.nCode):
413 tempx = numpy.zeros(numpy.ceil(self.nBaud/32.))
476 tempx = numpy.zeros(numpy.ceil(self.nBaud/32.))
414 start = 0
477 start = 0
415 end = 32
478 end = 32
416 for i in range(len(tempx)):
479 for i in range(len(tempx)):
417 code_selected = code1[ic,start:end]
480 code_selected = code1[ic,start:end]
418 for j in range(len(code_selected)-1,-1,-1):
481 for j in range(len(code_selected)-1,-1,-1):
419 if code_selected[j] == 1:
482 if code_selected[j] == 1:
420 tempx[i] = tempx[i] + 2**(len(code_selected)-1-j)
483 tempx[i] = tempx[i] + 2**(len(code_selected)-1-j)
421 start = start + 32
484 start = start + 32
422 end = end + 32
485 end = end + 32
423
486
424 tempx = tempx.astype('u4')
487 tempx = tempx.astype('u4')
425 tempx.tofile(fp)
488 tempx.tofile(fp)
426
489
427 # if self.line5Function == RCfunction.FLIP:
490 # if self.line5Function == RCfunction.FLIP:
428 # self.flip1.tofile(fp)
491 # self.flip1.tofile(fp)
429 #
492 #
430 # if self.line6Function == RCfunction.FLIP:
493 # if self.line6Function == RCfunction.FLIP:
431 # self.flip2.tofile(fp)
494 # self.flip2.tofile(fp)
432
495
433 return 1
496 return 1
434
497
435 def get_ippSeconds(self):
498 def get_ippSeconds(self):
436 '''
499 '''
437 '''
500 '''
438 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
501 ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT
439
502
440 return ippSeconds
503 return ippSeconds
441
504
442 def set_ippSeconds(self, ippSeconds):
505 def set_ippSeconds(self, ippSeconds):
443 '''
506 '''
444 '''
507 '''
445
508
446 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000)
509 self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000)
447
510
448 return
511 return
449
512
450 def get_size(self):
513 def get_size(self):
451
514
452 self.__size = 116 + 12*self.nWindows + 4*self.numTaus
515 self.__size = 116 + 12*self.nWindows + 4*self.numTaus
453
516
454 if self.codeType != 0:
517 if self.codeType != 0:
455 self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
518 self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
456
519
457 return self.__size
520 return self.__size
458
521
459 def set_size(self, value):
522 def set_size(self, value):
460
523
461 raise IOError, "size is a property and it cannot be set, just read"
524 raise IOError, "size is a property and it cannot be set, just read"
462
525
463 return
526 return
464
527
465 ippSeconds = property(get_ippSeconds, set_ippSeconds)
528 ippSeconds = property(get_ippSeconds, set_ippSeconds)
466 size = property(get_size, set_size)
529 size = property(get_size, set_size)
467
530
468 class ProcessingHeader(Header):
531 class ProcessingHeader(Header):
469
532
470 # size = None
533 # size = None
471 dtype = None
534 dtype = None
472 blockSize = None
535 blockSize = None
473 profilesPerBlock = None
536 profilesPerBlock = None
474 dataBlocksPerFile = None
537 dataBlocksPerFile = None
475 nWindows = None
538 nWindows = None
476 processFlags = None
539 processFlags = None
477 nCohInt = None
540 nCohInt = None
478 nIncohInt = None
541 nIncohInt = None
479 totalSpectra = None
542 totalSpectra = None
480
543
481 flag_dc = None
544 flag_dc = None
482 flag_cspc = None
545 flag_cspc = None
483
546
484 def __init__(self):
547 def __init__(self):
485
548
486 # self.size = 0
549 # self.size = 0
487 self.dtype = 0
550 self.dtype = 0
488 self.blockSize = 0
551 self.blockSize = 0
489 self.profilesPerBlock = 0
552 self.profilesPerBlock = 0
490 self.dataBlocksPerFile = 0
553 self.dataBlocksPerFile = 0
491 self.nWindows = 0
554 self.nWindows = 0
492 self.processFlags = 0
555 self.processFlags = 0
493 self.nCohInt = 0
556 self.nCohInt = 0
494 self.nIncohInt = 0
557 self.nIncohInt = 0
495 self.totalSpectra = 0
558 self.totalSpectra = 0
496
559
497 self.nHeights = 0
560 self.nHeights = 0
498 self.firstHeight = 0
561 self.firstHeight = 0
499 self.deltaHeight = 0
562 self.deltaHeight = 0
500 self.samplesWin = 0
563 self.samplesWin = 0
501 self.spectraComb = 0
564 self.spectraComb = 0
502 self.nCode = None
565 self.nCode = None
503 self.code = None
566 self.code = None
504 self.nBaud = None
567 self.nBaud = None
505
568
506 self.shif_fft = False
569 self.shif_fft = False
507 self.flag_dc = False
570 self.flag_dc = False
508 self.flag_cspc = False
571 self.flag_cspc = False
509 self.flag_decode = False
572 self.flag_decode = False
510 self.flag_deflip = False
573 self.flag_deflip = False
511
574 self.length = 0
512 def read(self, fp):
575 def read(self, fp):
513
576 self.length = 0
514 startFp = fp.tell()
577 try:
578 startFp = fp.tell()
579 except Exception, e:
580 startFp = None
581 pass
515
582
516 try:
583 try:
517 header = numpy.fromfile(fp,PROCESSING_STRUCTURE,1)
584 if hasattr(fp, 'read'):
585 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
586 else:
587 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
588 self.length += header.nbytes
518 except Exception, e:
589 except Exception, e:
519 print "ProcessingHeader: " + e
590 print "ProcessingHeader: " + str(e)
520 return 0
591 return 0
521
592
522 size = int(header['nSize'][0])
593 size = int(header['nSize'][0])
523 self.dtype = int(header['nDataType'][0])
594 self.dtype = int(header['nDataType'][0])
524 self.blockSize = int(header['nSizeOfDataBlock'][0])
595 self.blockSize = int(header['nSizeOfDataBlock'][0])
525 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
596 self.profilesPerBlock = int(header['nProfilesperBlock'][0])
526 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
597 self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0])
527 self.nWindows = int(header['nNumWindows'][0])
598 self.nWindows = int(header['nNumWindows'][0])
528 self.processFlags = header['nProcessFlags']
599 self.processFlags = header['nProcessFlags']
529 self.nCohInt = int(header['nCoherentIntegrations'][0])
600 self.nCohInt = int(header['nCoherentIntegrations'][0])
530 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
601 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
531 self.totalSpectra = int(header['nTotalSpectra'][0])
602 self.totalSpectra = int(header['nTotalSpectra'][0])
532
603
533 samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
604 try:
605 if hasattr(fp, 'read'):
606 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
607 else:
608 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
609 self.length += samplingWindow.nbytes
610 except Exception, e:
611 print "ProcessingHeader: " + str(e)
612 return 0
534
613
535 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
614 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
536 self.firstHeight = float(samplingWindow['h0'][0])
615 self.firstHeight = float(samplingWindow['h0'][0])
537 self.deltaHeight = float(samplingWindow['dh'][0])
616 self.deltaHeight = float(samplingWindow['dh'][0])
538 self.samplesWin = samplingWindow['nsa'][0]
617 self.samplesWin = samplingWindow['nsa'][0]
539
618
540 self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
619
620 try:
621 if hasattr(fp, 'read'):
622 self.spectraComb = numpy.fromfile(fp, 'u1', 2*self.totalSpectra)
623 else:
624 self.spectraComb = numpy.fromstring(fp[self.length:], 'u1', 2*self.totalSpectra)
625 self.length += self.spectraComb.nbytes
626 except Exception, e:
627 print "ProcessingHeader: " + str(e)
628 return 0
541
629
542 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
630 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
543 self.nCode = int(numpy.fromfile(fp,'<u4',1))
631 self.nCode = int(numpy.fromfile(fp,'<u4',1))
544 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
632 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
545 self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud)
633 self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud)
546
634
547 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
635 if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP):
548 exp_name_len = int(numpy.fromfile(fp,'<u4',1))
636 exp_name_len = int(numpy.fromfile(fp,'<u4',1))
549 exp_name = numpy.fromfile(fp,'u1',exp_name_len+1)
637 exp_name = numpy.fromfile(fp,'u1',exp_name_len+1)
550
638
551 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
639 if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA):
552 self.shif_fft = True
640 self.shif_fft = True
553 else:
641 else:
554 self.shif_fft = False
642 self.shif_fft = False
555
643
556 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
644 if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC):
557 self.flag_dc = True
645 self.flag_dc = True
558 else:
646 else:
559 self.flag_dc = False
647 self.flag_dc = False
560
648
561 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
649 if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA):
562 self.flag_decode = True
650 self.flag_decode = True
563 else:
651 else:
564 self.flag_decode = False
652 self.flag_decode = False
565
653
566 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
654 if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA):
567 self.flag_deflip = True
655 self.flag_deflip = True
568 else:
656 else:
569 self.flag_deflip = False
657 self.flag_deflip = False
570
658
571 nChannels = 0
659 nChannels = 0
572 nPairs = 0
660 nPairs = 0
573 pairList = []
661 pairList = []
574
662
575 for i in range( 0, self.totalSpectra*2, 2 ):
663 for i in range( 0, self.totalSpectra*2, 2 ):
576 if self.spectraComb[i] == self.spectraComb[i+1]:
664 if self.spectraComb[i] == self.spectraComb[i+1]:
577 nChannels = nChannels + 1 #par de canales iguales
665 nChannels = nChannels + 1 #par de canales iguales
578 else:
666 else:
579 nPairs = nPairs + 1 #par de canales diferentes
667 nPairs = nPairs + 1 #par de canales diferentes
580 pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
668 pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) )
581
669
582 self.flag_cspc = False
670 self.flag_cspc = False
583 if nPairs > 0:
671 if nPairs > 0:
584 self.flag_cspc = True
672 self.flag_cspc = True
585
673
586 endFp = size + startFp
587
674
588 if fp.tell() > endFp:
675
589 sys.stderr.write("Warning: Processing header size is lower than it has to be")
676 if startFp is not None:
590 return 0
677 endFp = size + startFp
591
678 if fp.tell() > endFp:
592 if fp.tell() < endFp:
679 sys.stderr.write("Warning: Processing header size is lower than it has to be")
593 sys.stderr.write("Warning: Processing header size is greater than it is considered")
680 return 0
681
682 if fp.tell() < endFp:
683 sys.stderr.write("Warning: Processing header size is greater than it is considered")
594
684
595 return 1
685 return 1
596
686
597 def write(self, fp):
687 def write(self, fp):
598 #Clear DEFINE_PROCESS_CODE
688 #Clear DEFINE_PROCESS_CODE
599 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
689 self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE)
600
690
601 headerTuple = (self.size,
691 headerTuple = (self.size,
602 self.dtype,
692 self.dtype,
603 self.blockSize,
693 self.blockSize,
604 self.profilesPerBlock,
694 self.profilesPerBlock,
605 self.dataBlocksPerFile,
695 self.dataBlocksPerFile,
606 self.nWindows,
696 self.nWindows,
607 self.processFlags,
697 self.processFlags,
608 self.nCohInt,
698 self.nCohInt,
609 self.nIncohInt,
699 self.nIncohInt,
610 self.totalSpectra)
700 self.totalSpectra)
611
701
612 header = numpy.array(headerTuple,PROCESSING_STRUCTURE)
702 header = numpy.array(headerTuple,PROCESSING_STRUCTURE)
613 header.tofile(fp)
703 header.tofile(fp)
614
704
615 if self.nWindows != 0:
705 if self.nWindows != 0:
616 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
706 sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin)
617 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
707 samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE)
618 samplingWindow.tofile(fp)
708 samplingWindow.tofile(fp)
619
709
620 if self.totalSpectra != 0:
710 if self.totalSpectra != 0:
621 # spectraComb = numpy.array([],numpy.dtype('u1'))
711 # spectraComb = numpy.array([],numpy.dtype('u1'))
622 spectraComb = self.spectraComb
712 spectraComb = self.spectraComb
623 spectraComb.tofile(fp)
713 spectraComb.tofile(fp)
624
714
625 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
715 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
626 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
716 # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba
627 # nCode.tofile(fp)
717 # nCode.tofile(fp)
628 #
718 #
629 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
719 # nBaud = numpy.array([self.nBaud], numpy.dtype('u4'))
630 # nBaud.tofile(fp)
720 # nBaud.tofile(fp)
631 #
721 #
632 # code = self.code.reshape(self.nCode*self.nBaud)
722 # code = self.code.reshape(self.nCode*self.nBaud)
633 # code = code.astype(numpy.dtype('<f4'))
723 # code = code.astype(numpy.dtype('<f4'))
634 # code.tofile(fp)
724 # code.tofile(fp)
635
725
636 return 1
726 return 1
637
727
638 def get_size(self):
728 def get_size(self):
639
729
640 self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra
730 self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra
641
731
642 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
732 # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE:
643 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
733 # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.)
644 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
734 # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud
645
735
646 return self.__size
736 return self.__size
647
737
648 def set_size(self, value):
738 def set_size(self, value):
649
739
650 raise IOError, "size is a property and it cannot be set, just read"
740 raise IOError, "size is a property and it cannot be set, just read"
651
741
652 return
742 return
653
743
654 size = property(get_size, set_size)
744 size = property(get_size, set_size)
655
745
656 class RCfunction:
746 class RCfunction:
657 NONE=0
747 NONE=0
658 FLIP=1
748 FLIP=1
659 CODE=2
749 CODE=2
660 SAMPLING=3
750 SAMPLING=3
661 LIN6DIV256=4
751 LIN6DIV256=4
662 SYNCHRO=5
752 SYNCHRO=5
663
753
664 class nCodeType:
754 class nCodeType:
665 NONE=0
755 NONE=0
666 USERDEFINE=1
756 USERDEFINE=1
667 BARKER2=2
757 BARKER2=2
668 BARKER3=3
758 BARKER3=3
669 BARKER4=4
759 BARKER4=4
670 BARKER5=5
760 BARKER5=5
671 BARKER7=6
761 BARKER7=6
672 BARKER11=7
762 BARKER11=7
673 BARKER13=8
763 BARKER13=8
674 AC128=9
764 AC128=9
675 COMPLEMENTARYCODE2=10
765 COMPLEMENTARYCODE2=10
676 COMPLEMENTARYCODE4=11
766 COMPLEMENTARYCODE4=11
677 COMPLEMENTARYCODE8=12
767 COMPLEMENTARYCODE8=12
678 COMPLEMENTARYCODE16=13
768 COMPLEMENTARYCODE16=13
679 COMPLEMENTARYCODE32=14
769 COMPLEMENTARYCODE32=14
680 COMPLEMENTARYCODE64=15
770 COMPLEMENTARYCODE64=15
681 COMPLEMENTARYCODE128=16
771 COMPLEMENTARYCODE128=16
682 CODE_BINARY28=17
772 CODE_BINARY28=17
683
773
684 class PROCFLAG:
774 class PROCFLAG:
685
775
686 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
776 COHERENT_INTEGRATION = numpy.uint32(0x00000001)
687 DECODE_DATA = numpy.uint32(0x00000002)
777 DECODE_DATA = numpy.uint32(0x00000002)
688 SPECTRA_CALC = numpy.uint32(0x00000004)
778 SPECTRA_CALC = numpy.uint32(0x00000004)
689 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
779 INCOHERENT_INTEGRATION = numpy.uint32(0x00000008)
690 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
780 POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010)
691 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
781 SHIFT_FFT_DATA = numpy.uint32(0x00000020)
692
782
693 DATATYPE_CHAR = numpy.uint32(0x00000040)
783 DATATYPE_CHAR = numpy.uint32(0x00000040)
694 DATATYPE_SHORT = numpy.uint32(0x00000080)
784 DATATYPE_SHORT = numpy.uint32(0x00000080)
695 DATATYPE_LONG = numpy.uint32(0x00000100)
785 DATATYPE_LONG = numpy.uint32(0x00000100)
696 DATATYPE_INT64 = numpy.uint32(0x00000200)
786 DATATYPE_INT64 = numpy.uint32(0x00000200)
697 DATATYPE_FLOAT = numpy.uint32(0x00000400)
787 DATATYPE_FLOAT = numpy.uint32(0x00000400)
698 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
788 DATATYPE_DOUBLE = numpy.uint32(0x00000800)
699
789
700 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
790 DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000)
701 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
791 DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000)
702 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
792 DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000)
703
793
704 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
794 SAVE_CHANNELS_DC = numpy.uint32(0x00008000)
705 DEFLIP_DATA = numpy.uint32(0x00010000)
795 DEFLIP_DATA = numpy.uint32(0x00010000)
706 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
796 DEFINE_PROCESS_CODE = numpy.uint32(0x00020000)
707
797
708 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
798 ACQ_SYS_NATALIA = numpy.uint32(0x00040000)
709 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
799 ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000)
710 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
800 ACQ_SYS_ADRXD = numpy.uint32(0x000C0000)
711 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
801 ACQ_SYS_JULIA = numpy.uint32(0x00100000)
712 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
802 ACQ_SYS_XXXXXX = numpy.uint32(0x00140000)
713
803
714 EXP_NAME_ESP = numpy.uint32(0x00200000)
804 EXP_NAME_ESP = numpy.uint32(0x00200000)
715 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
805 CHANNEL_NAMES_ESP = numpy.uint32(0x00400000)
716
806
717 OPERATION_MASK = numpy.uint32(0x0000003F)
807 OPERATION_MASK = numpy.uint32(0x0000003F)
718 DATATYPE_MASK = numpy.uint32(0x00000FC0)
808 DATATYPE_MASK = numpy.uint32(0x00000FC0)
719 DATAARRANGE_MASK = numpy.uint32(0x00007000)
809 DATAARRANGE_MASK = numpy.uint32(0x00007000)
720 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
810 ACQ_SYS_MASK = numpy.uint32(0x001C0000)
721
811
722 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
812 dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')])
723 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
813 dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')])
724 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
814 dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')])
725 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
815 dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')])
726 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
816 dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')])
727 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
817 dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')])
728
818
729 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
819 NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5]
730
820
731 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
821 PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR,
732 PROCFLAG.DATATYPE_SHORT,
822 PROCFLAG.DATATYPE_SHORT,
733 PROCFLAG.DATATYPE_LONG,
823 PROCFLAG.DATATYPE_LONG,
734 PROCFLAG.DATATYPE_INT64,
824 PROCFLAG.DATATYPE_INT64,
735 PROCFLAG.DATATYPE_FLOAT,
825 PROCFLAG.DATATYPE_FLOAT,
736 PROCFLAG.DATATYPE_DOUBLE]
826 PROCFLAG.DATATYPE_DOUBLE]
737
827
738 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
828 DTYPE_WIDTH = [1, 2, 4, 8, 4, 8]
739
829
740 def get_dtype_index(numpy_dtype):
830 def get_dtype_index(numpy_dtype):
741
831
742 index = None
832 index = None
743
833
744 for i in range(len(NUMPY_DTYPE_LIST)):
834 for i in range(len(NUMPY_DTYPE_LIST)):
745 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
835 if numpy_dtype == NUMPY_DTYPE_LIST[i]:
746 index = i
836 index = i
747 break
837 break
748
838
749 return index
839 return index
750
840
751 def get_numpy_dtype(index):
841 def get_numpy_dtype(index):
752
842
753 return NUMPY_DTYPE_LIST[index]
843 return NUMPY_DTYPE_LIST[index]
754
844
755 def get_procflag_dtype(index):
845 def get_procflag_dtype(index):
756
846
757 return PROCFLAG_DTYPE_LIST[index]
847 return PROCFLAG_DTYPE_LIST[index]
758
848
759 def get_dtype_width(index):
849 def get_dtype_width(index):
760
850
761 return DTYPE_WIDTH[index] No newline at end of file
851 return DTYPE_WIDTH[index]
Show file before | Show file after | Hide comments
@@ -1,1794 +1,1816
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time, datetime
13 import time, datetime
14 #import h5py
15 import traceback
14 import traceback
15 import zmq
16
16
17 try:
17 try:
18 from gevent import sleep
18 from gevent import sleep
19 except:
19 except:
20 from time import sleep
20 from time import sleep
21
21
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24
24
25 LOCALTIME = True
25 LOCALTIME = True
26
26
27 def isNumber(cad):
27 def isNumber(cad):
28 """
28 """
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30
30
31 Excepciones:
31 Excepciones:
32 Si un determinado string no puede ser convertido a numero
32 Si un determinado string no puede ser convertido a numero
33 Input:
33 Input:
34 str, string al cual se le analiza para determinar si convertible a un numero o no
34 str, string al cual se le analiza para determinar si convertible a un numero o no
35
35
36 Return:
36 Return:
37 True : si el string es uno numerico
37 True : si el string es uno numerico
38 False : no es un string numerico
38 False : no es un string numerico
39 """
39 """
40 try:
40 try:
41 float( cad )
41 float( cad )
42 return True
42 return True
43 except:
43 except:
44 return False
44 return False
45
45
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 """
47 """
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49
49
50 Inputs:
50 Inputs:
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52
52
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 segundos contados desde 01/01/1970.
54 segundos contados desde 01/01/1970.
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 segundos contados desde 01/01/1970.
56 segundos contados desde 01/01/1970.
57
57
58 Return:
58 Return:
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 fecha especificado, de lo contrario retorna False.
60 fecha especificado, de lo contrario retorna False.
61
61
62 Excepciones:
62 Excepciones:
63 Si el archivo no existe o no puede ser abierto
63 Si el archivo no existe o no puede ser abierto
64 Si la cabecera no puede ser leida.
64 Si la cabecera no puede ser leida.
65
65
66 """
66 """
67 basicHeaderObj = BasicHeader(LOCALTIME)
67 basicHeaderObj = BasicHeader(LOCALTIME)
68
68
69 try:
69 try:
70 fp = open(filename,'rb')
70 fp = open(filename,'rb')
71 except IOError:
71 except IOError:
72 print "The file %s can't be opened" %(filename)
72 print "The file %s can't be opened" %(filename)
73 return 0
73 return 0
74
74
75 sts = basicHeaderObj.read(fp)
75 sts = basicHeaderObj.read(fp)
76 fp.close()
76 fp.close()
77
77
78 if not(sts):
78 if not(sts):
79 print "Skipping the file %s because it has not a valid header" %(filename)
79 print "Skipping the file %s because it has not a valid header" %(filename)
80 return 0
80 return 0
81
81
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 return 0
83 return 0
84
84
85 return 1
85 return 1
86
86
87 def isTimeInRange(thisTime, startTime, endTime):
87 def isTimeInRange(thisTime, startTime, endTime):
88
88
89 if endTime >= startTime:
89 if endTime >= startTime:
90 if (thisTime < startTime) or (thisTime > endTime):
90 if (thisTime < startTime) or (thisTime > endTime):
91 return 0
91 return 0
92
92
93 return 1
93 return 1
94 else:
94 else:
95 if (thisTime < startTime) and (thisTime > endTime):
95 if (thisTime < startTime) and (thisTime > endTime):
96 return 0
96 return 0
97
97
98 return 1
98 return 1
99
99
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 """
101 """
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103
103
104 Inputs:
104 Inputs:
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106
106
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108
108
109 endDate : fecha final del rango seleccionado en formato datetime.date
109 endDate : fecha final del rango seleccionado en formato datetime.date
110
110
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112
112
113 endTime : tiempo final del rango seleccionado en formato datetime.time
113 endTime : tiempo final del rango seleccionado en formato datetime.time
114
114
115 Return:
115 Return:
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 fecha especificado, de lo contrario retorna False.
117 fecha especificado, de lo contrario retorna False.
118
118
119 Excepciones:
119 Excepciones:
120 Si el archivo no existe o no puede ser abierto
120 Si el archivo no existe o no puede ser abierto
121 Si la cabecera no puede ser leida.
121 Si la cabecera no puede ser leida.
122
122
123 """
123 """
124
124
125
125
126 try:
126 try:
127 fp = open(filename,'rb')
127 fp = open(filename,'rb')
128 except IOError:
128 except IOError:
129 print "The file %s can't be opened" %(filename)
129 print "The file %s can't be opened" %(filename)
130 return None
130 return None
131
131
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 systemHeaderObj = SystemHeader()
133 systemHeaderObj = SystemHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
134 radarControllerHeaderObj = RadarControllerHeader()
135 processingHeaderObj = ProcessingHeader()
135 processingHeaderObj = ProcessingHeader()
136
136
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138
138
139 sts = firstBasicHeaderObj.read(fp)
139 sts = firstBasicHeaderObj.read(fp)
140
140
141 if not(sts):
141 if not(sts):
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 return None
143 return None
144
144
145 if not systemHeaderObj.read(fp):
145 if not systemHeaderObj.read(fp):
146 return None
146 return None
147
147
148 if not radarControllerHeaderObj.read(fp):
148 if not radarControllerHeaderObj.read(fp):
149 return None
149 return None
150
150
151 if not processingHeaderObj.read(fp):
151 if not processingHeaderObj.read(fp):
152 return None
152 return None
153
153
154 filesize = os.path.getsize(filename)
154 filesize = os.path.getsize(filename)
155
155
156 offset = processingHeaderObj.blockSize + 24 #header size
156 offset = processingHeaderObj.blockSize + 24 #header size
157
157
158 if filesize <= offset:
158 if filesize <= offset:
159 print "[Reading] %s: This file has not enough data" %filename
159 print "[Reading] %s: This file has not enough data" %filename
160 return None
160 return None
161
161
162 fp.seek(-offset, 2)
162 fp.seek(-offset, 2)
163
163
164 sts = lastBasicHeaderObj.read(fp)
164 sts = lastBasicHeaderObj.read(fp)
165
165
166 fp.close()
166 fp.close()
167
167
168 thisDatetime = lastBasicHeaderObj.datatime
168 thisDatetime = lastBasicHeaderObj.datatime
169 thisTime_last_block = thisDatetime.time()
169 thisTime_last_block = thisDatetime.time()
170
170
171 thisDatetime = firstBasicHeaderObj.datatime
171 thisDatetime = firstBasicHeaderObj.datatime
172 thisDate = thisDatetime.date()
172 thisDate = thisDatetime.date()
173 thisTime_first_block = thisDatetime.time()
173 thisTime_first_block = thisDatetime.time()
174
174
175 #General case
175 #General case
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 #-----------o----------------------------o-----------
177 #-----------o----------------------------o-----------
178 # startTime endTime
178 # startTime endTime
179
179
180 if endTime >= startTime:
180 if endTime >= startTime:
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 return None
182 return None
183
183
184 return thisDatetime
184 return thisDatetime
185
185
186 #If endTime < startTime then endTime belongs to the next day
186 #If endTime < startTime then endTime belongs to the next day
187
187
188
188
189 #<<<<<<<<<<<o o>>>>>>>>>>>
189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 #-----------o----------------------------o-----------
190 #-----------o----------------------------o-----------
191 # endTime startTime
191 # endTime startTime
192
192
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 return None
194 return None
195
195
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 return None
197 return None
198
198
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 return None
200 return None
201
201
202 return thisDatetime
202 return thisDatetime
203
203
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 """
205 """
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207
207
208 Inputs:
208 Inputs:
209 folder : nombre completo del directorio.
209 folder : nombre completo del directorio.
210 Su formato deberia ser "/path_root/?YYYYDDD"
210 Su formato deberia ser "/path_root/?YYYYDDD"
211
211
212 siendo:
212 siendo:
213 YYYY : Anio (ejemplo 2015)
213 YYYY : Anio (ejemplo 2015)
214 DDD : Dia del anio (ejemplo 305)
214 DDD : Dia del anio (ejemplo 305)
215
215
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217
217
218 endDate : fecha final del rango seleccionado en formato datetime.date
218 endDate : fecha final del rango seleccionado en formato datetime.date
219
219
220 Return:
220 Return:
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 fecha especificado, de lo contrario retorna False.
222 fecha especificado, de lo contrario retorna False.
223 Excepciones:
223 Excepciones:
224 Si el directorio no tiene el formato adecuado
224 Si el directorio no tiene el formato adecuado
225 """
225 """
226
226
227 basename = os.path.basename(folder)
227 basename = os.path.basename(folder)
228
228
229 if not isRadarFolder(basename):
229 if not isRadarFolder(basename):
230 print "The folder %s has not the rigth format" %folder
230 print "The folder %s has not the rigth format" %folder
231 return 0
231 return 0
232
232
233 if startDate and endDate:
233 if startDate and endDate:
234 thisDate = getDateFromRadarFolder(basename)
234 thisDate = getDateFromRadarFolder(basename)
235
235
236 if thisDate < startDate:
236 if thisDate < startDate:
237 return 0
237 return 0
238
238
239 if thisDate > endDate:
239 if thisDate > endDate:
240 return 0
240 return 0
241
241
242 return 1
242 return 1
243
243
244 def isFileInDateRange(filename, startDate=None, endDate=None):
244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 """
245 """
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247
247
248 Inputs:
248 Inputs:
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250
250
251 Su formato deberia ser "?YYYYDDDsss"
251 Su formato deberia ser "?YYYYDDDsss"
252
252
253 siendo:
253 siendo:
254 YYYY : Anio (ejemplo 2015)
254 YYYY : Anio (ejemplo 2015)
255 DDD : Dia del anio (ejemplo 305)
255 DDD : Dia del anio (ejemplo 305)
256 sss : set
256 sss : set
257
257
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259
259
260 endDate : fecha final del rango seleccionado en formato datetime.date
260 endDate : fecha final del rango seleccionado en formato datetime.date
261
261
262 Return:
262 Return:
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 fecha especificado, de lo contrario retorna False.
264 fecha especificado, de lo contrario retorna False.
265 Excepciones:
265 Excepciones:
266 Si el archivo no tiene el formato adecuado
266 Si el archivo no tiene el formato adecuado
267 """
267 """
268
268
269 basename = os.path.basename(filename)
269 basename = os.path.basename(filename)
270
270
271 if not isRadarFile(basename):
271 if not isRadarFile(basename):
272 print "The filename %s has not the rigth format" %filename
272 print "The filename %s has not the rigth format" %filename
273 return 0
273 return 0
274
274
275 if startDate and endDate:
275 if startDate and endDate:
276 thisDate = getDateFromRadarFile(basename)
276 thisDate = getDateFromRadarFile(basename)
277
277
278 if thisDate < startDate:
278 if thisDate < startDate:
279 return 0
279 return 0
280
280
281 if thisDate > endDate:
281 if thisDate > endDate:
282 return 0
282 return 0
283
283
284 return 1
284 return 1
285
285
286 def getFileFromSet(path, ext, set):
286 def getFileFromSet(path, ext, set):
287 validFilelist = []
287 validFilelist = []
288 fileList = os.listdir(path)
288 fileList = os.listdir(path)
289
289
290 # 0 1234 567 89A BCDE
290 # 0 1234 567 89A BCDE
291 # H YYYY DDD SSS .ext
291 # H YYYY DDD SSS .ext
292
292
293 for thisFile in fileList:
293 for thisFile in fileList:
294 try:
294 try:
295 year = int(thisFile[1:5])
295 year = int(thisFile[1:5])
296 doy = int(thisFile[5:8])
296 doy = int(thisFile[5:8])
297 except:
297 except:
298 continue
298 continue
299
299
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 continue
301 continue
302
302
303 validFilelist.append(thisFile)
303 validFilelist.append(thisFile)
304
304
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306
306
307 if len(myfile)!= 0:
307 if len(myfile)!= 0:
308 return myfile[0]
308 return myfile[0]
309 else:
309 else:
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 print 'the filename %s does not exist'%filename
311 print 'the filename %s does not exist'%filename
312 print '...going to the last file: '
312 print '...going to the last file: '
313
313
314 if validFilelist:
314 if validFilelist:
315 validFilelist = sorted( validFilelist, key=str.lower )
315 validFilelist = sorted( validFilelist, key=str.lower )
316 return validFilelist[-1]
316 return validFilelist[-1]
317
317
318 return None
318 return None
319
319
320 def getlastFileFromPath(path, ext):
320 def getlastFileFromPath(path, ext):
321 """
321 """
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324
324
325 Input:
325 Input:
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 ext : extension de los files contenidos en una carpeta
327 ext : extension de los files contenidos en una carpeta
328
328
329 Return:
329 Return:
330 El ultimo file de una determinada carpeta, no se considera el path.
330 El ultimo file de una determinada carpeta, no se considera el path.
331 """
331 """
332 validFilelist = []
332 validFilelist = []
333 fileList = os.listdir(path)
333 fileList = os.listdir(path)
334
334
335 # 0 1234 567 89A BCDE
335 # 0 1234 567 89A BCDE
336 # H YYYY DDD SSS .ext
336 # H YYYY DDD SSS .ext
337
337
338 for thisFile in fileList:
338 for thisFile in fileList:
339
339
340 year = thisFile[1:5]
340 year = thisFile[1:5]
341 if not isNumber(year):
341 if not isNumber(year):
342 continue
342 continue
343
343
344 doy = thisFile[5:8]
344 doy = thisFile[5:8]
345 if not isNumber(doy):
345 if not isNumber(doy):
346 continue
346 continue
347
347
348 year = int(year)
348 year = int(year)
349 doy = int(doy)
349 doy = int(doy)
350
350
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 continue
352 continue
353
353
354 validFilelist.append(thisFile)
354 validFilelist.append(thisFile)
355
355
356 if validFilelist:
356 if validFilelist:
357 validFilelist = sorted( validFilelist, key=str.lower )
357 validFilelist = sorted( validFilelist, key=str.lower )
358 return validFilelist[-1]
358 return validFilelist[-1]
359
359
360 return None
360 return None
361
361
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 """
363 """
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 el path exacto de un determinado file.
366 el path exacto de un determinado file.
367
367
368 Example :
368 Example :
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370
370
371 Entonces la funcion prueba con las siguientes combinaciones
371 Entonces la funcion prueba con las siguientes combinaciones
372 .../.../y2009307367.ext
372 .../.../y2009307367.ext
373 .../.../Y2009307367.ext
373 .../.../Y2009307367.ext
374 .../.../x2009307/y2009307367.ext
374 .../.../x2009307/y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
375 .../.../x2009307/Y2009307367.ext
376 .../.../X2009307/y2009307367.ext
376 .../.../X2009307/y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
377 .../.../X2009307/Y2009307367.ext
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379
379
380 Return:
380 Return:
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 para el filename
383 para el filename
384 """
384 """
385 fullfilename = None
385 fullfilename = None
386 find_flag = False
386 find_flag = False
387 filename = None
387 filename = None
388
388
389 prefixDirList = [None,'d','D']
389 prefixDirList = [None,'d','D']
390 if ext.lower() == ".r": #voltage
390 if ext.lower() == ".r": #voltage
391 prefixFileList = ['d','D']
391 prefixFileList = ['d','D']
392 elif ext.lower() == ".pdata": #spectra
392 elif ext.lower() == ".pdata": #spectra
393 prefixFileList = ['p','P']
393 prefixFileList = ['p','P']
394 else:
394 else:
395 return None, filename
395 return None, filename
396
396
397 #barrido por las combinaciones posibles
397 #barrido por las combinaciones posibles
398 for prefixDir in prefixDirList:
398 for prefixDir in prefixDirList:
399 thispath = path
399 thispath = path
400 if prefixDir != None:
400 if prefixDir != None:
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 if foldercounter == 0:
402 if foldercounter == 0:
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 else:
404 else:
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409
409
410 if os.path.exists( fullfilename ): #verifico que exista
410 if os.path.exists( fullfilename ): #verifico que exista
411 find_flag = True
411 find_flag = True
412 break
412 break
413 if find_flag:
413 if find_flag:
414 break
414 break
415
415
416 if not(find_flag):
416 if not(find_flag):
417 return None, filename
417 return None, filename
418
418
419 return fullfilename, filename
419 return fullfilename, filename
420
420
421 def isRadarFolder(folder):
421 def isRadarFolder(folder):
422 try:
422 try:
423 year = int(folder[1:5])
423 year = int(folder[1:5])
424 doy = int(folder[5:8])
424 doy = int(folder[5:8])
425 except:
425 except:
426 return 0
426 return 0
427
427
428 return 1
428 return 1
429
429
430 def isRadarFile(file):
430 def isRadarFile(file):
431 try:
431 try:
432 year = int(file[1:5])
432 year = int(file[1:5])
433 doy = int(file[5:8])
433 doy = int(file[5:8])
434 set = int(file[8:11])
434 set = int(file[8:11])
435 except:
435 except:
436 return 0
436 return 0
437
437
438 return 1
438 return 1
439
439
440 def getDateFromRadarFile(file):
440 def getDateFromRadarFile(file):
441 try:
441 try:
442 year = int(file[1:5])
442 year = int(file[1:5])
443 doy = int(file[5:8])
443 doy = int(file[5:8])
444 set = int(file[8:11])
444 set = int(file[8:11])
445 except:
445 except:
446 return None
446 return None
447
447
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 return thisDate
449 return thisDate
450
450
451 def getDateFromRadarFolder(folder):
451 def getDateFromRadarFolder(folder):
452 try:
452 try:
453 year = int(folder[1:5])
453 year = int(folder[1:5])
454 doy = int(folder[5:8])
454 doy = int(folder[5:8])
455 except:
455 except:
456 return None
456 return None
457
457
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 return thisDate
459 return thisDate
460
460
461 class JRODataIO:
461 class JRODataIO:
462
462
463 c = 3E8
463 c = 3E8
464
464
465 isConfig = False
465 isConfig = False
466
466
467 basicHeaderObj = None
467 basicHeaderObj = None
468
468
469 systemHeaderObj = None
469 systemHeaderObj = None
470
470
471 radarControllerHeaderObj = None
471 radarControllerHeaderObj = None
472
472
473 processingHeaderObj = None
473 processingHeaderObj = None
474
474
475 dtype = None
475 dtype = None
476
476
477 pathList = []
477 pathList = []
478
478
479 filenameList = []
479 filenameList = []
480
480
481 filename = None
481 filename = None
482
482
483 ext = None
483 ext = None
484
484
485 flagIsNewFile = 1
485 flagIsNewFile = 1
486
486
487 flagDiscontinuousBlock = 0
487 flagDiscontinuousBlock = 0
488
488
489 flagIsNewBlock = 0
489 flagIsNewBlock = 0
490
490
491 fp = None
491 fp = None
492
492
493 firstHeaderSize = 0
493 firstHeaderSize = 0
494
494
495 basicHeaderSize = 24
495 basicHeaderSize = 24
496
496
497 versionFile = 1103
497 versionFile = 1103
498
498
499 fileSize = None
499 fileSize = None
500
500
501 # ippSeconds = None
501 # ippSeconds = None
502
502
503 fileSizeByHeader = None
503 fileSizeByHeader = None
504
504
505 fileIndex = None
505 fileIndex = None
506
506
507 profileIndex = None
507 profileIndex = None
508
508
509 blockIndex = None
509 blockIndex = None
510
510
511 nTotalBlocks = None
511 nTotalBlocks = None
512
512
513 maxTimeStep = 30
513 maxTimeStep = 30
514
514
515 lastUTTime = None
515 lastUTTime = None
516
516
517 datablock = None
517 datablock = None
518
518
519 dataOut = None
519 dataOut = None
520
520
521 blocksize = None
521 blocksize = None
522
522
523 getByBlock = False
523 getByBlock = False
524
524
525 def __init__(self):
525 def __init__(self):
526
526
527 raise NotImplementedError
527 raise NotImplementedError
528
528
529 def run(self):
529 def run(self):
530
530
531 raise NotImplementedError
531 raise NotImplementedError
532
532
533 def getDtypeWidth(self):
533 def getDtypeWidth(self):
534
534
535 dtype_index = get_dtype_index(self.dtype)
535 dtype_index = get_dtype_index(self.dtype)
536 dtype_width = get_dtype_width(dtype_index)
536 dtype_width = get_dtype_width(dtype_index)
537
537
538 return dtype_width
538 return dtype_width
539
539
540 def getAllowedArgs(self):
540 def getAllowedArgs(self):
541 return inspect.getargspec(self.run).args
541 return inspect.getargspec(self.run).args
542
542
543 class JRODataReader(JRODataIO):
543 class JRODataReader(JRODataIO):
544
544
545
545
546 online = 0
546 online = 0
547
547
548 realtime = 0
548 realtime = 0
549
549
550 nReadBlocks = 0
550 nReadBlocks = 0
551
551
552 delay = 10 #number of seconds waiting a new file
552 delay = 10 #number of seconds waiting a new file
553
553
554 nTries = 3 #quantity tries
554 nTries = 3 #quantity tries
555
555
556 nFiles = 3 #number of files for searching
556 nFiles = 3 #number of files for searching
557
557
558 path = None
558 path = None
559
559
560 foldercounter = 0
560 foldercounter = 0
561
561
562 flagNoMoreFiles = 0
562 flagNoMoreFiles = 0
563
563
564 datetimeList = []
564 datetimeList = []
565
565
566 __isFirstTimeOnline = 1
566 __isFirstTimeOnline = 1
567
567
568 __printInfo = True
568 __printInfo = True
569
569
570 profileIndex = None
570 profileIndex = None
571
571
572 nTxs = 1
572 nTxs = 1
573
573
574 txIndex = None
574 txIndex = None
575
575
576 #Added--------------------
576 #Added--------------------
577
577
578 selBlocksize = None
578 selBlocksize = None
579
579
580 selBlocktime = None
580 selBlocktime = None
581
581
582
582
583 def __init__(self):
583 def __init__(self):
584
584
585 """
585 """
586 This class is used to find data files
586 This class is used to find data files
587
587
588 Example:
588 Example:
589 reader = JRODataReader()
589 reader = JRODataReader()
590 fileList = reader.findDataFiles()
590 fileList = reader.findDataFiles()
591
591
592 """
592 """
593 pass
593 pass
594
594
595
595
596 def createObjByDefault(self):
596 def createObjByDefault(self):
597 """
597 """
598
598
599 """
599 """
600 raise NotImplementedError
600 raise NotImplementedError
601
601
602 def getBlockDimension(self):
602 def getBlockDimension(self):
603
603
604 raise NotImplementedError
604 raise NotImplementedError
605
605
606 def __searchFilesOffLine(self,
606 def __searchFilesOffLine(self,
607 path,
607 path,
608 startDate=None,
608 startDate=None,
609 endDate=None,
609 endDate=None,
610 startTime=datetime.time(0,0,0),
610 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
611 endTime=datetime.time(23,59,59),
612 set=None,
612 set=None,
613 expLabel='',
613 expLabel='',
614 ext='.r',
614 ext='.r',
615 queue=None,
615 queue=None,
616 cursor=None,
616 cursor=None,
617 skip=None,
617 skip=None,
618 walk=True):
618 walk=True):
619
619
620 self.filenameList = []
620 self.filenameList = []
621 self.datetimeList = []
621 self.datetimeList = []
622
622
623 pathList = []
623 pathList = []
624
624
625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
626
626
627 if dateList == []:
627 if dateList == []:
628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
629 return None, None
629 return None, None
630
630
631 if len(dateList) > 1:
631 if len(dateList) > 1:
632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
633 else:
633 else:
634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
635
635
636 filenameList = []
636 filenameList = []
637 datetimeList = []
637 datetimeList = []
638
638
639 for thisPath in pathList:
639 for thisPath in pathList:
640 # thisPath = pathList[pathDict[file]]
640 # thisPath = pathList[pathDict[file]]
641
641
642 fileList = glob.glob1(thisPath, "*%s" %ext)
642 fileList = glob.glob1(thisPath, "*%s" %ext)
643 fileList.sort()
643 fileList.sort()
644
644
645 skippedFileList = []
645 skippedFileList = []
646
646
647 if cursor is not None and skip is not None:
647 if cursor is not None and skip is not None:
648 # if cursor*skip > len(fileList):
648 # if cursor*skip > len(fileList):
649 if skip == 0:
649 if skip == 0:
650 if queue is not None:
650 if queue is not None:
651 queue.put(len(fileList))
651 queue.put(len(fileList))
652 skippedFileList = []
652 skippedFileList = []
653 else:
653 else:
654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
655
655
656 else:
656 else:
657 skippedFileList = fileList
657 skippedFileList = fileList
658
658
659 for file in skippedFileList:
659 for file in skippedFileList:
660
660
661 filename = os.path.join(thisPath,file)
661 filename = os.path.join(thisPath,file)
662
662
663 if not isFileInDateRange(filename, startDate, endDate):
663 if not isFileInDateRange(filename, startDate, endDate):
664 continue
664 continue
665
665
666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
667
667
668 if not(thisDatetime):
668 if not(thisDatetime):
669 continue
669 continue
670
670
671 filenameList.append(filename)
671 filenameList.append(filename)
672 datetimeList.append(thisDatetime)
672 datetimeList.append(thisDatetime)
673
673
674 if not(filenameList):
674 if not(filenameList):
675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
676 return None, None
676 return None, None
677
677
678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
679 print
679 print
680
680
681 for i in range(len(filenameList)):
681 for i in range(len(filenameList)):
682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
683
683
684 self.filenameList = filenameList
684 self.filenameList = filenameList
685 self.datetimeList = datetimeList
685 self.datetimeList = datetimeList
686
686
687 return pathList, filenameList
687 return pathList, filenameList
688
688
689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
690
690
691 """
691 """
692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
693 devuelve el archivo encontrado ademas de otros datos.
693 devuelve el archivo encontrado ademas de otros datos.
694
694
695 Input:
695 Input:
696 path : carpeta donde estan contenidos los files que contiene data
696 path : carpeta donde estan contenidos los files que contiene data
697
697
698 expLabel : Nombre del subexperimento (subfolder)
698 expLabel : Nombre del subexperimento (subfolder)
699
699
700 ext : extension de los files
700 ext : extension de los files
701
701
702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
703
703
704 Return:
704 Return:
705 directory : eL directorio donde esta el file encontrado
705 directory : eL directorio donde esta el file encontrado
706 filename : el ultimo file de una determinada carpeta
706 filename : el ultimo file de una determinada carpeta
707 year : el anho
707 year : el anho
708 doy : el numero de dia del anho
708 doy : el numero de dia del anho
709 set : el set del archivo
709 set : el set del archivo
710
710
711
711
712 """
712 """
713 if not os.path.isdir(path):
713 if not os.path.isdir(path):
714 return None, None, None, None, None, None
714 return None, None, None, None, None, None
715
715
716 dirList = []
716 dirList = []
717
717
718 if not walk:
718 if not walk:
719 fullpath = path
719 fullpath = path
720 foldercounter = 0
720 foldercounter = 0
721 else:
721 else:
722 #Filtra solo los directorios
722 #Filtra solo los directorios
723 for thisPath in os.listdir(path):
723 for thisPath in os.listdir(path):
724 if not os.path.isdir(os.path.join(path,thisPath)):
724 if not os.path.isdir(os.path.join(path,thisPath)):
725 continue
725 continue
726 if not isRadarFolder(thisPath):
726 if not isRadarFolder(thisPath):
727 continue
727 continue
728
728
729 dirList.append(thisPath)
729 dirList.append(thisPath)
730
730
731 if not(dirList):
731 if not(dirList):
732 return None, None, None, None, None, None
732 return None, None, None, None, None, None
733
733
734 dirList = sorted( dirList, key=str.lower )
734 dirList = sorted( dirList, key=str.lower )
735
735
736 doypath = dirList[-1]
736 doypath = dirList[-1]
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 fullpath = os.path.join(path, doypath, expLabel)
738 fullpath = os.path.join(path, doypath, expLabel)
739
739
740
740
741 print "[Reading] %s folder was found: " %(fullpath )
741 print "[Reading] %s folder was found: " %(fullpath )
742
742
743 if set == None:
743 if set == None:
744 filename = getlastFileFromPath(fullpath, ext)
744 filename = getlastFileFromPath(fullpath, ext)
745 else:
745 else:
746 filename = getFileFromSet(fullpath, ext, set)
746 filename = getFileFromSet(fullpath, ext, set)
747
747
748 if not(filename):
748 if not(filename):
749 return None, None, None, None, None, None
749 return None, None, None, None, None, None
750
750
751 print "[Reading] %s file was found" %(filename)
751 print "[Reading] %s file was found" %(filename)
752
752
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 return None, None, None, None, None, None
754 return None, None, None, None, None, None
755
755
756 year = int( filename[1:5] )
756 year = int( filename[1:5] )
757 doy = int( filename[5:8] )
757 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
758 set = int( filename[8:11] )
759
759
760 return fullpath, foldercounter, filename, year, doy, set
760 return fullpath, foldercounter, filename, year, doy, set
761
761
762 def __setNextFileOffline(self):
762 def __setNextFileOffline(self):
763
763
764 idFile = self.fileIndex
764 idFile = self.fileIndex
765
765
766 while (True):
766 while (True):
767 idFile += 1
767 idFile += 1
768 if not(idFile < len(self.filenameList)):
768 if not(idFile < len(self.filenameList)):
769 self.flagNoMoreFiles = 1
769 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
770 # print "[Reading] No more Files"
771 return 0
771 return 0
772
772
773 filename = self.filenameList[idFile]
773 filename = self.filenameList[idFile]
774
774
775 if not(self.__verifyFile(filename)):
775 if not(self.__verifyFile(filename)):
776 continue
776 continue
777
777
778 fileSize = os.path.getsize(filename)
778 fileSize = os.path.getsize(filename)
779 fp = open(filename,'rb')
779 fp = open(filename,'rb')
780 break
780 break
781
781
782 self.flagIsNewFile = 1
782 self.flagIsNewFile = 1
783 self.fileIndex = idFile
783 self.fileIndex = idFile
784 self.filename = filename
784 self.filename = filename
785 self.fileSize = fileSize
785 self.fileSize = fileSize
786 self.fp = fp
786 self.fp = fp
787
787
788 # print "[Reading] Setting the file: %s"%self.filename
788 # print "[Reading] Setting the file: %s"%self.filename
789
789
790 return 1
790 return 1
791
791
792 def __setNextFileOnline(self):
792 def __setNextFileOnline(self):
793 """
793 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
796 siguientes.
797
797
798 Affected:
798 Affected:
799 self.flagIsNewFile
799 self.flagIsNewFile
800 self.filename
800 self.filename
801 self.fileSize
801 self.fileSize
802 self.fp
802 self.fp
803 self.set
803 self.set
804 self.flagNoMoreFiles
804 self.flagNoMoreFiles
805
805
806 Return:
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
809
810 Excepciones:
810 Excepciones:
811 Si un determinado file no puede ser abierto
811 Si un determinado file no puede ser abierto
812 """
812 """
813 nFiles = 0
813 nFiles = 0
814 fileOk_flag = False
814 fileOk_flag = False
815 firstTime_flag = True
815 firstTime_flag = True
816
816
817 self.set += 1
817 self.set += 1
818
818
819 if self.set > 999:
819 if self.set > 999:
820 self.set = 0
820 self.set = 0
821 self.foldercounter += 1
821 self.foldercounter += 1
822
822
823 #busca el 1er file disponible
823 #busca el 1er file disponible
824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 if fullfilename:
825 if fullfilename:
826 if self.__verifyFile(fullfilename, False):
826 if self.__verifyFile(fullfilename, False):
827 fileOk_flag = True
827 fileOk_flag = True
828
828
829 #si no encuentra un file entonces espera y vuelve a buscar
829 #si no encuentra un file entonces espera y vuelve a buscar
830 if not(fileOk_flag):
830 if not(fileOk_flag):
831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832
832
833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 tries = self.nTries
834 tries = self.nTries
835 else:
835 else:
836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837
837
838 for nTries in range( tries ):
838 for nTries in range( tries ):
839 if firstTime_flag:
839 if firstTime_flag:
840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 sleep( self.delay )
841 sleep( self.delay )
842 else:
842 else:
843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844
844
845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 if fullfilename:
846 if fullfilename:
847 if self.__verifyFile(fullfilename):
847 if self.__verifyFile(fullfilename):
848 fileOk_flag = True
848 fileOk_flag = True
849 break
849 break
850
850
851 if fileOk_flag:
851 if fileOk_flag:
852 break
852 break
853
853
854 firstTime_flag = False
854 firstTime_flag = False
855
855
856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 self.set += 1
857 self.set += 1
858
858
859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 self.set = 0
860 self.set = 0
861 self.doy += 1
861 self.doy += 1
862 self.foldercounter = 0
862 self.foldercounter = 0
863
863
864 if fileOk_flag:
864 if fileOk_flag:
865 self.fileSize = os.path.getsize( fullfilename )
865 self.fileSize = os.path.getsize( fullfilename )
866 self.filename = fullfilename
866 self.filename = fullfilename
867 self.flagIsNewFile = 1
867 self.flagIsNewFile = 1
868 if self.fp != None: self.fp.close()
868 if self.fp != None: self.fp.close()
869 self.fp = open(fullfilename, 'rb')
869 self.fp = open(fullfilename, 'rb')
870 self.flagNoMoreFiles = 0
870 self.flagNoMoreFiles = 0
871 # print '[Reading] Setting the file: %s' % fullfilename
871 # print '[Reading] Setting the file: %s' % fullfilename
872 else:
872 else:
873 self.fileSize = 0
873 self.fileSize = 0
874 self.filename = None
874 self.filename = None
875 self.flagIsNewFile = 0
875 self.flagIsNewFile = 0
876 self.fp = None
876 self.fp = None
877 self.flagNoMoreFiles = 1
877 self.flagNoMoreFiles = 1
878 # print '[Reading] No more files to read'
878 # print '[Reading] No more files to read'
879
879
880 return fileOk_flag
880 return fileOk_flag
881
881
882 def setNextFile(self):
882 def setNextFile(self):
883 if self.fp != None:
883 if self.fp != None:
884 self.fp.close()
884 self.fp.close()
885
885
886 if self.online:
886 if self.online:
887 newFile = self.__setNextFileOnline()
887 newFile = self.__setNextFileOnline()
888 else:
888 else:
889 newFile = self.__setNextFileOffline()
889 newFile = self.__setNextFileOffline()
890
890
891 if not(newFile):
891 if not(newFile):
892 print '[Reading] No more files to read'
892 print '[Reading] No more files to read'
893 return 0
893 return 0
894
894
895 if self.verbose:
895 if self.verbose:
896 print '[Reading] Setting the file: %s' % self.filename
896 print '[Reading] Setting the file: %s' % self.filename
897
897
898 self.__readFirstHeader()
898 self.__readFirstHeader()
899 self.nReadBlocks = 0
899 self.nReadBlocks = 0
900 return 1
900 return 1
901
901
902 def __waitNewBlock(self):
902 def __waitNewBlock(self):
903 """
903 """
904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
905
905
906 Si el modo de lectura es OffLine siempre retorn 0
906 Si el modo de lectura es OffLine siempre retorn 0
907 """
907 """
908 if not self.online:
908 if not self.online:
909 return 0
909 return 0
910
910
911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
912 return 0
912 return 0
913
913
914 currentPointer = self.fp.tell()
914 currentPointer = self.fp.tell()
915
915
916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917
917
918 for nTries in range( self.nTries ):
918 for nTries in range( self.nTries ):
919
919
920 self.fp.close()
920 self.fp.close()
921 self.fp = open( self.filename, 'rb' )
921 self.fp = open( self.filename, 'rb' )
922 self.fp.seek( currentPointer )
922 self.fp.seek( currentPointer )
923
923
924 self.fileSize = os.path.getsize( self.filename )
924 self.fileSize = os.path.getsize( self.filename )
925 currentSize = self.fileSize - currentPointer
925 currentSize = self.fileSize - currentPointer
926
926
927 if ( currentSize >= neededSize ):
927 if ( currentSize >= neededSize ):
928 self.basicHeaderObj.read(self.fp)
928 self.basicHeaderObj.read(self.fp)
929 return 1
929 return 1
930
930
931 if self.fileSize == self.fileSizeByHeader:
931 if self.fileSize == self.fileSizeByHeader:
932 # self.flagEoF = True
932 # self.flagEoF = True
933 return 0
933 return 0
934
934
935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
936 sleep( self.delay )
936 sleep( self.delay )
937
937
938
938
939 return 0
939 return 0
940
940
941 def waitDataBlock(self,pointer_location):
941 def waitDataBlock(self,pointer_location):
942
942
943 currentPointer = pointer_location
943 currentPointer = pointer_location
944
944
945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
946
946
947 for nTries in range( self.nTries ):
947 for nTries in range( self.nTries ):
948 self.fp.close()
948 self.fp.close()
949 self.fp = open( self.filename, 'rb' )
949 self.fp = open( self.filename, 'rb' )
950 self.fp.seek( currentPointer )
950 self.fp.seek( currentPointer )
951
951
952 self.fileSize = os.path.getsize( self.filename )
952 self.fileSize = os.path.getsize( self.filename )
953 currentSize = self.fileSize - currentPointer
953 currentSize = self.fileSize - currentPointer
954
954
955 if ( currentSize >= neededSize ):
955 if ( currentSize >= neededSize ):
956 return 1
956 return 1
957
957
958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
959 sleep( self.delay )
959 sleep( self.delay )
960
960
961 return 0
961 return 0
962
962
963 def __jumpToLastBlock(self):
963 def __jumpToLastBlock(self):
964
964
965 if not(self.__isFirstTimeOnline):
965 if not(self.__isFirstTimeOnline):
966 return
966 return
967
967
968 csize = self.fileSize - self.fp.tell()
968 csize = self.fileSize - self.fp.tell()
969 blocksize = self.processingHeaderObj.blockSize
969 blocksize = self.processingHeaderObj.blockSize
970
970
971 #salta el primer bloque de datos
971 #salta el primer bloque de datos
972 if csize > self.processingHeaderObj.blockSize:
972 if csize > self.processingHeaderObj.blockSize:
973 self.fp.seek(self.fp.tell() + blocksize)
973 self.fp.seek(self.fp.tell() + blocksize)
974 else:
974 else:
975 return
975 return
976
976
977 csize = self.fileSize - self.fp.tell()
977 csize = self.fileSize - self.fp.tell()
978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
979 while True:
979 while True:
980
980
981 if self.fp.tell()<self.fileSize:
981 if self.fp.tell()<self.fileSize:
982 self.fp.seek(self.fp.tell() + neededsize)
982 self.fp.seek(self.fp.tell() + neededsize)
983 else:
983 else:
984 self.fp.seek(self.fp.tell() - neededsize)
984 self.fp.seek(self.fp.tell() - neededsize)
985 break
985 break
986
986
987 # csize = self.fileSize - self.fp.tell()
987 # csize = self.fileSize - self.fp.tell()
988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
989 # factor = int(csize/neededsize)
989 # factor = int(csize/neededsize)
990 # if factor > 0:
990 # if factor > 0:
991 # self.fp.seek(self.fp.tell() + factor*neededsize)
991 # self.fp.seek(self.fp.tell() + factor*neededsize)
992
992
993 self.flagIsNewFile = 0
993 self.flagIsNewFile = 0
994 self.__isFirstTimeOnline = 0
994 self.__isFirstTimeOnline = 0
995
995
996 def __setNewBlock(self):
996 def __setNewBlock(self):
997
997 #if self.server is None:
998 if self.fp == None:
998 if self.fp == None:
999 return 0
999 return 0
1000
1000
1001 # if self.online:
1001 # if self.online:
1002 # self.__jumpToLastBlock()
1002 # self.__jumpToLastBlock()
1003 print 'xxxx'
1003
1004
1004 if self.flagIsNewFile:
1005 if self.flagIsNewFile:
1005 self.lastUTTime = self.basicHeaderObj.utc
1006 self.lastUTTime = self.basicHeaderObj.utc
1006 return 1
1007 return 1
1007
1008
1008 if self.realtime:
1009 if self.realtime:
1009 self.flagDiscontinuousBlock = 1
1010 self.flagDiscontinuousBlock = 1
1010 if not(self.setNextFile()):
1011 if not(self.setNextFile()):
1011 return 0
1012 return 0
1012 else:
1013 else:
1013 return 1
1014 return 1
1014
1015 print 'xxxx'
1016 #if self.server is None:
1015 currentSize = self.fileSize - self.fp.tell()
1017 currentSize = self.fileSize - self.fp.tell()
1016 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1018 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1017
1018 if (currentSize >= neededSize):
1019 if (currentSize >= neededSize):
1019 self.basicHeaderObj.read(self.fp)
1020 self.basicHeaderObj.read(self.fp)
1020 self.lastUTTime = self.basicHeaderObj.utc
1021 self.lastUTTime = self.basicHeaderObj.utc
1021 return 1
1022 return 1
1022
1023 # else:
1024 # self.basicHeaderObj.read(self.zHeader)
1025 # self.lastUTTime = self.basicHeaderObj.utc
1026 # return 1
1023 if self.__waitNewBlock():
1027 if self.__waitNewBlock():
1024 self.lastUTTime = self.basicHeaderObj.utc
1028 self.lastUTTime = self.basicHeaderObj.utc
1025 return 1
1029 return 1
1026
1030 #if self.server is None:
1027 if not(self.setNextFile()):
1031 if not(self.setNextFile()):
1028 return 0
1032 return 0
1029
1033
1030 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1034 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1031 self.lastUTTime = self.basicHeaderObj.utc
1035 self.lastUTTime = self.basicHeaderObj.utc
1032
1036
1033 self.flagDiscontinuousBlock = 0
1037 self.flagDiscontinuousBlock = 0
1034
1038
1035 if deltaTime > self.maxTimeStep:
1039 if deltaTime > self.maxTimeStep:
1036 self.flagDiscontinuousBlock = 1
1040 self.flagDiscontinuousBlock = 1
1037
1041
1038 return 1
1042 return 1
1039
1043
1040 def readNextBlock(self):
1044 def readNextBlock(self):
1041
1045
1042 #Skip block out of startTime and endTime
1046 #Skip block out of startTime and endTime
1043 while True:
1047 while True:
1048 print 'cxxxx'
1044 if not(self.__setNewBlock()):
1049 if not(self.__setNewBlock()):
1050 print 'returning'
1045 return 0
1051 return 0
1046
1052 print 'dxxx'
1047 if not(self.readBlock()):
1053 if not(self.readBlock()):
1048 return 0
1054 return 0
1049
1055
1050 self.getBasicHeader()
1056 self.getBasicHeader()
1051
1057
1052 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1058 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1053
1059
1054 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1060 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1055 self.processingHeaderObj.dataBlocksPerFile,
1061 self.processingHeaderObj.dataBlocksPerFile,
1056 self.dataOut.datatime.ctime())
1062 self.dataOut.datatime.ctime())
1057 continue
1063 continue
1058
1064
1059 break
1065 break
1060
1066
1061 if self.verbose:
1067 if self.verbose:
1062 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1068 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1063 self.processingHeaderObj.dataBlocksPerFile,
1069 self.processingHeaderObj.dataBlocksPerFile,
1064 self.dataOut.datatime.ctime())
1070 self.dataOut.datatime.ctime())
1065 return 1
1071 return 1
1066
1072
1067 def __readFirstHeader(self):
1073 def __readFirstHeader(self):
1068
1074
1069 self.basicHeaderObj.read(self.fp)
1075 self.basicHeaderObj.read(self.fp)
1070 self.systemHeaderObj.read(self.fp)
1076 self.systemHeaderObj.read(self.fp)
1071 self.radarControllerHeaderObj.read(self.fp)
1077 self.radarControllerHeaderObj.read(self.fp)
1072 self.processingHeaderObj.read(self.fp)
1078 self.processingHeaderObj.read(self.fp)
1073
1079
1074 self.firstHeaderSize = self.basicHeaderObj.size
1080 self.firstHeaderSize = self.basicHeaderObj.size
1075
1081
1076 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1082 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1077 if datatype == 0:
1083 if datatype == 0:
1078 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1084 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1079 elif datatype == 1:
1085 elif datatype == 1:
1080 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1086 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1081 elif datatype == 2:
1087 elif datatype == 2:
1082 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1088 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1083 elif datatype == 3:
1089 elif datatype == 3:
1084 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1090 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1085 elif datatype == 4:
1091 elif datatype == 4:
1086 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1092 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1087 elif datatype == 5:
1093 elif datatype == 5:
1088 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1094 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1089 else:
1095 else:
1090 raise ValueError, 'Data type was not defined'
1096 raise ValueError, 'Data type was not defined'
1091
1097
1092 self.dtype = datatype_str
1098 self.dtype = datatype_str
1093 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1099 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1094 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1100 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1095 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1101 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1096 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1102 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1097 self.getBlockDimension()
1103 self.getBlockDimension()
1098
1104
1099 def __verifyFile(self, filename, msgFlag=True):
1105 def __verifyFile(self, filename, msgFlag=True):
1100
1106
1101 msg = None
1107 msg = None
1102
1108
1103 try:
1109 try:
1104 fp = open(filename, 'rb')
1110 fp = open(filename, 'rb')
1105 except IOError:
1111 except IOError:
1106
1112
1107 if msgFlag:
1113 if msgFlag:
1108 print "[Reading] File %s can't be opened" % (filename)
1114 print "[Reading] File %s can't be opened" % (filename)
1109
1115
1110 return False
1116 return False
1111
1117
1112 currentPosition = fp.tell()
1118 currentPosition = fp.tell()
1113 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1119 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1114
1120
1115 if neededSize == 0:
1121 if neededSize == 0:
1116 basicHeaderObj = BasicHeader(LOCALTIME)
1122 basicHeaderObj = BasicHeader(LOCALTIME)
1117 systemHeaderObj = SystemHeader()
1123 systemHeaderObj = SystemHeader()
1118 radarControllerHeaderObj = RadarControllerHeader()
1124 radarControllerHeaderObj = RadarControllerHeader()
1119 processingHeaderObj = ProcessingHeader()
1125 processingHeaderObj = ProcessingHeader()
1120
1126
1121 if not( basicHeaderObj.read(fp) ):
1127 if not( basicHeaderObj.read(fp) ):
1122 fp.close()
1128 fp.close()
1123 return False
1129 return False
1124
1130
1125 if not( systemHeaderObj.read(fp) ):
1131 if not( systemHeaderObj.read(fp) ):
1126 fp.close()
1132 fp.close()
1127 return False
1133 return False
1128
1134
1129 if not( radarControllerHeaderObj.read(fp) ):
1135 if not( radarControllerHeaderObj.read(fp) ):
1130 fp.close()
1136 fp.close()
1131 return False
1137 return False
1132
1138
1133 if not( processingHeaderObj.read(fp) ):
1139 if not( processingHeaderObj.read(fp) ):
1134 fp.close()
1140 fp.close()
1135 return False
1141 return False
1136
1142
1137 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1143 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1138 else:
1144 else:
1139 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1145 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1140
1146
1141 fp.close()
1147 fp.close()
1142
1148
1143 fileSize = os.path.getsize(filename)
1149 fileSize = os.path.getsize(filename)
1144 currentSize = fileSize - currentPosition
1150 currentSize = fileSize - currentPosition
1145
1151
1146 if currentSize < neededSize:
1152 if currentSize < neededSize:
1147 if msgFlag and (msg != None):
1153 if msgFlag and (msg != None):
1148 print msg
1154 print msg
1149 return False
1155 return False
1150
1156
1151 return True
1157 return True
1152
1158
1153 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1159 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1154
1160
1155 path_empty = True
1161 path_empty = True
1156
1162
1157 dateList = []
1163 dateList = []
1158 pathList = []
1164 pathList = []
1159
1165
1160 multi_path = path.split(',')
1166 multi_path = path.split(',')
1161
1167
1162 if not walk:
1168 if not walk:
1163
1169
1164 for single_path in multi_path:
1170 for single_path in multi_path:
1165
1171
1166 if not os.path.isdir(single_path):
1172 if not os.path.isdir(single_path):
1167 continue
1173 continue
1168
1174
1169 fileList = glob.glob1(single_path, "*"+ext)
1175 fileList = glob.glob1(single_path, "*"+ext)
1170
1176
1171 if not fileList:
1177 if not fileList:
1172 continue
1178 continue
1173
1179
1174 path_empty = False
1180 path_empty = False
1175
1181
1176 fileList.sort()
1182 fileList.sort()
1177
1183
1178 for thisFile in fileList:
1184 for thisFile in fileList:
1179
1185
1180 if not os.path.isfile(os.path.join(single_path, thisFile)):
1186 if not os.path.isfile(os.path.join(single_path, thisFile)):
1181 continue
1187 continue
1182
1188
1183 if not isRadarFile(thisFile):
1189 if not isRadarFile(thisFile):
1184 continue
1190 continue
1185
1191
1186 if not isFileInDateRange(thisFile, startDate, endDate):
1192 if not isFileInDateRange(thisFile, startDate, endDate):
1187 continue
1193 continue
1188
1194
1189 thisDate = getDateFromRadarFile(thisFile)
1195 thisDate = getDateFromRadarFile(thisFile)
1190
1196
1191 if thisDate in dateList:
1197 if thisDate in dateList:
1192 continue
1198 continue
1193
1199
1194 dateList.append(thisDate)
1200 dateList.append(thisDate)
1195 pathList.append(single_path)
1201 pathList.append(single_path)
1196
1202
1197 else:
1203 else:
1198 for single_path in multi_path:
1204 for single_path in multi_path:
1199
1205
1200 if not os.path.isdir(single_path):
1206 if not os.path.isdir(single_path):
1201 continue
1207 continue
1202
1208
1203 dirList = []
1209 dirList = []
1204
1210
1205 for thisPath in os.listdir(single_path):
1211 for thisPath in os.listdir(single_path):
1206
1212
1207 if not os.path.isdir(os.path.join(single_path,thisPath)):
1213 if not os.path.isdir(os.path.join(single_path,thisPath)):
1208 continue
1214 continue
1209
1215
1210 if not isRadarFolder(thisPath):
1216 if not isRadarFolder(thisPath):
1211 continue
1217 continue
1212
1218
1213 if not isFolderInDateRange(thisPath, startDate, endDate):
1219 if not isFolderInDateRange(thisPath, startDate, endDate):
1214 continue
1220 continue
1215
1221
1216 dirList.append(thisPath)
1222 dirList.append(thisPath)
1217
1223
1218 if not dirList:
1224 if not dirList:
1219 continue
1225 continue
1220
1226
1221 dirList.sort()
1227 dirList.sort()
1222
1228
1223 for thisDir in dirList:
1229 for thisDir in dirList:
1224
1230
1225 datapath = os.path.join(single_path, thisDir, expLabel)
1231 datapath = os.path.join(single_path, thisDir, expLabel)
1226 fileList = glob.glob1(datapath, "*"+ext)
1232 fileList = glob.glob1(datapath, "*"+ext)
1227
1233
1228 if not fileList:
1234 if not fileList:
1229 continue
1235 continue
1230
1236
1231 path_empty = False
1237 path_empty = False
1232
1238
1233 thisDate = getDateFromRadarFolder(thisDir)
1239 thisDate = getDateFromRadarFolder(thisDir)
1234
1240
1235 pathList.append(datapath)
1241 pathList.append(datapath)
1236 dateList.append(thisDate)
1242 dateList.append(thisDate)
1237
1243
1238 dateList.sort()
1244 dateList.sort()
1239
1245
1240 if walk:
1246 if walk:
1241 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1247 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1242 else:
1248 else:
1243 pattern_path = multi_path[0]
1249 pattern_path = multi_path[0]
1244
1250
1245 if path_empty:
1251 if path_empty:
1246 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1252 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1247 else:
1253 else:
1248 if not dateList:
1254 if not dateList:
1249 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1255 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1250
1256
1251 if include_path:
1257 if include_path:
1252 return dateList, pathList
1258 return dateList, pathList
1253
1259
1254 return dateList
1260 return dateList
1255
1261
1256 def setup(self,
1262 def setup(self,
1257 path=None,
1263 path=None,
1258 startDate=None,
1264 startDate=None,
1259 endDate=None,
1265 endDate=None,
1260 startTime=datetime.time(0,0,0),
1266 startTime=datetime.time(0,0,0),
1261 endTime=datetime.time(23,59,59),
1267 endTime=datetime.time(23,59,59),
1262 set=None,
1268 set=None,
1263 expLabel = "",
1269 expLabel = "",
1264 ext = None,
1270 ext = None,
1265 online = False,
1271 online = False,
1266 delay = 60,
1272 delay = 60,
1267 walk = True,
1273 walk = True,
1268 getblock = False,
1274 getblock = False,
1269 nTxs = 1,
1275 nTxs = 1,
1270 realtime=False,
1276 realtime=False,
1271 blocksize=None,
1277 blocksize=None,
1272 blocktime=None,
1278 blocktime=None,
1273 queue=None,
1279 queue=None,
1274 skip=None,
1280 skip=None,
1275 cursor=None,
1281 cursor=None,
1276 warnings=True,
1282 warnings=True,
1277 verbose=True):
1283 verbose=True,
1278
1284 server=None):
1279 if path == None:
1285 if server is not None:
1280 raise ValueError, "[Reading] The path is not valid"
1286 if 'tcp://' in server:
1281
1287 address = server
1282 if ext == None:
1288 else:
1283 ext = self.ext
1289 address = 'ipc:///tmp/%s' % server
1284
1290 self.server = address
1285 if online:
1291 self.context = zmq.Context()
1286 print "[Reading] Searching files in online mode..."
1292 self.receiver = self.context.socket(zmq.PULL)
1287
1293 self.receiver.connect(self.server)
1288 for nTries in range( self.nTries ):
1294 time.sleep(0.5)
1289 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1295 print '[Starting] ReceiverData from {}'.format(self.server)
1290
1296 else:
1291 if fullpath:
1297 self.server = None
1292 break
1298 if path == None:
1293
1299 raise ValueError, "[Reading] The path is not valid"
1294 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1300
1295 sleep( self.delay )
1301 if ext == None:
1296
1302 ext = self.ext
1297 if not(fullpath):
1303
1298 print "[Reading] There 'isn't any valid file in %s" % path
1304 if online:
1299 return
1305 print "[Reading] Searching files in online mode..."
1300
1306
1301 self.year = year
1307 for nTries in range( self.nTries ):
1302 self.doy = doy
1308 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1303 self.set = set - 1
1309
1304 self.path = path
1310 if fullpath:
1305 self.foldercounter = foldercounter
1311 break
1306 last_set = None
1312
1313 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1314 sleep( self.delay )
1315
1316 if not(fullpath):
1317 print "[Reading] There 'isn't any valid file in %s" % path
1318 return
1319
1320 self.year = year
1321 self.doy = doy
1322 self.set = set - 1
1323 self.path = path
1324 self.foldercounter = foldercounter
1325 last_set = None
1326 else:
1327 print "[Reading] Searching files in offline mode ..."
1328 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1329 startTime=startTime, endTime=endTime,
1330 set=set, expLabel=expLabel, ext=ext,
1331 walk=walk, cursor=cursor,
1332 skip=skip, queue=queue)
1307
1333
1308 else:
1334 if not(pathList):
1309 print "[Reading] Searching files in offline mode ..."
1335 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1310 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1336 # datetime.datetime.combine(startDate,startTime).ctime(),
1311 startTime=startTime, endTime=endTime,
1337 # datetime.datetime.combine(endDate,endTime).ctime())
1312 set=set, expLabel=expLabel, ext=ext,
1313 walk=walk, cursor=cursor,
1314 skip=skip, queue=queue)
1315
1338
1316 if not(pathList):
1339 # sys.exit(-1)
1317 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1318 # datetime.datetime.combine(startDate,startTime).ctime(),
1319 # datetime.datetime.combine(endDate,endTime).ctime())
1320
1340
1321 # sys.exit(-1)
1341 self.fileIndex = -1
1342 self.pathList = []
1343 self.filenameList = []
1344 return
1322
1345
1323 self.fileIndex = -1
1346 self.fileIndex = -1
1324 self.pathList = []
1347 self.pathList = pathList
1325 self.filenameList = []
1348 self.filenameList = filenameList
1326 return
1349 file_name = os.path.basename(filenameList[-1])
1327
1350 basename, ext = os.path.splitext(file_name)
1328 self.fileIndex = -1
1351 last_set = int(basename[-3:])
1329 self.pathList = pathList
1352
1330 self.filenameList = filenameList
1353 self.online = online
1331 file_name = os.path.basename(filenameList[-1])
1354 self.realtime = realtime
1332 basename, ext = os.path.splitext(file_name)
1355 self.delay = delay
1333 last_set = int(basename[-3:])
1356 ext = ext.lower()
1334
1357 self.ext = ext
1335 self.online = online
1358 self.getByBlock = getblock
1336 self.realtime = realtime
1359 self.nTxs = nTxs
1337 self.delay = delay
1360 self.startTime = startTime
1338 ext = ext.lower()
1361 self.endTime = endTime
1339 self.ext = ext
1362
1340 self.getByBlock = getblock
1363 #Added-----------------
1341 self.nTxs = nTxs
1364 self.selBlocksize = blocksize
1342 self.startTime = startTime
1365 self.selBlocktime = blocktime
1343 self.endTime = endTime
1366
1344
1367 # Verbose-----------
1345 #Added-----------------
1368 self.verbose = verbose
1346 self.selBlocksize = blocksize
1369 self.warnings = warnings
1347 self.selBlocktime = blocktime
1348
1349 # Verbose-----------
1350 self.verbose = verbose
1351 self.warnings = warnings
1352
1370
1353 if not(self.setNextFile()):
1371 if not(self.setNextFile()):
1354 if (startDate!=None) and (endDate!=None):
1372 if (startDate!=None) and (endDate!=None):
1355 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1373 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1356 elif startDate != None:
1374 elif startDate != None:
1357 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1375 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1358 else:
1376 else:
1359 print "[Reading] No files"
1377 print "[Reading] No files"
1360
1378
1361 self.fileIndex = -1
1379 self.fileIndex = -1
1362 self.pathList = []
1380 self.pathList = []
1363 self.filenameList = []
1381 self.filenameList = []
1364 return
1382 return
1365
1383
1366 # self.getBasicHeader()
1384 # self.getBasicHeader()
1367
1385
1368 if last_set != None:
1386 if last_set != None:
1369 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1387 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1370 return
1388 return
1371
1389
1372 def getBasicHeader(self):
1390 def getBasicHeader(self):
1373
1391
1374 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1392 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1375
1393
1376 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1394 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1377
1395
1378 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1396 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1379
1397
1380 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1398 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1381
1399
1382 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1400 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1383
1401
1384 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1402 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1385
1403
1386 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1404 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1387
1405
1388 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1406 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1389
1407
1390
1408
1391 def getFirstHeader(self):
1409 def getFirstHeader(self):
1392
1410
1393 raise NotImplementedError
1411 raise NotImplementedError
1394
1412
1395 def getData(self):
1413 def getData(self):
1396
1414
1397 raise NotImplementedError
1415 raise NotImplementedError
1398
1416
1399 def hasNotDataInBuffer(self):
1417 def hasNotDataInBuffer(self):
1400
1418
1401 raise NotImplementedError
1419 raise NotImplementedError
1402
1420
1403 def readBlock(self):
1421 def readBlock(self):
1404
1422
1405 raise NotImplementedError
1423 raise NotImplementedError
1406
1424
1407 def isEndProcess(self):
1425 def isEndProcess(self):
1408
1426
1409 return self.flagNoMoreFiles
1427 return self.flagNoMoreFiles
1410
1428
1411 def printReadBlocks(self):
1429 def printReadBlocks(self):
1412
1430
1413 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1431 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1414
1432
1415 def printTotalBlocks(self):
1433 def printTotalBlocks(self):
1416
1434
1417 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1435 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1418
1436
1419 def printNumberOfBlock(self):
1437 def printNumberOfBlock(self):
1420
1438
1421 if self.flagIsNewBlock:
1439 if self.flagIsNewBlock:
1422 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1440 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1423 self.processingHeaderObj.dataBlocksPerFile,
1441 self.processingHeaderObj.dataBlocksPerFile,
1424 self.dataOut.datatime.ctime())
1442 self.dataOut.datatime.ctime())
1425
1443
1426 def printInfo(self):
1444 def printInfo(self):
1427
1445
1428 if self.__printInfo == False:
1446 if self.__printInfo == False:
1429 return
1447 return
1430
1448
1431 self.basicHeaderObj.printInfo()
1449 self.basicHeaderObj.printInfo()
1432 self.systemHeaderObj.printInfo()
1450 self.systemHeaderObj.printInfo()
1433 self.radarControllerHeaderObj.printInfo()
1451 self.radarControllerHeaderObj.printInfo()
1434 self.processingHeaderObj.printInfo()
1452 self.processingHeaderObj.printInfo()
1435
1453
1436 self.__printInfo = False
1454 self.__printInfo = False
1437
1455
1438
1456
1439 def run(self,
1457 def run(self,
1440 path=None,
1458 path=None,
1441 startDate=None,
1459 startDate=None,
1442 endDate=None,
1460 endDate=None,
1443 startTime=datetime.time(0,0,0),
1461 startTime=datetime.time(0,0,0),
1444 endTime=datetime.time(23,59,59),
1462 endTime=datetime.time(23,59,59),
1445 set=None,
1463 set=None,
1446 expLabel = "",
1464 expLabel = "",
1447 ext = None,
1465 ext = None,
1448 online = False,
1466 online = False,
1449 delay = 60,
1467 delay = 60,
1450 walk = True,
1468 walk = True,
1451 getblock = False,
1469 getblock = False,
1452 nTxs = 1,
1470 nTxs = 1,
1453 realtime=False,
1471 realtime=False,
1454 blocksize=None,
1472 blocksize=None,
1455 blocktime=None,
1473 blocktime=None,
1456 queue=None,
1474 queue=None,
1457 skip=None,
1475 skip=None,
1458 cursor=None,
1476 cursor=None,
1459 warnings=True,
1477 warnings=True,
1478 server=None,
1460 verbose=True, **kwargs):
1479 verbose=True, **kwargs):
1461
1480
1462 if not(self.isConfig):
1481 if not(self.isConfig):
1463 # self.dataOut = dataOut
1482 # self.dataOut = dataOut
1464 self.setup( path=path,
1483 self.setup( path=path,
1465 startDate=startDate,
1484 startDate=startDate,
1466 endDate=endDate,
1485 endDate=endDate,
1467 startTime=startTime,
1486 startTime=startTime,
1468 endTime=endTime,
1487 endTime=endTime,
1469 set=set,
1488 set=set,
1470 expLabel=expLabel,
1489 expLabel=expLabel,
1471 ext=ext,
1490 ext=ext,
1472 online=online,
1491 online=online,
1473 delay=delay,
1492 delay=delay,
1474 walk=walk,
1493 walk=walk,
1475 getblock=getblock,
1494 getblock=getblock,
1476 nTxs=nTxs,
1495 nTxs=nTxs,
1477 realtime=realtime,
1496 realtime=realtime,
1478 blocksize=blocksize,
1497 blocksize=blocksize,
1479 blocktime=blocktime,
1498 blocktime=blocktime,
1480 queue=queue,
1499 queue=queue,
1481 skip=skip,
1500 skip=skip,
1482 cursor=cursor,
1501 cursor=cursor,
1483 warnings=warnings,
1502 warnings=warnings,
1503 server=server,
1484 verbose=verbose)
1504 verbose=verbose)
1485 self.isConfig = True
1505 self.isConfig = True
1486
1506 if server is None:
1487 self.getData()
1507 self.getData()
1508 else:
1509 self.getFromServer()
1488
1510
1489 class JRODataWriter(JRODataIO):
1511 class JRODataWriter(JRODataIO):
1490
1512
1491 """
1513 """
1492 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1514 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1493 de los datos siempre se realiza por bloques.
1515 de los datos siempre se realiza por bloques.
1494 """
1516 """
1495
1517
1496 blockIndex = 0
1518 blockIndex = 0
1497
1519
1498 path = None
1520 path = None
1499
1521
1500 setFile = None
1522 setFile = None
1501
1523
1502 profilesPerBlock = None
1524 profilesPerBlock = None
1503
1525
1504 blocksPerFile = None
1526 blocksPerFile = None
1505
1527
1506 nWriteBlocks = 0
1528 nWriteBlocks = 0
1507
1529
1508 fileDate = None
1530 fileDate = None
1509
1531
1510 def __init__(self, dataOut=None):
1532 def __init__(self, dataOut=None):
1511 raise NotImplementedError
1533 raise NotImplementedError
1512
1534
1513
1535
1514 def hasAllDataInBuffer(self):
1536 def hasAllDataInBuffer(self):
1515 raise NotImplementedError
1537 raise NotImplementedError
1516
1538
1517
1539
1518 def setBlockDimension(self):
1540 def setBlockDimension(self):
1519 raise NotImplementedError
1541 raise NotImplementedError
1520
1542
1521
1543
1522 def writeBlock(self):
1544 def writeBlock(self):
1523 raise NotImplementedError
1545 raise NotImplementedError
1524
1546
1525
1547
1526 def putData(self):
1548 def putData(self):
1527 raise NotImplementedError
1549 raise NotImplementedError
1528
1550
1529
1551
1530 def getProcessFlags(self):
1552 def getProcessFlags(self):
1531
1553
1532 processFlags = 0
1554 processFlags = 0
1533
1555
1534 dtype_index = get_dtype_index(self.dtype)
1556 dtype_index = get_dtype_index(self.dtype)
1535 procflag_dtype = get_procflag_dtype(dtype_index)
1557 procflag_dtype = get_procflag_dtype(dtype_index)
1536
1558
1537 processFlags += procflag_dtype
1559 processFlags += procflag_dtype
1538
1560
1539 if self.dataOut.flagDecodeData:
1561 if self.dataOut.flagDecodeData:
1540 processFlags += PROCFLAG.DECODE_DATA
1562 processFlags += PROCFLAG.DECODE_DATA
1541
1563
1542 if self.dataOut.flagDeflipData:
1564 if self.dataOut.flagDeflipData:
1543 processFlags += PROCFLAG.DEFLIP_DATA
1565 processFlags += PROCFLAG.DEFLIP_DATA
1544
1566
1545 if self.dataOut.code is not None:
1567 if self.dataOut.code is not None:
1546 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1568 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1547
1569
1548 if self.dataOut.nCohInt > 1:
1570 if self.dataOut.nCohInt > 1:
1549 processFlags += PROCFLAG.COHERENT_INTEGRATION
1571 processFlags += PROCFLAG.COHERENT_INTEGRATION
1550
1572
1551 if self.dataOut.type == "Spectra":
1573 if self.dataOut.type == "Spectra":
1552 if self.dataOut.nIncohInt > 1:
1574 if self.dataOut.nIncohInt > 1:
1553 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1575 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1554
1576
1555 if self.dataOut.data_dc is not None:
1577 if self.dataOut.data_dc is not None:
1556 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1578 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1557
1579
1558 if self.dataOut.flagShiftFFT:
1580 if self.dataOut.flagShiftFFT:
1559 processFlags += PROCFLAG.SHIFT_FFT_DATA
1581 processFlags += PROCFLAG.SHIFT_FFT_DATA
1560
1582
1561 return processFlags
1583 return processFlags
1562
1584
1563 def setBasicHeader(self):
1585 def setBasicHeader(self):
1564
1586
1565 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1587 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1566 self.basicHeaderObj.version = self.versionFile
1588 self.basicHeaderObj.version = self.versionFile
1567 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1589 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1568
1590
1569 utc = numpy.floor(self.dataOut.utctime)
1591 utc = numpy.floor(self.dataOut.utctime)
1570 milisecond = (self.dataOut.utctime - utc)* 1000.0
1592 milisecond = (self.dataOut.utctime - utc)* 1000.0
1571
1593
1572 self.basicHeaderObj.utc = utc
1594 self.basicHeaderObj.utc = utc
1573 self.basicHeaderObj.miliSecond = milisecond
1595 self.basicHeaderObj.miliSecond = milisecond
1574 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1596 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1575 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1597 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1576 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1598 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1577
1599
1578 def setFirstHeader(self):
1600 def setFirstHeader(self):
1579 """
1601 """
1580 Obtiene una copia del First Header
1602 Obtiene una copia del First Header
1581
1603
1582 Affected:
1604 Affected:
1583
1605
1584 self.basicHeaderObj
1606 self.basicHeaderObj
1585 self.systemHeaderObj
1607 self.systemHeaderObj
1586 self.radarControllerHeaderObj
1608 self.radarControllerHeaderObj
1587 self.processingHeaderObj self.
1609 self.processingHeaderObj self.
1588
1610
1589 Return:
1611 Return:
1590 None
1612 None
1591 """
1613 """
1592
1614
1593 raise NotImplementedError
1615 raise NotImplementedError
1594
1616
1595 def __writeFirstHeader(self):
1617 def __writeFirstHeader(self):
1596 """
1618 """
1597 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1619 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1598
1620
1599 Affected:
1621 Affected:
1600 __dataType
1622 __dataType
1601
1623
1602 Return:
1624 Return:
1603 None
1625 None
1604 """
1626 """
1605
1627
1606 # CALCULAR PARAMETROS
1628 # CALCULAR PARAMETROS
1607
1629
1608 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1630 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1609 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1631 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1610
1632
1611 self.basicHeaderObj.write(self.fp)
1633 self.basicHeaderObj.write(self.fp)
1612 self.systemHeaderObj.write(self.fp)
1634 self.systemHeaderObj.write(self.fp)
1613 self.radarControllerHeaderObj.write(self.fp)
1635 self.radarControllerHeaderObj.write(self.fp)
1614 self.processingHeaderObj.write(self.fp)
1636 self.processingHeaderObj.write(self.fp)
1615
1637
1616 def __setNewBlock(self):
1638 def __setNewBlock(self):
1617 """
1639 """
1618 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1640 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1619
1641
1620 Return:
1642 Return:
1621 0 : si no pudo escribir nada
1643 0 : si no pudo escribir nada
1622 1 : Si escribio el Basic el First Header
1644 1 : Si escribio el Basic el First Header
1623 """
1645 """
1624 if self.fp == None:
1646 if self.fp == None:
1625 self.setNextFile()
1647 self.setNextFile()
1626
1648
1627 if self.flagIsNewFile:
1649 if self.flagIsNewFile:
1628 return 1
1650 return 1
1629
1651
1630 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1652 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1631 self.basicHeaderObj.write(self.fp)
1653 self.basicHeaderObj.write(self.fp)
1632 return 1
1654 return 1
1633
1655
1634 if not( self.setNextFile() ):
1656 if not( self.setNextFile() ):
1635 return 0
1657 return 0
1636
1658
1637 return 1
1659 return 1
1638
1660
1639
1661
1640 def writeNextBlock(self):
1662 def writeNextBlock(self):
1641 """
1663 """
1642 Selecciona el bloque siguiente de datos y los escribe en un file
1664 Selecciona el bloque siguiente de datos y los escribe en un file
1643
1665
1644 Return:
1666 Return:
1645 0 : Si no hizo pudo escribir el bloque de datos
1667 0 : Si no hizo pudo escribir el bloque de datos
1646 1 : Si no pudo escribir el bloque de datos
1668 1 : Si no pudo escribir el bloque de datos
1647 """
1669 """
1648 if not( self.__setNewBlock() ):
1670 if not( self.__setNewBlock() ):
1649 return 0
1671 return 0
1650
1672
1651 self.writeBlock()
1673 self.writeBlock()
1652
1674
1653 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1675 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1654 self.processingHeaderObj.dataBlocksPerFile)
1676 self.processingHeaderObj.dataBlocksPerFile)
1655
1677
1656 return 1
1678 return 1
1657
1679
1658 def setNextFile(self):
1680 def setNextFile(self):
1659 """
1681 """
1660 Determina el siguiente file que sera escrito
1682 Determina el siguiente file que sera escrito
1661
1683
1662 Affected:
1684 Affected:
1663 self.filename
1685 self.filename
1664 self.subfolder
1686 self.subfolder
1665 self.fp
1687 self.fp
1666 self.setFile
1688 self.setFile
1667 self.flagIsNewFile
1689 self.flagIsNewFile
1668
1690
1669 Return:
1691 Return:
1670 0 : Si el archivo no puede ser escrito
1692 0 : Si el archivo no puede ser escrito
1671 1 : Si el archivo esta listo para ser escrito
1693 1 : Si el archivo esta listo para ser escrito
1672 """
1694 """
1673 ext = self.ext
1695 ext = self.ext
1674 path = self.path
1696 path = self.path
1675
1697
1676 if self.fp != None:
1698 if self.fp != None:
1677 self.fp.close()
1699 self.fp.close()
1678
1700
1679 timeTuple = time.localtime( self.dataOut.utctime)
1701 timeTuple = time.localtime( self.dataOut.utctime)
1680 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1702 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1681
1703
1682 fullpath = os.path.join( path, subfolder )
1704 fullpath = os.path.join( path, subfolder )
1683 setFile = self.setFile
1705 setFile = self.setFile
1684
1706
1685 if not( os.path.exists(fullpath) ):
1707 if not( os.path.exists(fullpath) ):
1686 os.mkdir(fullpath)
1708 os.mkdir(fullpath)
1687 setFile = -1 #inicializo mi contador de seteo
1709 setFile = -1 #inicializo mi contador de seteo
1688 else:
1710 else:
1689 filesList = os.listdir( fullpath )
1711 filesList = os.listdir( fullpath )
1690 if len( filesList ) > 0:
1712 if len( filesList ) > 0:
1691 filesList = sorted( filesList, key=str.lower )
1713 filesList = sorted( filesList, key=str.lower )
1692 filen = filesList[-1]
1714 filen = filesList[-1]
1693 # el filename debera tener el siguiente formato
1715 # el filename debera tener el siguiente formato
1694 # 0 1234 567 89A BCDE (hex)
1716 # 0 1234 567 89A BCDE (hex)
1695 # x YYYY DDD SSS .ext
1717 # x YYYY DDD SSS .ext
1696 if isNumber( filen[8:11] ):
1718 if isNumber( filen[8:11] ):
1697 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1719 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1698 else:
1720 else:
1699 setFile = -1
1721 setFile = -1
1700 else:
1722 else:
1701 setFile = -1 #inicializo mi contador de seteo
1723 setFile = -1 #inicializo mi contador de seteo
1702
1724
1703 setFile += 1
1725 setFile += 1
1704
1726
1705 #If this is a new day it resets some values
1727 #If this is a new day it resets some values
1706 if self.dataOut.datatime.date() > self.fileDate:
1728 if self.dataOut.datatime.date() > self.fileDate:
1707 setFile = 0
1729 setFile = 0
1708 self.nTotalBlocks = 0
1730 self.nTotalBlocks = 0
1709
1731
1710 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1732 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1711
1733
1712 filename = os.path.join( path, subfolder, filen )
1734 filename = os.path.join( path, subfolder, filen )
1713
1735
1714 fp = open( filename,'wb' )
1736 fp = open( filename,'wb' )
1715
1737
1716 self.blockIndex = 0
1738 self.blockIndex = 0
1717
1739
1718 #guardando atributos
1740 #guardando atributos
1719 self.filename = filename
1741 self.filename = filename
1720 self.subfolder = subfolder
1742 self.subfolder = subfolder
1721 self.fp = fp
1743 self.fp = fp
1722 self.setFile = setFile
1744 self.setFile = setFile
1723 self.flagIsNewFile = 1
1745 self.flagIsNewFile = 1
1724 self.fileDate = self.dataOut.datatime.date()
1746 self.fileDate = self.dataOut.datatime.date()
1725
1747
1726 self.setFirstHeader()
1748 self.setFirstHeader()
1727
1749
1728 print '[Writing] Opening file: %s'%self.filename
1750 print '[Writing] Opening file: %s'%self.filename
1729
1751
1730 self.__writeFirstHeader()
1752 self.__writeFirstHeader()
1731
1753
1732 return 1
1754 return 1
1733
1755
1734 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1756 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1735 """
1757 """
1736 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1758 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1737
1759
1738 Inputs:
1760 Inputs:
1739 path : directory where data will be saved
1761 path : directory where data will be saved
1740 profilesPerBlock : number of profiles per block
1762 profilesPerBlock : number of profiles per block
1741 set : initial file set
1763 set : initial file set
1742 datatype : An integer number that defines data type:
1764 datatype : An integer number that defines data type:
1743 0 : int8 (1 byte)
1765 0 : int8 (1 byte)
1744 1 : int16 (2 bytes)
1766 1 : int16 (2 bytes)
1745 2 : int32 (4 bytes)
1767 2 : int32 (4 bytes)
1746 3 : int64 (8 bytes)
1768 3 : int64 (8 bytes)
1747 4 : float32 (4 bytes)
1769 4 : float32 (4 bytes)
1748 5 : double64 (8 bytes)
1770 5 : double64 (8 bytes)
1749
1771
1750 Return:
1772 Return:
1751 0 : Si no realizo un buen seteo
1773 0 : Si no realizo un buen seteo
1752 1 : Si realizo un buen seteo
1774 1 : Si realizo un buen seteo
1753 """
1775 """
1754
1776
1755 if ext == None:
1777 if ext == None:
1756 ext = self.ext
1778 ext = self.ext
1757
1779
1758 self.ext = ext.lower()
1780 self.ext = ext.lower()
1759
1781
1760 self.path = path
1782 self.path = path
1761
1783
1762 if set is None:
1784 if set is None:
1763 self.setFile = -1
1785 self.setFile = -1
1764 else:
1786 else:
1765 self.setFile = set - 1
1787 self.setFile = set - 1
1766
1788
1767 self.blocksPerFile = blocksPerFile
1789 self.blocksPerFile = blocksPerFile
1768
1790
1769 self.profilesPerBlock = profilesPerBlock
1791 self.profilesPerBlock = profilesPerBlock
1770
1792
1771 self.dataOut = dataOut
1793 self.dataOut = dataOut
1772 self.fileDate = self.dataOut.datatime.date()
1794 self.fileDate = self.dataOut.datatime.date()
1773 #By default
1795 #By default
1774 self.dtype = self.dataOut.dtype
1796 self.dtype = self.dataOut.dtype
1775
1797
1776 if datatype is not None:
1798 if datatype is not None:
1777 self.dtype = get_numpy_dtype(datatype)
1799 self.dtype = get_numpy_dtype(datatype)
1778
1800
1779 if not(self.setNextFile()):
1801 if not(self.setNextFile()):
1780 print "[Writing] There isn't a next file"
1802 print "[Writing] There isn't a next file"
1781 return 0
1803 return 0
1782
1804
1783 self.setBlockDimension()
1805 self.setBlockDimension()
1784
1806
1785 return 1
1807 return 1
1786
1808
1787 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1809 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1788
1810
1789 if not(self.isConfig):
1811 if not(self.isConfig):
1790
1812
1791 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1813 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1792 self.isConfig = True
1814 self.isConfig = True
1793
1815
1794 self.putData()
1816 self.putData()
Show file before | Show file after | Hide comments
@@ -1,637 +1,739
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6
6
7 import numpy
7 import numpy
8
8
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13 import zmq
14 import tempfile
15 from StringIO import StringIO
13 # from _sha import blocksize
16 # from _sha import blocksize
14
17
15 class VoltageReader(JRODataReader, ProcessingUnit):
18 class VoltageReader(JRODataReader, ProcessingUnit):
16 """
19 """
17 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
18 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
21 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
19 perfiles*alturas*canales) son almacenados en la variable "buffer".
22 perfiles*alturas*canales) son almacenados en la variable "buffer".
20
23
21 perfiles * alturas * canales
24 perfiles * alturas * canales
22
25
23 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
26 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
24 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
27 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
25 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
28 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
26 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
29 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
27
30
28 Example:
31 Example:
29
32
30 dpath = "/home/myuser/data"
33 dpath = "/home/myuser/data"
31
34
32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
35 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
33
36
34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
37 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
35
38
36 readerObj = VoltageReader()
39 readerObj = VoltageReader()
37
40
38 readerObj.setup(dpath, startTime, endTime)
41 readerObj.setup(dpath, startTime, endTime)
39
42
40 while(True):
43 while(True):
41
44
42 #to get one profile
45 #to get one profile
43 profile = readerObj.getData()
46 profile = readerObj.getData()
44
47
45 #print the profile
48 #print the profile
46 print profile
49 print profile
47
50
48 #If you want to see all datablock
51 #If you want to see all datablock
49 print readerObj.datablock
52 print readerObj.datablock
50
53
51 if readerObj.flagNoMoreFiles:
54 if readerObj.flagNoMoreFiles:
52 break
55 break
53
56
54 """
57 """
55
58
56 ext = ".r"
59 ext = ".r"
57
60
58 optchar = "D"
61 optchar = "D"
59 dataOut = None
62 dataOut = None
60
63
61 def __init__(self, **kwargs):
64 def __init__(self, **kwargs):
62 """
65 """
63 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
66 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
64
67
65 Input:
68 Input:
66 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
69 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
67 almacenar un perfil de datos cada vez que se haga un requerimiento
70 almacenar un perfil de datos cada vez que se haga un requerimiento
68 (getData). El perfil sera obtenido a partir del buffer de datos,
71 (getData). El perfil sera obtenido a partir del buffer de datos,
69 si el buffer esta vacio se hara un nuevo proceso de lectura de un
72 si el buffer esta vacio se hara un nuevo proceso de lectura de un
70 bloque de datos.
73 bloque de datos.
71 Si este parametro no es pasado se creara uno internamente.
74 Si este parametro no es pasado se creara uno internamente.
72
75
73 Variables afectadas:
76 Variables afectadas:
74 self.dataOut
77 self.dataOut
75
78
76 Return:
79 Return:
77 None
80 None
78 """
81 """
79
82
80 ProcessingUnit.__init__(self, **kwargs)
83 ProcessingUnit.__init__(self, **kwargs)
81
84
82 self.isConfig = False
85 self.isConfig = False
83
86
84 self.datablock = None
87 self.datablock = None
85
88
86 self.utc = 0
89 self.utc = 0
87
90
88 self.ext = ".r"
91 self.ext = ".r"
89
92
90 self.optchar = "D"
93 self.optchar = "D"
91
94
92 self.basicHeaderObj = BasicHeader(LOCALTIME)
95 self.basicHeaderObj = BasicHeader(LOCALTIME)
93
96
94 self.systemHeaderObj = SystemHeader()
97 self.systemHeaderObj = SystemHeader()
95
98
96 self.radarControllerHeaderObj = RadarControllerHeader()
99 self.radarControllerHeaderObj = RadarControllerHeader()
97
100
98 self.processingHeaderObj = ProcessingHeader()
101 self.processingHeaderObj = ProcessingHeader()
99
102
100 self.online = 0
103 self.online = 0
101
104
102 self.fp = None
105 self.fp = None
103
106
104 self.idFile = None
107 self.idFile = None
105
108
106 self.dtype = None
109 self.dtype = None
107
110
108 self.fileSizeByHeader = None
111 self.fileSizeByHeader = None
109
112
110 self.filenameList = []
113 self.filenameList = []
111
114
112 self.filename = None
115 self.filename = None
113
116
114 self.fileSize = None
117 self.fileSize = None
115
118
116 self.firstHeaderSize = 0
119 self.firstHeaderSize = 0
117
120
118 self.basicHeaderSize = 24
121 self.basicHeaderSize = 24
119
122
120 self.pathList = []
123 self.pathList = []
121
124
122 self.filenameList = []
125 self.filenameList = []
123
126
124 self.lastUTTime = 0
127 self.lastUTTime = 0
125
128
126 self.maxTimeStep = 30
129 self.maxTimeStep = 30
127
130
128 self.flagNoMoreFiles = 0
131 self.flagNoMoreFiles = 0
129
132
130 self.set = 0
133 self.set = 0
131
134
132 self.path = None
135 self.path = None
133
136
134 self.profileIndex = 2**32-1
137 self.profileIndex = 2**32-1
135
138
136 self.delay = 3 #seconds
139 self.delay = 3 #seconds
137
140
138 self.nTries = 3 #quantity tries
141 self.nTries = 3 #quantity tries
139
142
140 self.nFiles = 3 #number of files for searching
143 self.nFiles = 3 #number of files for searching
141
144
142 self.nReadBlocks = 0
145 self.nReadBlocks = 0
143
146
144 self.flagIsNewFile = 1
147 self.flagIsNewFile = 1
145
148
146 self.__isFirstTimeOnline = 1
149 self.__isFirstTimeOnline = 1
147
150
148 # self.ippSeconds = 0
151 # self.ippSeconds = 0
149
152
150 self.flagDiscontinuousBlock = 0
153 self.flagDiscontinuousBlock = 0
151
154
152 self.flagIsNewBlock = 0
155 self.flagIsNewBlock = 0
153
156
154 self.nTotalBlocks = 0
157 self.nTotalBlocks = 0
155
158
156 self.blocksize = 0
159 self.blocksize = 0
157
160
158 self.dataOut = self.createObjByDefault()
161 self.dataOut = self.createObjByDefault()
159
162
160 self.nTxs = 1
163 self.nTxs = 1
161
164
162 self.txIndex = 0
165 self.txIndex = 0
163
166
164 def createObjByDefault(self):
167 def createObjByDefault(self):
165
168
166 dataObj = Voltage()
169 dataObj = Voltage()
167
170
168 return dataObj
171 return dataObj
169
172
170 def __hasNotDataInBuffer(self):
173 def __hasNotDataInBuffer(self):
171
174
172 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
173 return 1
176 return 1
174
177
175 return 0
178 return 0
176
179
177
180
178 def getBlockDimension(self):
181 def getBlockDimension(self):
179 """
182 """
180 Obtiene la cantidad de puntos a leer por cada bloque de datos
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
181
184
182 Affected:
185 Affected:
183 self.blocksize
186 self.blocksize
184
187
185 Return:
188 Return:
186 None
189 None
187 """
190 """
188 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
191 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
189 self.blocksize = pts2read
192 self.blocksize = pts2read
190
193
194
191
195
192 def readBlock(self):
196 def readBlock(self):
193 """
197 """
194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
195 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
199 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
196 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
200 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
197 es seteado a 0
201 es seteado a 0
198
202
199 Inputs:
203 Inputs:
200 None
204 None
201
205
202 Return:
206 Return:
203 None
207 None
204
208
205 Affected:
209 Affected:
206 self.profileIndex
210 self.profileIndex
207 self.datablock
211 self.datablock
208 self.flagIsNewFile
212 self.flagIsNewFile
209 self.flagIsNewBlock
213 self.flagIsNewBlock
210 self.nTotalBlocks
214 self.nTotalBlocks
211
215
212 Exceptions:
216 Exceptions:
213 Si un bloque leido no es un bloque valido
217 Si un bloque leido no es un bloque valido
214 """
218 """
219
220 print 'READ BLOCK'
221 # if self.server is not None:
222 # self.zBlock = self.receiver.recv()
223 # self.zHeader = self.zBlock[:24]
224 # self.zDataBlock = self.zBlock[24:]
225 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
226 # self.processingHeaderObj.profilesPerBlock = 240
227 # self.processingHeaderObj.nHeights = 248
228 # self.systemHeaderObj.nChannels
229 # else:
215 current_pointer_location = self.fp.tell()
230 current_pointer_location = self.fp.tell()
216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
231 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217
232
218 try:
233 try:
219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
234 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
235 print'junked'
220 except:
236 except:
221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
237 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222
238
223 if self.waitDataBlock(pointer_location=current_pointer_location):
239 if self.waitDataBlock(pointer_location=current_pointer_location):
224 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
240 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
225 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
241 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
226 # return 0
242 # return 0
227
243
228 #Dimensions : nChannels, nProfiles, nSamples
244 #Dimensions : nChannels, nProfiles, nSamples
229
245
230 junk = numpy.transpose(junk, (2,0,1))
246 junk = numpy.transpose(junk, (2,0,1))
231 self.datablock = junk['real'] + junk['imag']*1j
247 self.datablock = junk['real'] + junk['imag']*1j
232
248
233 self.profileIndex = 0
249 self.profileIndex = 0
234
250
235 self.flagIsNewFile = 0
251 self.flagIsNewFile = 0
236 self.flagIsNewBlock = 1
252 self.flagIsNewBlock = 1
237
253
238 self.nTotalBlocks += 1
254 self.nTotalBlocks += 1
239 self.nReadBlocks += 1
255 self.nReadBlocks += 1
240
256
241 return 1
257 return 1
242
258
243 def getFirstHeader(self):
259 def getFirstHeader(self):
244
260
245 self.getBasicHeader()
261 self.getBasicHeader()
246
262
247 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
263 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
248
264
249 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
265 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
250
266
251 if self.nTxs > 1:
267 if self.nTxs > 1:
252 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
268 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
253
269
254 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
270 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
255
271
256 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
272 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
257 #
273 #
258 # if self.radarControllerHeaderObj.code is not None:
274 # if self.radarControllerHeaderObj.code is not None:
259 #
275 #
260 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
276 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
261 #
277 #
262 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
278 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
263 #
279 #
264 # self.dataOut.code = self.radarControllerHeaderObj.code
280 # self.dataOut.code = self.radarControllerHeaderObj.code
265
281
266 self.dataOut.dtype = self.dtype
282 self.dataOut.dtype = self.dtype
267
283
268 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
284 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
269
285
270 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
286 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
271
287
272 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
288 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
273
289
274 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
290 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
275
291
276 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
292 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
277
293
278 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
294 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
279
295
280 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
296 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
281
297
282 def reshapeData(self):
298 def reshapeData(self):
283
299
284 if self.nTxs < 0:
300 if self.nTxs < 0:
285 return
301 return
286
302
287 if self.nTxs == 1:
303 if self.nTxs == 1:
288 return
304 return
289
305
290 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
306 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
291 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
307 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
292
308
293 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
309 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
294 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
310 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
295
311
296 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
312 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
297
313
298 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
314 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
299 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
315 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
300 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
316 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
301
317
302 return
318 return
303
319
320 def readFirstHeaderFromServer(self):
321
322 self.getFirstHeader()
323
324 self.firstHeaderSize = self.basicHeaderObj.size
325
326 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
327 if datatype == 0:
328 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
329 elif datatype == 1:
330 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
331 elif datatype == 2:
332 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
333 elif datatype == 3:
334 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
335 elif datatype == 4:
336 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
337 elif datatype == 5:
338 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
339 else:
340 raise ValueError, 'Data type was not defined'
341
342 self.dtype = datatype_str
343 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
344 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
345 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
346 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
347 self.getBlockDimension()
348
349
350 def getFromServer(self):
351 self.flagDiscontinuousBlock = 0
352 self.profileIndex = 0
353 self.flagIsNewBlock = 1
354 self.dataOut.flagNoData = False
355 self.nTotalBlocks += 1
356 self.nReadBlocks += 1
357 self.blockPointer = 0
358
359 block = self.receiver.recv()
360
361 self.basicHeaderObj.read(block[self.blockPointer:])
362 self.blockPointer += self.basicHeaderObj.length
363 self.systemHeaderObj.read(block[self.blockPointer:])
364 self.blockPointer += self.systemHeaderObj.length
365 self.radarControllerHeaderObj.read(block[self.blockPointer:])
366 self.blockPointer += self.radarControllerHeaderObj.length
367 self.processingHeaderObj.read(block[self.blockPointer:])
368 self.blockPointer += self.processingHeaderObj.length
369 self.readFirstHeaderFromServer()
370
371 timestamp = self.basicHeaderObj.get_datatime()
372 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
373 current_pointer_location = self.blockPointer
374 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
375
376 try:
377 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
378 except:
379 #print "The read block (%3d) has not enough data" %self.nReadBlocks
380 if self.waitDataBlock(pointer_location=current_pointer_location):
381 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
382 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
383 # return 0
384
385 #Dimensions : nChannels, nProfiles, nSamples
386
387 junk = numpy.transpose(junk, (2,0,1))
388 self.datablock = junk['real'] + junk['imag'] * 1j
389 self.profileIndex = 0
390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 if self.selBlocktime != None:
392 if self.dataOut.nCohInt is not None:
393 nCohInt = self.dataOut.nCohInt
394 else:
395 nCohInt = 1
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
398 datasize = self.dataOut.data.shape[1]
399 if datasize < self.selBlocksize:
400 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
401 buffer[:,:datasize,:] = self.dataOut.data
402 self.dataOut.data = buffer
403 self.profileIndex = blockIndex
404
405 self.dataOut.flagDataAsBlock = True
406 self.flagIsNewBlock = 1
407 self.dataOut.realtime = self.online
408
409 return self.dataOut.data
410
304 def getData(self):
411 def getData(self):
305 """
412 """
306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
413 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
307 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
414 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
308 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
415 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
309 "readNextBlock"
416 "readNextBlock"
310
417
311 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
418 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
312
419
313 Return:
420 Return:
314
421
315 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
422 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
316 es igual al total de perfiles leidos desde el archivo.
423 es igual al total de perfiles leidos desde el archivo.
317
424
318 Si self.getByBlock == False:
425 Si self.getByBlock == False:
319
426
320 self.dataOut.data = buffer[:, thisProfile, :]
427 self.dataOut.data = buffer[:, thisProfile, :]
321
428
322 shape = [nChannels, nHeis]
429 shape = [nChannels, nHeis]
323
430
324 Si self.getByBlock == True:
431 Si self.getByBlock == True:
325
432
326 self.dataOut.data = buffer[:, :, :]
433 self.dataOut.data = buffer[:, :, :]
327
434
328 shape = [nChannels, nProfiles, nHeis]
435 shape = [nChannels, nProfiles, nHeis]
329
436
330 Variables afectadas:
437 Variables afectadas:
331 self.dataOut
438 self.dataOut
332 self.profileIndex
439 self.profileIndex
333
440
334 Affected:
441 Affected:
335 self.dataOut
442 self.dataOut
336 self.profileIndex
443 self.profileIndex
337 self.flagDiscontinuousBlock
444 self.flagDiscontinuousBlock
338 self.flagIsNewBlock
445 self.flagIsNewBlock
339 """
446 """
340
341 if self.flagNoMoreFiles:
447 if self.flagNoMoreFiles:
342 self.dataOut.flagNoData = True
448 self.dataOut.flagNoData = True
343 print 'Process finished'
449 print 'Process finished'
344 return 0
450 return 0
345
346 self.flagDiscontinuousBlock = 0
451 self.flagDiscontinuousBlock = 0
347 self.flagIsNewBlock = 0
452 self.flagIsNewBlock = 0
348
349 if self.__hasNotDataInBuffer():
453 if self.__hasNotDataInBuffer():
350
351 if not( self.readNextBlock() ):
454 if not( self.readNextBlock() ):
352 return 0
455 return 0
353
456
354 self.getFirstHeader()
457 self.getFirstHeader()
355
458
356 self.reshapeData()
459 self.reshapeData()
357
358 if self.datablock is None:
460 if self.datablock is None:
359 self.dataOut.flagNoData = True
461 self.dataOut.flagNoData = True
360 return 0
462 return 0
361
463
362 if not self.getByBlock:
464 if not self.getByBlock:
363
465
364 """
466 """
365 Return profile by profile
467 Return profile by profile
366
468
367 If nTxs > 1 then one profile is divided by nTxs and number of total
469 If nTxs > 1 then one profile is divided by nTxs and number of total
368 blocks is increased by nTxs (nProfiles *= nTxs)
470 blocks is increased by nTxs (nProfiles *= nTxs)
369 """
471 """
370 self.dataOut.flagDataAsBlock = False
472 self.dataOut.flagDataAsBlock = False
371 self.dataOut.data = self.datablock[:,self.profileIndex,:]
473 self.dataOut.data = self.datablock[:,self.profileIndex,:]
372 self.dataOut.profileIndex = self.profileIndex
474 self.dataOut.profileIndex = self.profileIndex
373
475
374 self.profileIndex += 1
476 self.profileIndex += 1
375
477
376 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
478 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
377 # """
479 # """
378 # Return all block
480 # Return all block
379 # """
481 # """
380 # self.dataOut.flagDataAsBlock = True
482 # self.dataOut.flagDataAsBlock = True
381 # self.dataOut.data = self.datablock
483 # self.dataOut.data = self.datablock
382 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
484 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
383 #
485 #
384 # self.profileIndex = self.dataOut.nProfiles
486 # self.profileIndex = self.dataOut.nProfiles
385
487
386 else:
488 else:
387 """
489 """
388 Return a block
490 Return a block
389 """
491 """
390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
492 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 if self.selBlocktime != None:
493 if self.selBlocktime != None:
392 if self.dataOut.nCohInt is not None:
494 if self.dataOut.nCohInt is not None:
393 nCohInt = self.dataOut.nCohInt
495 nCohInt = self.dataOut.nCohInt
394 else:
496 else:
395 nCohInt = 1
497 nCohInt = 1
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
498 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397
499
398 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
500 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
399 self.profileIndex += self.selBlocksize
501 self.profileIndex += self.selBlocksize
400 datasize = self.dataOut.data.shape[1]
502 datasize = self.dataOut.data.shape[1]
401
503
402 if datasize < self.selBlocksize:
504 if datasize < self.selBlocksize:
403 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
505 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
404 buffer[:,:datasize,:] = self.dataOut.data
506 buffer[:,:datasize,:] = self.dataOut.data
405
507
406 while datasize < self.selBlocksize: #Not enough profiles to fill the block
508 while datasize < self.selBlocksize: #Not enough profiles to fill the block
407 if not( self.readNextBlock() ):
509 if not( self.readNextBlock() ):
408 return 0
510 return 0
409 self.getFirstHeader()
511 self.getFirstHeader()
410 self.reshapeData()
512 self.reshapeData()
411 if self.datablock is None:
513 if self.datablock is None:
412 self.dataOut.flagNoData = True
514 self.dataOut.flagNoData = True
413 return 0
515 return 0
414 #stack data
516 #stack data
415 blockIndex = self.selBlocksize - datasize
517 blockIndex = self.selBlocksize - datasize
416 datablock1 = self.datablock[:,:blockIndex,:]
518 datablock1 = self.datablock[:,:blockIndex,:]
417
519
418 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
520 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
419 datasize += datablock1.shape[1]
521 datasize += datablock1.shape[1]
420
522
421 self.dataOut.data = buffer
523 self.dataOut.data = buffer
422 self.profileIndex = blockIndex
524 self.profileIndex = blockIndex
423
525
424 self.dataOut.flagDataAsBlock = True
526 self.dataOut.flagDataAsBlock = True
425 self.dataOut.nProfiles = self.dataOut.data.shape[1]
527 self.dataOut.nProfiles = self.dataOut.data.shape[1]
426
528
427 self.dataOut.flagNoData = False
529 self.dataOut.flagNoData = False
428
530
429 self.getBasicHeader()
531 self.getBasicHeader()
430
532
431 self.dataOut.realtime = self.online
533 self.dataOut.realtime = self.online
432
534
433 return self.dataOut.data
535 return self.dataOut.data
434
536
435 class VoltageWriter(JRODataWriter, Operation):
537 class VoltageWriter(JRODataWriter, Operation):
436 """
538 """
437 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
539 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
438 de los datos siempre se realiza por bloques.
540 de los datos siempre se realiza por bloques.
439 """
541 """
440
542
441 ext = ".r"
543 ext = ".r"
442
544
443 optchar = "D"
545 optchar = "D"
444
546
445 shapeBuffer = None
547 shapeBuffer = None
446
548
447
549
448 def __init__(self, **kwargs):
550 def __init__(self, **kwargs):
449 """
551 """
450 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
552 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
451
553
452 Affected:
554 Affected:
453 self.dataOut
555 self.dataOut
454
556
455 Return: None
557 Return: None
456 """
558 """
457 Operation.__init__(self, **kwargs)
559 Operation.__init__(self, **kwargs)
458
560
459 self.nTotalBlocks = 0
561 self.nTotalBlocks = 0
460
562
461 self.profileIndex = 0
563 self.profileIndex = 0
462
564
463 self.isConfig = False
565 self.isConfig = False
464
566
465 self.fp = None
567 self.fp = None
466
568
467 self.flagIsNewFile = 1
569 self.flagIsNewFile = 1
468
570
469 self.blockIndex = 0
571 self.blockIndex = 0
470
572
471 self.flagIsNewBlock = 0
573 self.flagIsNewBlock = 0
472
574
473 self.setFile = None
575 self.setFile = None
474
576
475 self.dtype = None
577 self.dtype = None
476
578
477 self.path = None
579 self.path = None
478
580
479 self.filename = None
581 self.filename = None
480
582
481 self.basicHeaderObj = BasicHeader(LOCALTIME)
583 self.basicHeaderObj = BasicHeader(LOCALTIME)
482
584
483 self.systemHeaderObj = SystemHeader()
585 self.systemHeaderObj = SystemHeader()
484
586
485 self.radarControllerHeaderObj = RadarControllerHeader()
587 self.radarControllerHeaderObj = RadarControllerHeader()
486
588
487 self.processingHeaderObj = ProcessingHeader()
589 self.processingHeaderObj = ProcessingHeader()
488
590
489 def hasAllDataInBuffer(self):
591 def hasAllDataInBuffer(self):
490 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
592 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
491 return 1
593 return 1
492 return 0
594 return 0
493
595
494
596
495 def setBlockDimension(self):
597 def setBlockDimension(self):
496 """
598 """
497 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
599 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
498
600
499 Affected:
601 Affected:
500 self.shape_spc_Buffer
602 self.shape_spc_Buffer
501 self.shape_cspc_Buffer
603 self.shape_cspc_Buffer
502 self.shape_dc_Buffer
604 self.shape_dc_Buffer
503
605
504 Return: None
606 Return: None
505 """
607 """
506 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
608 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
507 self.processingHeaderObj.nHeights,
609 self.processingHeaderObj.nHeights,
508 self.systemHeaderObj.nChannels)
610 self.systemHeaderObj.nChannels)
509
611
510 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
612 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
511 self.processingHeaderObj.profilesPerBlock,
613 self.processingHeaderObj.profilesPerBlock,
512 self.processingHeaderObj.nHeights),
614 self.processingHeaderObj.nHeights),
513 dtype=numpy.dtype('complex64'))
615 dtype=numpy.dtype('complex64'))
514
616
515 def writeBlock(self):
617 def writeBlock(self):
516 """
618 """
517 Escribe el buffer en el file designado
619 Escribe el buffer en el file designado
518
620
519 Affected:
621 Affected:
520 self.profileIndex
622 self.profileIndex
521 self.flagIsNewFile
623 self.flagIsNewFile
522 self.flagIsNewBlock
624 self.flagIsNewBlock
523 self.nTotalBlocks
625 self.nTotalBlocks
524 self.blockIndex
626 self.blockIndex
525
627
526 Return: None
628 Return: None
527 """
629 """
528 data = numpy.zeros( self.shapeBuffer, self.dtype )
630 data = numpy.zeros( self.shapeBuffer, self.dtype )
529
631
530 junk = numpy.transpose(self.datablock, (1,2,0))
632 junk = numpy.transpose(self.datablock, (1,2,0))
531
633
532 data['real'] = junk.real
634 data['real'] = junk.real
533 data['imag'] = junk.imag
635 data['imag'] = junk.imag
534
636
535 data = data.reshape( (-1) )
637 data = data.reshape( (-1) )
536
638
537 data.tofile( self.fp )
639 data.tofile( self.fp )
538
640
539 self.datablock.fill(0)
641 self.datablock.fill(0)
540
642
541 self.profileIndex = 0
643 self.profileIndex = 0
542 self.flagIsNewFile = 0
644 self.flagIsNewFile = 0
543 self.flagIsNewBlock = 1
645 self.flagIsNewBlock = 1
544
646
545 self.blockIndex += 1
647 self.blockIndex += 1
546 self.nTotalBlocks += 1
648 self.nTotalBlocks += 1
547
649
548 # print "[Writing] Block = %04d" %self.blockIndex
650 # print "[Writing] Block = %04d" %self.blockIndex
549
651
550 def putData(self):
652 def putData(self):
551 """
653 """
552 Setea un bloque de datos y luego los escribe en un file
654 Setea un bloque de datos y luego los escribe en un file
553
655
554 Affected:
656 Affected:
555 self.flagIsNewBlock
657 self.flagIsNewBlock
556 self.profileIndex
658 self.profileIndex
557
659
558 Return:
660 Return:
559 0 : Si no hay data o no hay mas files que puedan escribirse
661 0 : Si no hay data o no hay mas files que puedan escribirse
560 1 : Si se escribio la data de un bloque en un file
662 1 : Si se escribio la data de un bloque en un file
561 """
663 """
562 if self.dataOut.flagNoData:
664 if self.dataOut.flagNoData:
563 return 0
665 return 0
564
666
565 self.flagIsNewBlock = 0
667 self.flagIsNewBlock = 0
566
668
567 if self.dataOut.flagDiscontinuousBlock:
669 if self.dataOut.flagDiscontinuousBlock:
568 self.datablock.fill(0)
670 self.datablock.fill(0)
569 self.profileIndex = 0
671 self.profileIndex = 0
570 self.setNextFile()
672 self.setNextFile()
571
673
572 if self.profileIndex == 0:
674 if self.profileIndex == 0:
573 self.setBasicHeader()
675 self.setBasicHeader()
574
676
575 self.datablock[:,self.profileIndex,:] = self.dataOut.data
677 self.datablock[:,self.profileIndex,:] = self.dataOut.data
576
678
577 self.profileIndex += 1
679 self.profileIndex += 1
578
680
579 if self.hasAllDataInBuffer():
681 if self.hasAllDataInBuffer():
580 #if self.flagIsNewFile:
682 #if self.flagIsNewFile:
581 self.writeNextBlock()
683 self.writeNextBlock()
582 # self.setFirstHeader()
684 # self.setFirstHeader()
583
685
584 return 1
686 return 1
585
687
586 def __getBlockSize(self):
688 def __getBlockSize(self):
587 '''
689 '''
588 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
690 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
589 '''
691 '''
590
692
591 dtype_width = self.getDtypeWidth()
693 dtype_width = self.getDtypeWidth()
592
694
593 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
695 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
594
696
595 return blocksize
697 return blocksize
596
698
597 def setFirstHeader(self):
699 def setFirstHeader(self):
598
700
599 """
701 """
600 Obtiene una copia del First Header
702 Obtiene una copia del First Header
601
703
602 Affected:
704 Affected:
603 self.systemHeaderObj
705 self.systemHeaderObj
604 self.radarControllerHeaderObj
706 self.radarControllerHeaderObj
605 self.dtype
707 self.dtype
606
708
607 Return:
709 Return:
608 None
710 None
609 """
711 """
610
712
611 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
713 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
612 self.systemHeaderObj.nChannels = self.dataOut.nChannels
714 self.systemHeaderObj.nChannels = self.dataOut.nChannels
613 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
715 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
614
716
615 self.processingHeaderObj.dtype = 0 # Voltage
717 self.processingHeaderObj.dtype = 0 # Voltage
616 self.processingHeaderObj.blockSize = self.__getBlockSize()
718 self.processingHeaderObj.blockSize = self.__getBlockSize()
617 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
719 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
618 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
720 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
619 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
721 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
620 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
722 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
621 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
723 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
622 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
724 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
623
725
624 if self.dataOut.code is not None:
726 if self.dataOut.code is not None:
625 self.processingHeaderObj.code = self.dataOut.code
727 self.processingHeaderObj.code = self.dataOut.code
626 self.processingHeaderObj.nCode = self.dataOut.nCode
728 self.processingHeaderObj.nCode = self.dataOut.nCode
627 self.processingHeaderObj.nBaud = self.dataOut.nBaud
729 self.processingHeaderObj.nBaud = self.dataOut.nBaud
628
730
629 if self.processingHeaderObj.nWindows != 0:
731 if self.processingHeaderObj.nWindows != 0:
630 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
732 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
631 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
733 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
632 self.processingHeaderObj.nHeights = self.dataOut.nHeights
734 self.processingHeaderObj.nHeights = self.dataOut.nHeights
633 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
735 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
634
736
635 self.processingHeaderObj.processFlags = self.getProcessFlags()
737 self.processingHeaderObj.processFlags = self.getProcessFlags()
636
738
637 self.setBasicHeader()
739 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -1,1283 +1,1283
1 import sys
1 import sys
2 import numpy
2 import numpy
3 from scipy import interpolate
3 from scipy import interpolate
4
4
5 from jroproc_base import ProcessingUnit, Operation
5 from jroproc_base import ProcessingUnit, Operation
6 from schainpy.model.data.jrodata import Voltage
6 from schainpy.model.data.jrodata import Voltage
7
7
8 class VoltageProc(ProcessingUnit):
8 class VoltageProc(ProcessingUnit):
9
9
10
10
11 def __init__(self, **kwargs):
11 def __init__(self, **kwargs):
12
12
13 ProcessingUnit.__init__(self, **kwargs)
13 ProcessingUnit.__init__(self, **kwargs)
14
14
15 # self.objectDict = {}
15 # self.objectDict = {}
16 self.dataOut = Voltage()
16 self.dataOut = Voltage()
17 self.flip = 1
17 self.flip = 1
18
18
19 def run(self):
19 def run(self):
20 if self.dataIn.type == 'AMISR':
20 if self.dataIn.type == 'AMISR':
21 self.__updateObjFromAmisrInput()
21 self.__updateObjFromAmisrInput()
22
22
23 if self.dataIn.type == 'Voltage':
23 if self.dataIn.type == 'Voltage':
24 self.dataOut.copy(self.dataIn)
24 self.dataOut.copy(self.dataIn)
25
25
26 # self.dataOut.copy(self.dataIn)
26 # self.dataOut.copy(self.dataIn)
27
27
28 def __updateObjFromAmisrInput(self):
28 def __updateObjFromAmisrInput(self):
29
29
30 self.dataOut.timeZone = self.dataIn.timeZone
30 self.dataOut.timeZone = self.dataIn.timeZone
31 self.dataOut.dstFlag = self.dataIn.dstFlag
31 self.dataOut.dstFlag = self.dataIn.dstFlag
32 self.dataOut.errorCount = self.dataIn.errorCount
32 self.dataOut.errorCount = self.dataIn.errorCount
33 self.dataOut.useLocalTime = self.dataIn.useLocalTime
33 self.dataOut.useLocalTime = self.dataIn.useLocalTime
34
34
35 self.dataOut.flagNoData = self.dataIn.flagNoData
35 self.dataOut.flagNoData = self.dataIn.flagNoData
36 self.dataOut.data = self.dataIn.data
36 self.dataOut.data = self.dataIn.data
37 self.dataOut.utctime = self.dataIn.utctime
37 self.dataOut.utctime = self.dataIn.utctime
38 self.dataOut.channelList = self.dataIn.channelList
38 self.dataOut.channelList = self.dataIn.channelList
39 # self.dataOut.timeInterval = self.dataIn.timeInterval
39 # self.dataOut.timeInterval = self.dataIn.timeInterval
40 self.dataOut.heightList = self.dataIn.heightList
40 self.dataOut.heightList = self.dataIn.heightList
41 self.dataOut.nProfiles = self.dataIn.nProfiles
41 self.dataOut.nProfiles = self.dataIn.nProfiles
42
42
43 self.dataOut.nCohInt = self.dataIn.nCohInt
43 self.dataOut.nCohInt = self.dataIn.nCohInt
44 self.dataOut.ippSeconds = self.dataIn.ippSeconds
44 self.dataOut.ippSeconds = self.dataIn.ippSeconds
45 self.dataOut.frequency = self.dataIn.frequency
45 self.dataOut.frequency = self.dataIn.frequency
46
46
47 self.dataOut.azimuth = self.dataIn.azimuth
47 self.dataOut.azimuth = self.dataIn.azimuth
48 self.dataOut.zenith = self.dataIn.zenith
48 self.dataOut.zenith = self.dataIn.zenith
49
49
50 self.dataOut.beam.codeList = self.dataIn.beam.codeList
50 self.dataOut.beam.codeList = self.dataIn.beam.codeList
51 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
51 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
52 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
52 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
53 #
53 #
54 # pass#
54 # pass#
55 #
55 #
56 # def init(self):
56 # def init(self):
57 #
57 #
58 #
58 #
59 # if self.dataIn.type == 'AMISR':
59 # if self.dataIn.type == 'AMISR':
60 # self.__updateObjFromAmisrInput()
60 # self.__updateObjFromAmisrInput()
61 #
61 #
62 # if self.dataIn.type == 'Voltage':
62 # if self.dataIn.type == 'Voltage':
63 # self.dataOut.copy(self.dataIn)
63 # self.dataOut.copy(self.dataIn)
64 # # No necesita copiar en cada init() los atributos de dataIn
64 # # No necesita copiar en cada init() los atributos de dataIn
65 # # la copia deberia hacerse por cada nuevo bloque de datos
65 # # la copia deberia hacerse por cada nuevo bloque de datos
66
66
67 def selectChannels(self, channelList):
67 def selectChannels(self, channelList):
68
68
69 channelIndexList = []
69 channelIndexList = []
70
70
71 for channel in channelList:
71 for channel in channelList:
72 if channel not in self.dataOut.channelList:
72 if channel not in self.dataOut.channelList:
73 raise ValueError, "Channel %d is not in %s" %(channel, str(self.dataOut.channelList))
73 raise ValueError, "Channel %d is not in %s" %(channel, str(self.dataOut.channelList))
74
74
75 index = self.dataOut.channelList.index(channel)
75 index = self.dataOut.channelList.index(channel)
76 channelIndexList.append(index)
76 channelIndexList.append(index)
77
77
78 self.selectChannelsByIndex(channelIndexList)
78 self.selectChannelsByIndex(channelIndexList)
79
79
80 def selectChannelsByIndex(self, channelIndexList):
80 def selectChannelsByIndex(self, channelIndexList):
81 """
81 """
82 Selecciona un bloque de datos en base a canales segun el channelIndexList
82 Selecciona un bloque de datos en base a canales segun el channelIndexList
83
83
84 Input:
84 Input:
85 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
85 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
86
86
87 Affected:
87 Affected:
88 self.dataOut.data
88 self.dataOut.data
89 self.dataOut.channelIndexList
89 self.dataOut.channelIndexList
90 self.dataOut.nChannels
90 self.dataOut.nChannels
91 self.dataOut.m_ProcessingHeader.totalSpectra
91 self.dataOut.m_ProcessingHeader.totalSpectra
92 self.dataOut.systemHeaderObj.numChannels
92 self.dataOut.systemHeaderObj.numChannels
93 self.dataOut.m_ProcessingHeader.blockSize
93 self.dataOut.m_ProcessingHeader.blockSize
94
94
95 Return:
95 Return:
96 None
96 None
97 """
97 """
98
98
99 for channelIndex in channelIndexList:
99 for channelIndex in channelIndexList:
100 if channelIndex not in self.dataOut.channelIndexList:
100 if channelIndex not in self.dataOut.channelIndexList:
101 print channelIndexList
101 print channelIndexList
102 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
102 raise ValueError, "The value %d in channelIndexList is not valid" %channelIndex
103
103
104 if self.dataOut.flagDataAsBlock:
104 if self.dataOut.flagDataAsBlock:
105 """
105 """
106 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
106 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
107 """
107 """
108 data = self.dataOut.data[channelIndexList,:,:]
108 data = self.dataOut.data[channelIndexList,:,:]
109 else:
109 else:
110 data = self.dataOut.data[channelIndexList,:]
110 data = self.dataOut.data[channelIndexList,:]
111
111
112 self.dataOut.data = data
112 self.dataOut.data = data
113 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
113 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
114 # self.dataOut.nChannels = nChannels
114 # self.dataOut.nChannels = nChannels
115
115
116 return 1
116 return 1
117
117
118 def selectHeights(self, minHei=None, maxHei=None):
118 def selectHeights(self, minHei=None, maxHei=None):
119 """
119 """
120 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
120 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
121 minHei <= height <= maxHei
121 minHei <= height <= maxHei
122
122
123 Input:
123 Input:
124 minHei : valor minimo de altura a considerar
124 minHei : valor minimo de altura a considerar
125 maxHei : valor maximo de altura a considerar
125 maxHei : valor maximo de altura a considerar
126
126
127 Affected:
127 Affected:
128 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
128 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
129
129
130 Return:
130 Return:
131 1 si el metodo se ejecuto con exito caso contrario devuelve 0
131 1 si el metodo se ejecuto con exito caso contrario devuelve 0
132 """
132 """
133
133
134 if minHei == None:
134 if minHei == None:
135 minHei = self.dataOut.heightList[0]
135 minHei = self.dataOut.heightList[0]
136
136
137 if maxHei == None:
137 if maxHei == None:
138 maxHei = self.dataOut.heightList[-1]
138 maxHei = self.dataOut.heightList[-1]
139
139
140 if (minHei < self.dataOut.heightList[0]):
140 if (minHei < self.dataOut.heightList[0]):
141 minHei = self.dataOut.heightList[0]
141 minHei = self.dataOut.heightList[0]
142
142
143 if (maxHei > self.dataOut.heightList[-1]):
143 if (maxHei > self.dataOut.heightList[-1]):
144 maxHei = self.dataOut.heightList[-1]
144 maxHei = self.dataOut.heightList[-1]
145
145
146 minIndex = 0
146 minIndex = 0
147 maxIndex = 0
147 maxIndex = 0
148 heights = self.dataOut.heightList
148 heights = self.dataOut.heightList
149
149
150 inda = numpy.where(heights >= minHei)
150 inda = numpy.where(heights >= minHei)
151 indb = numpy.where(heights <= maxHei)
151 indb = numpy.where(heights <= maxHei)
152
152
153 try:
153 try:
154 minIndex = inda[0][0]
154 minIndex = inda[0][0]
155 except:
155 except:
156 minIndex = 0
156 minIndex = 0
157
157
158 try:
158 try:
159 maxIndex = indb[0][-1]
159 maxIndex = indb[0][-1]
160 except:
160 except:
161 maxIndex = len(heights)
161 maxIndex = len(heights)
162
162
163 self.selectHeightsByIndex(minIndex, maxIndex)
163 self.selectHeightsByIndex(minIndex, maxIndex)
164
164
165 return 1
165 return 1
166
166
167
167
168 def selectHeightsByIndex(self, minIndex, maxIndex):
168 def selectHeightsByIndex(self, minIndex, maxIndex):
169 """
169 """
170 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
170 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
171 minIndex <= index <= maxIndex
171 minIndex <= index <= maxIndex
172
172
173 Input:
173 Input:
174 minIndex : valor de indice minimo de altura a considerar
174 minIndex : valor de indice minimo de altura a considerar
175 maxIndex : valor de indice maximo de altura a considerar
175 maxIndex : valor de indice maximo de altura a considerar
176
176
177 Affected:
177 Affected:
178 self.dataOut.data
178 self.dataOut.data
179 self.dataOut.heightList
179 self.dataOut.heightList
180
180
181 Return:
181 Return:
182 1 si el metodo se ejecuto con exito caso contrario devuelve 0
182 1 si el metodo se ejecuto con exito caso contrario devuelve 0
183 """
183 """
184
184
185 if (minIndex < 0) or (minIndex > maxIndex):
185 if (minIndex < 0) or (minIndex > maxIndex):
186 raise ValueError, "Height index range (%d,%d) is not valid" % (minIndex, maxIndex)
186 raise ValueError, "Height index range (%d,%d) is not valid" % (minIndex, maxIndex)
187
187
188 if (maxIndex >= self.dataOut.nHeights):
188 if (maxIndex >= self.dataOut.nHeights):
189 maxIndex = self.dataOut.nHeights
189 maxIndex = self.dataOut.nHeights
190
190
191 #voltage
191 #voltage
192 if self.dataOut.flagDataAsBlock:
192 if self.dataOut.flagDataAsBlock:
193 """
193 """
194 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
194 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
195 """
195 """
196 data = self.dataOut.data[:,:, minIndex:maxIndex]
196 data = self.dataOut.data[:,:, minIndex:maxIndex]
197 else:
197 else:
198 data = self.dataOut.data[:, minIndex:maxIndex]
198 data = self.dataOut.data[:, minIndex:maxIndex]
199
199
200 # firstHeight = self.dataOut.heightList[minIndex]
200 # firstHeight = self.dataOut.heightList[minIndex]
201
201
202 self.dataOut.data = data
202 self.dataOut.data = data
203 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
203 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
204
204
205 if self.dataOut.nHeights <= 1:
205 if self.dataOut.nHeights <= 1:
206 raise ValueError, "selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights)
206 raise ValueError, "selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights)
207
207
208 return 1
208 return 1
209
209
210
210
211 def filterByHeights(self, window):
211 def filterByHeights(self, window):
212
212
213 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
213 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
214
214
215 if window == None:
215 if window == None:
216 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
216 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
217
217
218 newdelta = deltaHeight * window
218 newdelta = deltaHeight * window
219 r = self.dataOut.nHeights % window
219 r = self.dataOut.nHeights % window
220 newheights = (self.dataOut.nHeights-r)/window
220 newheights = (self.dataOut.nHeights-r)/window
221
221
222 if newheights <= 1:
222 if newheights <= 1:
223 raise ValueError, "filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window)
223 raise ValueError, "filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window)
224
224
225 if self.dataOut.flagDataAsBlock:
225 if self.dataOut.flagDataAsBlock:
226 """
226 """
227 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
227 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
228 """
228 """
229 buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
229 buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
230 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
230 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
231 buffer = numpy.sum(buffer,3)
231 buffer = numpy.sum(buffer,3)
232
232
233 else:
233 else:
234 buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
234 buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
235 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
235 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
236 buffer = numpy.sum(buffer,2)
236 buffer = numpy.sum(buffer,2)
237
237
238 self.dataOut.data = buffer
238 self.dataOut.data = buffer
239 self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
239 self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
240 self.dataOut.windowOfFilter = window
240 self.dataOut.windowOfFilter = window
241
241
242 def setH0(self, h0, deltaHeight = None):
242 def setH0(self, h0, deltaHeight = None):
243
243
244 if not deltaHeight:
244 if not deltaHeight:
245 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
245 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
246
246
247 nHeights = self.dataOut.nHeights
247 nHeights = self.dataOut.nHeights
248
248
249 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
249 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
250
250
251 self.dataOut.heightList = newHeiRange
251 self.dataOut.heightList = newHeiRange
252
252
253 def deFlip(self, channelList = []):
253 def deFlip(self, channelList = []):
254
254
255 data = self.dataOut.data.copy()
255 data = self.dataOut.data.copy()
256
256
257 if self.dataOut.flagDataAsBlock:
257 if self.dataOut.flagDataAsBlock:
258 flip = self.flip
258 flip = self.flip
259 profileList = range(self.dataOut.nProfiles)
259 profileList = range(self.dataOut.nProfiles)
260
260
261 if not channelList:
261 if not channelList:
262 for thisProfile in profileList:
262 for thisProfile in profileList:
263 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
263 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
264 flip *= -1.0
264 flip *= -1.0
265 else:
265 else:
266 for thisChannel in channelList:
266 for thisChannel in channelList:
267 if thisChannel not in self.dataOut.channelList:
267 if thisChannel not in self.dataOut.channelList:
268 continue
268 continue
269
269
270 for thisProfile in profileList:
270 for thisProfile in profileList:
271 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
271 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
272 flip *= -1.0
272 flip *= -1.0
273
273
274 self.flip = flip
274 self.flip = flip
275
275
276 else:
276 else:
277 if not channelList:
277 if not channelList:
278 data[:,:] = data[:,:]*self.flip
278 data[:,:] = data[:,:]*self.flip
279 else:
279 else:
280 for thisChannel in channelList:
280 for thisChannel in channelList:
281 if thisChannel not in self.dataOut.channelList:
281 if thisChannel not in self.dataOut.channelList:
282 continue
282 continue
283
283
284 data[thisChannel,:] = data[thisChannel,:]*self.flip
284 data[thisChannel,:] = data[thisChannel,:]*self.flip
285
285
286 self.flip *= -1.
286 self.flip *= -1.
287
287
288 self.dataOut.data = data
288 self.dataOut.data = data
289
289
290 def setRadarFrequency(self, frequency=None):
290 def setRadarFrequency(self, frequency=None):
291
291
292 if frequency != None:
292 if frequency != None:
293 self.dataOut.frequency = frequency
293 self.dataOut.frequency = frequency
294
294
295 return 1
295 return 1
296
296
297 def interpolateHeights(self, topLim, botLim):
297 def interpolateHeights(self, topLim, botLim):
298 #69 al 72 para julia
298 #69 al 72 para julia
299 #82-84 para meteoros
299 #82-84 para meteoros
300 if len(numpy.shape(self.dataOut.data))==2:
300 if len(numpy.shape(self.dataOut.data))==2:
301 sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
301 sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
302 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
302 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
303 #self.dataOut.data[:,botLim:limSup+1] = sampInterp
303 #self.dataOut.data[:,botLim:limSup+1] = sampInterp
304 self.dataOut.data[:,botLim:topLim+1] = sampInterp
304 self.dataOut.data[:,botLim:topLim+1] = sampInterp
305 else:
305 else:
306 nHeights = self.dataOut.data.shape[2]
306 nHeights = self.dataOut.data.shape[2]
307 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
307 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
308 y = self.dataOut.data[:,:,range(botLim)+range(topLim+1,nHeights)]
308 y = self.dataOut.data[:,:,range(botLim)+range(topLim+1,nHeights)]
309 f = interpolate.interp1d(x, y, axis = 2)
309 f = interpolate.interp1d(x, y, axis = 2)
310 xnew = numpy.arange(botLim,topLim+1)
310 xnew = numpy.arange(botLim,topLim+1)
311 ynew = f(xnew)
311 ynew = f(xnew)
312
312
313 self.dataOut.data[:,:,botLim:topLim+1] = ynew
313 self.dataOut.data[:,:,botLim:topLim+1] = ynew
314
314
315 # import collections
315 # import collections
316
316
317 class CohInt(Operation):
317 class CohInt(Operation):
318
318
319 isConfig = False
319 isConfig = False
320
320
321 __profIndex = 0
321 __profIndex = 0
322 __withOverapping = False
322 __withOverapping = False
323
323
324 __byTime = False
324 __byTime = False
325 __initime = None
325 __initime = None
326 __lastdatatime = None
326 __lastdatatime = None
327 __integrationtime = None
327 __integrationtime = None
328
328
329 __buffer = None
329 __buffer = None
330
330
331 __dataReady = False
331 __dataReady = False
332
332
333 n = None
333 n = None
334
334
335
335
336 def __init__(self, **kwargs):
336 def __init__(self, **kwargs):
337
337
338 Operation.__init__(self, **kwargs)
338 Operation.__init__(self, **kwargs)
339
339
340 # self.isConfig = False
340 # self.isConfig = False
341
341
342 def setup(self, n=None, timeInterval=None, overlapping=False, byblock=False):
342 def setup(self, n=None, timeInterval=None, overlapping=False, byblock=False):
343 """
343 """
344 Set the parameters of the integration class.
344 Set the parameters of the integration class.
345
345
346 Inputs:
346 Inputs:
347
347
348 n : Number of coherent integrations
348 n : Number of coherent integrations
349 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
349 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
350 overlapping :
350 overlapping :
351
351
352 """
352 """
353
353
354 self.__initime = None
354 self.__initime = None
355 self.__lastdatatime = 0
355 self.__lastdatatime = 0
356 self.__buffer = None
356 self.__buffer = None
357 self.__dataReady = False
357 self.__dataReady = False
358 self.byblock = byblock
358 self.byblock = byblock
359
359
360 if n == None and timeInterval == None:
360 if n == None and timeInterval == None:
361 raise ValueError, "n or timeInterval should be specified ..."
361 raise ValueError, "n or timeInterval should be specified ..."
362
362
363 if n != None:
363 if n != None:
364 self.n = n
364 self.n = n
365 self.__byTime = False
365 self.__byTime = False
366 else:
366 else:
367 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
367 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
368 self.n = 9999
368 self.n = 9999
369 self.__byTime = True
369 self.__byTime = True
370
370
371 if overlapping:
371 if overlapping:
372 self.__withOverapping = True
372 self.__withOverapping = True
373 self.__buffer = None
373 self.__buffer = None
374 else:
374 else:
375 self.__withOverapping = False
375 self.__withOverapping = False
376 self.__buffer = 0
376 self.__buffer = 0
377
377
378 self.__profIndex = 0
378 self.__profIndex = 0
379
379
380 def putData(self, data):
380 def putData(self, data):
381
381
382 """
382 """
383 Add a profile to the __buffer and increase in one the __profileIndex
383 Add a profile to the __buffer and increase in one the __profileIndex
384
384
385 """
385 """
386
386
387 if not self.__withOverapping:
387 if not self.__withOverapping:
388 self.__buffer += data.copy()
388 self.__buffer += data.copy()
389 self.__profIndex += 1
389 self.__profIndex += 1
390 return
390 return
391
391
392 #Overlapping data
392 #Overlapping data
393 nChannels, nHeis = data.shape
393 nChannels, nHeis = data.shape
394 data = numpy.reshape(data, (1, nChannels, nHeis))
394 data = numpy.reshape(data, (1, nChannels, nHeis))
395
395
396 #If the buffer is empty then it takes the data value
396 #If the buffer is empty then it takes the data value
397 if self.__buffer is None:
397 if self.__buffer is None:
398 self.__buffer = data
398 self.__buffer = data
399 self.__profIndex += 1
399 self.__profIndex += 1
400 return
400 return
401
401
402 #If the buffer length is lower than n then stakcing the data value
402 #If the buffer length is lower than n then stakcing the data value
403 if self.__profIndex < self.n:
403 if self.__profIndex < self.n:
404 self.__buffer = numpy.vstack((self.__buffer, data))
404 self.__buffer = numpy.vstack((self.__buffer, data))
405 self.__profIndex += 1
405 self.__profIndex += 1
406 return
406 return
407
407
408 #If the buffer length is equal to n then replacing the last buffer value with the data value
408 #If the buffer length is equal to n then replacing the last buffer value with the data value
409 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
409 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
410 self.__buffer[self.n-1] = data
410 self.__buffer[self.n-1] = data
411 self.__profIndex = self.n
411 self.__profIndex = self.n
412 return
412 return
413
413
414
414
415 def pushData(self):
415 def pushData(self):
416 """
416 """
417 Return the sum of the last profiles and the profiles used in the sum.
417 Return the sum of the last profiles and the profiles used in the sum.
418
418
419 Affected:
419 Affected:
420
420
421 self.__profileIndex
421 self.__profileIndex
422
422
423 """
423 """
424
424
425 if not self.__withOverapping:
425 if not self.__withOverapping:
426 data = self.__buffer
426 data = self.__buffer
427 n = self.__profIndex
427 n = self.__profIndex
428
428
429 self.__buffer = 0
429 self.__buffer = 0
430 self.__profIndex = 0
430 self.__profIndex = 0
431
431
432 return data, n
432 return data, n
433
433
434 #Integration with Overlapping
434 #Integration with Overlapping
435 data = numpy.sum(self.__buffer, axis=0)
435 data = numpy.sum(self.__buffer, axis=0)
436 n = self.__profIndex
436 n = self.__profIndex
437
437
438 return data, n
438 return data, n
439
439
440 def byProfiles(self, data):
440 def byProfiles(self, data):
441
441
442 self.__dataReady = False
442 self.__dataReady = False
443 avgdata = None
443 avgdata = None
444 # n = None
444 # n = None
445
445
446 self.putData(data)
446 self.putData(data)
447
447
448 if self.__profIndex == self.n:
448 if self.__profIndex == self.n:
449
449
450 avgdata, n = self.pushData()
450 avgdata, n = self.pushData()
451 self.__dataReady = True
451 self.__dataReady = True
452
452
453 return avgdata
453 return avgdata
454
454
455 def byTime(self, data, datatime):
455 def byTime(self, data, datatime):
456
456
457 self.__dataReady = False
457 self.__dataReady = False
458 avgdata = None
458 avgdata = None
459 n = None
459 n = None
460
460
461 self.putData(data)
461 self.putData(data)
462
462
463 if (datatime - self.__initime) >= self.__integrationtime:
463 if (datatime - self.__initime) >= self.__integrationtime:
464 avgdata, n = self.pushData()
464 avgdata, n = self.pushData()
465 self.n = n
465 self.n = n
466 self.__dataReady = True
466 self.__dataReady = True
467
467
468 return avgdata
468 return avgdata
469
469
470 def integrate(self, data, datatime=None):
470 def integrate(self, data, datatime=None):
471
471
472 if self.__initime == None:
472 if self.__initime == None:
473 self.__initime = datatime
473 self.__initime = datatime
474
474
475 if self.__byTime:
475 if self.__byTime:
476 avgdata = self.byTime(data, datatime)
476 avgdata = self.byTime(data, datatime)
477 else:
477 else:
478 avgdata = self.byProfiles(data)
478 avgdata = self.byProfiles(data)
479
479
480
480
481 self.__lastdatatime = datatime
481 self.__lastdatatime = datatime
482
482
483 if avgdata is None:
483 if avgdata is None:
484 return None, None
484 return None, None
485
485
486 avgdatatime = self.__initime
486 avgdatatime = self.__initime
487
487
488 deltatime = datatime -self.__lastdatatime
488 deltatime = datatime -self.__lastdatatime
489
489
490 if not self.__withOverapping:
490 if not self.__withOverapping:
491 self.__initime = datatime
491 self.__initime = datatime
492 else:
492 else:
493 self.__initime += deltatime
493 self.__initime += deltatime
494
494
495 return avgdata, avgdatatime
495 return avgdata, avgdatatime
496
496
497 def integrateByBlock(self, dataOut):
497 def integrateByBlock(self, dataOut):
498
498
499 times = int(dataOut.data.shape[1]/self.n)
499 times = int(dataOut.data.shape[1]/self.n)
500 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
500 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
501
501
502 id_min = 0
502 id_min = 0
503 id_max = self.n
503 id_max = self.n
504
504
505 for i in range(times):
505 for i in range(times):
506 junk = dataOut.data[:,id_min:id_max,:]
506 junk = dataOut.data[:,id_min:id_max,:]
507 avgdata[:,i,:] = junk.sum(axis=1)
507 avgdata[:,i,:] = junk.sum(axis=1)
508 id_min += self.n
508 id_min += self.n
509 id_max += self.n
509 id_max += self.n
510
510
511 timeInterval = dataOut.ippSeconds*self.n
511 timeInterval = dataOut.ippSeconds*self.n
512 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
512 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
513 self.__dataReady = True
513 self.__dataReady = True
514 return avgdata, avgdatatime
514 return avgdata, avgdatatime
515
515
516 def run(self, dataOut, **kwargs):
516
517
517 def run(self, dataOut, n=None, timeInterval=None, overlapping=False, byblock=False, **kwargs):
518 if not self.isConfig:
518 if not self.isConfig:
519 self.setup(**kwargs)
519 self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
520 self.isConfig = True
520 self.isConfig = True
521
521
522 if dataOut.flagDataAsBlock:
522 if dataOut.flagDataAsBlock:
523 """
523 """
524 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
524 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
525 """
525 """
526 avgdata, avgdatatime = self.integrateByBlock(dataOut)
526 avgdata, avgdatatime = self.integrateByBlock(dataOut)
527 dataOut.nProfiles /= self.n
527 dataOut.nProfiles /= self.n
528 else:
528 else:
529 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
529 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
530
530
531 # dataOut.timeInterval *= n
531 # dataOut.timeInterval *= n
532 dataOut.flagNoData = True
532 dataOut.flagNoData = True
533
533
534 if self.__dataReady:
534 if self.__dataReady:
535 dataOut.data = avgdata
535 dataOut.data = avgdata
536 dataOut.nCohInt *= self.n
536 dataOut.nCohInt *= self.n
537 dataOut.utctime = avgdatatime
537 dataOut.utctime = avgdatatime
538 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
538 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
539 dataOut.flagNoData = False
539 dataOut.flagNoData = False
540
540
541 class Decoder(Operation):
541 class Decoder(Operation):
542
542
543 isConfig = False
543 isConfig = False
544 __profIndex = 0
544 __profIndex = 0
545
545
546 code = None
546 code = None
547
547
548 nCode = None
548 nCode = None
549 nBaud = None
549 nBaud = None
550
550
551
551
552 def __init__(self, **kwargs):
552 def __init__(self, **kwargs):
553
553
554 Operation.__init__(self, **kwargs)
554 Operation.__init__(self, **kwargs)
555
555
556 self.times = None
556 self.times = None
557 self.osamp = None
557 self.osamp = None
558 # self.__setValues = False
558 # self.__setValues = False
559 self.isConfig = False
559 self.isConfig = False
560
560
561 def setup(self, code, osamp, dataOut):
561 def setup(self, code, osamp, dataOut):
562
562
563 self.__profIndex = 0
563 self.__profIndex = 0
564
564
565 self.code = code
565 self.code = code
566
566
567 self.nCode = len(code)
567 self.nCode = len(code)
568 self.nBaud = len(code[0])
568 self.nBaud = len(code[0])
569
569
570 if (osamp != None) and (osamp >1):
570 if (osamp != None) and (osamp >1):
571 self.osamp = osamp
571 self.osamp = osamp
572 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
572 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
573 self.nBaud = self.nBaud*self.osamp
573 self.nBaud = self.nBaud*self.osamp
574
574
575 self.__nChannels = dataOut.nChannels
575 self.__nChannels = dataOut.nChannels
576 self.__nProfiles = dataOut.nProfiles
576 self.__nProfiles = dataOut.nProfiles
577 self.__nHeis = dataOut.nHeights
577 self.__nHeis = dataOut.nHeights
578
578
579 if self.__nHeis < self.nBaud:
579 if self.__nHeis < self.nBaud:
580 raise ValueError, 'Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
580 raise ValueError, 'Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud)
581
581
582 #Frequency
582 #Frequency
583 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
583 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
584
584
585 __codeBuffer[:,0:self.nBaud] = self.code
585 __codeBuffer[:,0:self.nBaud] = self.code
586
586
587 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
587 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
588
588
589 if dataOut.flagDataAsBlock:
589 if dataOut.flagDataAsBlock:
590
590
591 self.ndatadec = self.__nHeis #- self.nBaud + 1
591 self.ndatadec = self.__nHeis #- self.nBaud + 1
592
592
593 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
593 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
594
594
595 else:
595 else:
596
596
597 #Time
597 #Time
598 self.ndatadec = self.__nHeis #- self.nBaud + 1
598 self.ndatadec = self.__nHeis #- self.nBaud + 1
599
599
600 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
600 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
601
601
602 def __convolutionInFreq(self, data):
602 def __convolutionInFreq(self, data):
603
603
604 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
604 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
605
605
606 fft_data = numpy.fft.fft(data, axis=1)
606 fft_data = numpy.fft.fft(data, axis=1)
607
607
608 conv = fft_data*fft_code
608 conv = fft_data*fft_code
609
609
610 data = numpy.fft.ifft(conv,axis=1)
610 data = numpy.fft.ifft(conv,axis=1)
611
611
612 return data
612 return data
613
613
614 def __convolutionInFreqOpt(self, data):
614 def __convolutionInFreqOpt(self, data):
615
615
616 raise NotImplementedError
616 raise NotImplementedError
617
617
618 def __convolutionInTime(self, data):
618 def __convolutionInTime(self, data):
619
619
620 code = self.code[self.__profIndex]
620 code = self.code[self.__profIndex]
621
621
622 for i in range(self.__nChannels):
622 for i in range(self.__nChannels):
623 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
623 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
624
624
625 return self.datadecTime
625 return self.datadecTime
626
626
627 def __convolutionByBlockInTime(self, data):
627 def __convolutionByBlockInTime(self, data):
628
628
629 repetitions = self.__nProfiles / self.nCode
629 repetitions = self.__nProfiles / self.nCode
630
630
631 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
631 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
632 junk = junk.flatten()
632 junk = junk.flatten()
633 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
633 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
634
634
635 for i in range(self.__nChannels):
635 for i in range(self.__nChannels):
636 for j in range(self.__nProfiles):
636 for j in range(self.__nProfiles):
637 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
637 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
638
638
639 return self.datadecTime
639 return self.datadecTime
640
640
641 def __convolutionByBlockInFreq(self, data):
641 def __convolutionByBlockInFreq(self, data):
642
642
643 raise NotImplementedError, "Decoder by frequency fro Blocks not implemented"
643 raise NotImplementedError, "Decoder by frequency fro Blocks not implemented"
644
644
645
645
646 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
646 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
647
647
648 fft_data = numpy.fft.fft(data, axis=2)
648 fft_data = numpy.fft.fft(data, axis=2)
649
649
650 conv = fft_data*fft_code
650 conv = fft_data*fft_code
651
651
652 data = numpy.fft.ifft(conv,axis=2)
652 data = numpy.fft.ifft(conv,axis=2)
653
653
654 return data
654 return data
655
655
656 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
656 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
657
657
658 if dataOut.flagDecodeData:
658 if dataOut.flagDecodeData:
659 print "This data is already decoded, recoding again ..."
659 print "This data is already decoded, recoding again ..."
660
660
661 if not self.isConfig:
661 if not self.isConfig:
662
662
663 if code is None:
663 if code is None:
664 if dataOut.code is None:
664 if dataOut.code is None:
665 raise ValueError, "Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type
665 raise ValueError, "Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type
666
666
667 code = dataOut.code
667 code = dataOut.code
668 else:
668 else:
669 code = numpy.array(code).reshape(nCode,nBaud)
669 code = numpy.array(code).reshape(nCode,nBaud)
670
670
671 self.setup(code, osamp, dataOut)
671 self.setup(code, osamp, dataOut)
672
672
673 self.isConfig = True
673 self.isConfig = True
674
674
675 if mode == 3:
675 if mode == 3:
676 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
676 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
677
677
678 if times != None:
678 if times != None:
679 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
679 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
680
680
681 if self.code is None:
681 if self.code is None:
682 print "Fail decoding: Code is not defined."
682 print "Fail decoding: Code is not defined."
683 return
683 return
684
684
685 datadec = None
685 datadec = None
686 if mode == 3:
686 if mode == 3:
687 mode = 0
687 mode = 0
688
688
689 if dataOut.flagDataAsBlock:
689 if dataOut.flagDataAsBlock:
690 """
690 """
691 Decoding when data have been read as block,
691 Decoding when data have been read as block,
692 """
692 """
693
693
694 if mode == 0:
694 if mode == 0:
695 datadec = self.__convolutionByBlockInTime(dataOut.data)
695 datadec = self.__convolutionByBlockInTime(dataOut.data)
696 if mode == 1:
696 if mode == 1:
697 datadec = self.__convolutionByBlockInFreq(dataOut.data)
697 datadec = self.__convolutionByBlockInFreq(dataOut.data)
698 else:
698 else:
699 """
699 """
700 Decoding when data have been read profile by profile
700 Decoding when data have been read profile by profile
701 """
701 """
702 if mode == 0:
702 if mode == 0:
703 datadec = self.__convolutionInTime(dataOut.data)
703 datadec = self.__convolutionInTime(dataOut.data)
704
704
705 if mode == 1:
705 if mode == 1:
706 datadec = self.__convolutionInFreq(dataOut.data)
706 datadec = self.__convolutionInFreq(dataOut.data)
707
707
708 if mode == 2:
708 if mode == 2:
709 datadec = self.__convolutionInFreqOpt(dataOut.data)
709 datadec = self.__convolutionInFreqOpt(dataOut.data)
710
710
711 if datadec is None:
711 if datadec is None:
712 raise ValueError, "Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode
712 raise ValueError, "Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode
713
713
714 dataOut.code = self.code
714 dataOut.code = self.code
715 dataOut.nCode = self.nCode
715 dataOut.nCode = self.nCode
716 dataOut.nBaud = self.nBaud
716 dataOut.nBaud = self.nBaud
717
717
718 dataOut.data = datadec
718 dataOut.data = datadec
719
719
720 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
720 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
721
721
722 dataOut.flagDecodeData = True #asumo q la data esta decodificada
722 dataOut.flagDecodeData = True #asumo q la data esta decodificada
723
723
724 if self.__profIndex == self.nCode-1:
724 if self.__profIndex == self.nCode-1:
725 self.__profIndex = 0
725 self.__profIndex = 0
726 return 1
726 return 1
727
727
728 self.__profIndex += 1
728 self.__profIndex += 1
729
729
730 return 1
730 return 1
731 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
731 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
732
732
733
733
734 class ProfileConcat(Operation):
734 class ProfileConcat(Operation):
735
735
736 isConfig = False
736 isConfig = False
737 buffer = None
737 buffer = None
738
738
739 def __init__(self, **kwargs):
739 def __init__(self, **kwargs):
740
740
741 Operation.__init__(self, **kwargs)
741 Operation.__init__(self, **kwargs)
742 self.profileIndex = 0
742 self.profileIndex = 0
743
743
744 def reset(self):
744 def reset(self):
745 self.buffer = numpy.zeros_like(self.buffer)
745 self.buffer = numpy.zeros_like(self.buffer)
746 self.start_index = 0
746 self.start_index = 0
747 self.times = 1
747 self.times = 1
748
748
749 def setup(self, data, m, n=1):
749 def setup(self, data, m, n=1):
750 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
750 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
751 self.nHeights = data.shape[1]#.nHeights
751 self.nHeights = data.shape[1]#.nHeights
752 self.start_index = 0
752 self.start_index = 0
753 self.times = 1
753 self.times = 1
754
754
755 def concat(self, data):
755 def concat(self, data):
756
756
757 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
757 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
758 self.start_index = self.start_index + self.nHeights
758 self.start_index = self.start_index + self.nHeights
759
759
760 def run(self, dataOut, m):
760 def run(self, dataOut, m):
761
761
762 dataOut.flagNoData = True
762 dataOut.flagNoData = True
763
763
764 if not self.isConfig:
764 if not self.isConfig:
765 self.setup(dataOut.data, m, 1)
765 self.setup(dataOut.data, m, 1)
766 self.isConfig = True
766 self.isConfig = True
767
767
768 if dataOut.flagDataAsBlock:
768 if dataOut.flagDataAsBlock:
769 raise ValueError, "ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False"
769 raise ValueError, "ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False"
770
770
771 else:
771 else:
772 self.concat(dataOut.data)
772 self.concat(dataOut.data)
773 self.times += 1
773 self.times += 1
774 if self.times > m:
774 if self.times > m:
775 dataOut.data = self.buffer
775 dataOut.data = self.buffer
776 self.reset()
776 self.reset()
777 dataOut.flagNoData = False
777 dataOut.flagNoData = False
778 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
778 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
779 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
779 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
780 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
780 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
781 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
781 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
782 dataOut.ippSeconds *= m
782 dataOut.ippSeconds *= m
783
783
784 class ProfileSelector(Operation):
784 class ProfileSelector(Operation):
785
785
786 profileIndex = None
786 profileIndex = None
787 # Tamanho total de los perfiles
787 # Tamanho total de los perfiles
788 nProfiles = None
788 nProfiles = None
789
789
790 def __init__(self, **kwargs):
790 def __init__(self, **kwargs):
791
791
792 Operation.__init__(self, **kwargs)
792 Operation.__init__(self, **kwargs)
793 self.profileIndex = 0
793 self.profileIndex = 0
794
794
795 def incProfileIndex(self):
795 def incProfileIndex(self):
796
796
797 self.profileIndex += 1
797 self.profileIndex += 1
798
798
799 if self.profileIndex >= self.nProfiles:
799 if self.profileIndex >= self.nProfiles:
800 self.profileIndex = 0
800 self.profileIndex = 0
801
801
802 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
802 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
803
803
804 if profileIndex < minIndex:
804 if profileIndex < minIndex:
805 return False
805 return False
806
806
807 if profileIndex > maxIndex:
807 if profileIndex > maxIndex:
808 return False
808 return False
809
809
810 return True
810 return True
811
811
812 def isThisProfileInList(self, profileIndex, profileList):
812 def isThisProfileInList(self, profileIndex, profileList):
813
813
814 if profileIndex not in profileList:
814 if profileIndex not in profileList:
815 return False
815 return False
816
816
817 return True
817 return True
818
818
819 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
819 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
820
820
821 """
821 """
822 ProfileSelector:
822 ProfileSelector:
823
823
824 Inputs:
824 Inputs:
825 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
825 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
826
826
827 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
827 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
828
828
829 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
829 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
830
830
831 """
831 """
832
832
833 if rangeList is not None:
833 if rangeList is not None:
834 if type(rangeList[0]) not in (tuple, list):
834 if type(rangeList[0]) not in (tuple, list):
835 rangeList = [rangeList]
835 rangeList = [rangeList]
836
836
837 dataOut.flagNoData = True
837 dataOut.flagNoData = True
838
838
839 if dataOut.flagDataAsBlock:
839 if dataOut.flagDataAsBlock:
840 """
840 """
841 data dimension = [nChannels, nProfiles, nHeis]
841 data dimension = [nChannels, nProfiles, nHeis]
842 """
842 """
843 if profileList != None:
843 if profileList != None:
844 dataOut.data = dataOut.data[:,profileList,:]
844 dataOut.data = dataOut.data[:,profileList,:]
845
845
846 if profileRangeList != None:
846 if profileRangeList != None:
847 minIndex = profileRangeList[0]
847 minIndex = profileRangeList[0]
848 maxIndex = profileRangeList[1]
848 maxIndex = profileRangeList[1]
849 profileList = range(minIndex, maxIndex+1)
849 profileList = range(minIndex, maxIndex+1)
850
850
851 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
851 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
852
852
853 if rangeList != None:
853 if rangeList != None:
854
854
855 profileList = []
855 profileList = []
856
856
857 for thisRange in rangeList:
857 for thisRange in rangeList:
858 minIndex = thisRange[0]
858 minIndex = thisRange[0]
859 maxIndex = thisRange[1]
859 maxIndex = thisRange[1]
860
860
861 profileList.extend(range(minIndex, maxIndex+1))
861 profileList.extend(range(minIndex, maxIndex+1))
862
862
863 dataOut.data = dataOut.data[:,profileList,:]
863 dataOut.data = dataOut.data[:,profileList,:]
864
864
865 dataOut.nProfiles = len(profileList)
865 dataOut.nProfiles = len(profileList)
866 dataOut.profileIndex = dataOut.nProfiles - 1
866 dataOut.profileIndex = dataOut.nProfiles - 1
867 dataOut.flagNoData = False
867 dataOut.flagNoData = False
868
868
869 return True
869 return True
870
870
871 """
871 """
872 data dimension = [nChannels, nHeis]
872 data dimension = [nChannels, nHeis]
873 """
873 """
874
874
875 if profileList != None:
875 if profileList != None:
876
876
877 if self.isThisProfileInList(dataOut.profileIndex, profileList):
877 if self.isThisProfileInList(dataOut.profileIndex, profileList):
878
878
879 self.nProfiles = len(profileList)
879 self.nProfiles = len(profileList)
880 dataOut.nProfiles = self.nProfiles
880 dataOut.nProfiles = self.nProfiles
881 dataOut.profileIndex = self.profileIndex
881 dataOut.profileIndex = self.profileIndex
882 dataOut.flagNoData = False
882 dataOut.flagNoData = False
883
883
884 self.incProfileIndex()
884 self.incProfileIndex()
885 return True
885 return True
886
886
887 if profileRangeList != None:
887 if profileRangeList != None:
888
888
889 minIndex = profileRangeList[0]
889 minIndex = profileRangeList[0]
890 maxIndex = profileRangeList[1]
890 maxIndex = profileRangeList[1]
891
891
892 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
892 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
893
893
894 self.nProfiles = maxIndex - minIndex + 1
894 self.nProfiles = maxIndex - minIndex + 1
895 dataOut.nProfiles = self.nProfiles
895 dataOut.nProfiles = self.nProfiles
896 dataOut.profileIndex = self.profileIndex
896 dataOut.profileIndex = self.profileIndex
897 dataOut.flagNoData = False
897 dataOut.flagNoData = False
898
898
899 self.incProfileIndex()
899 self.incProfileIndex()
900 return True
900 return True
901
901
902 if rangeList != None:
902 if rangeList != None:
903
903
904 nProfiles = 0
904 nProfiles = 0
905
905
906 for thisRange in rangeList:
906 for thisRange in rangeList:
907 minIndex = thisRange[0]
907 minIndex = thisRange[0]
908 maxIndex = thisRange[1]
908 maxIndex = thisRange[1]
909
909
910 nProfiles += maxIndex - minIndex + 1
910 nProfiles += maxIndex - minIndex + 1
911
911
912 for thisRange in rangeList:
912 for thisRange in rangeList:
913
913
914 minIndex = thisRange[0]
914 minIndex = thisRange[0]
915 maxIndex = thisRange[1]
915 maxIndex = thisRange[1]
916
916
917 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
917 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
918
918
919 self.nProfiles = nProfiles
919 self.nProfiles = nProfiles
920 dataOut.nProfiles = self.nProfiles
920 dataOut.nProfiles = self.nProfiles
921 dataOut.profileIndex = self.profileIndex
921 dataOut.profileIndex = self.profileIndex
922 dataOut.flagNoData = False
922 dataOut.flagNoData = False
923
923
924 self.incProfileIndex()
924 self.incProfileIndex()
925
925
926 break
926 break
927
927
928 return True
928 return True
929
929
930
930
931 if beam != None: #beam is only for AMISR data
931 if beam != None: #beam is only for AMISR data
932 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
932 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
933 dataOut.flagNoData = False
933 dataOut.flagNoData = False
934 dataOut.profileIndex = self.profileIndex
934 dataOut.profileIndex = self.profileIndex
935
935
936 self.incProfileIndex()
936 self.incProfileIndex()
937
937
938 return True
938 return True
939
939
940 raise ValueError, "ProfileSelector needs profileList, profileRangeList or rangeList parameter"
940 raise ValueError, "ProfileSelector needs profileList, profileRangeList or rangeList parameter"
941
941
942 return False
942 return False
943
943
944 class Reshaper(Operation):
944 class Reshaper(Operation):
945
945
946 def __init__(self, **kwargs):
946 def __init__(self, **kwargs):
947
947
948 Operation.__init__(self, **kwargs)
948 Operation.__init__(self, **kwargs)
949
949
950 self.__buffer = None
950 self.__buffer = None
951 self.__nitems = 0
951 self.__nitems = 0
952
952
953 def __appendProfile(self, dataOut, nTxs):
953 def __appendProfile(self, dataOut, nTxs):
954
954
955 if self.__buffer is None:
955 if self.__buffer is None:
956 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
956 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
957 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
957 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
958
958
959 ini = dataOut.nHeights * self.__nitems
959 ini = dataOut.nHeights * self.__nitems
960 end = ini + dataOut.nHeights
960 end = ini + dataOut.nHeights
961
961
962 self.__buffer[:, ini:end] = dataOut.data
962 self.__buffer[:, ini:end] = dataOut.data
963
963
964 self.__nitems += 1
964 self.__nitems += 1
965
965
966 return int(self.__nitems*nTxs)
966 return int(self.__nitems*nTxs)
967
967
968 def __getBuffer(self):
968 def __getBuffer(self):
969
969
970 if self.__nitems == int(1./self.__nTxs):
970 if self.__nitems == int(1./self.__nTxs):
971
971
972 self.__nitems = 0
972 self.__nitems = 0
973
973
974 return self.__buffer.copy()
974 return self.__buffer.copy()
975
975
976 return None
976 return None
977
977
978 def __checkInputs(self, dataOut, shape, nTxs):
978 def __checkInputs(self, dataOut, shape, nTxs):
979
979
980 if shape is None and nTxs is None:
980 if shape is None and nTxs is None:
981 raise ValueError, "Reshaper: shape of factor should be defined"
981 raise ValueError, "Reshaper: shape of factor should be defined"
982
982
983 if nTxs:
983 if nTxs:
984 if nTxs < 0:
984 if nTxs < 0:
985 raise ValueError, "nTxs should be greater than 0"
985 raise ValueError, "nTxs should be greater than 0"
986
986
987 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
987 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
988 raise ValueError, "nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs))
988 raise ValueError, "nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs))
989
989
990 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
990 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
991
991
992 return shape, nTxs
992 return shape, nTxs
993
993
994 if len(shape) != 2 and len(shape) != 3:
994 if len(shape) != 2 and len(shape) != 3:
995 raise ValueError, "shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights)
995 raise ValueError, "shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights)
996
996
997 if len(shape) == 2:
997 if len(shape) == 2:
998 shape_tuple = [dataOut.nChannels]
998 shape_tuple = [dataOut.nChannels]
999 shape_tuple.extend(shape)
999 shape_tuple.extend(shape)
1000 else:
1000 else:
1001 shape_tuple = list(shape)
1001 shape_tuple = list(shape)
1002
1002
1003 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1003 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1004
1004
1005 return shape_tuple, nTxs
1005 return shape_tuple, nTxs
1006
1006
1007 def run(self, dataOut, shape=None, nTxs=None):
1007 def run(self, dataOut, shape=None, nTxs=None):
1008
1008
1009 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1009 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1010
1010
1011 dataOut.flagNoData = True
1011 dataOut.flagNoData = True
1012 profileIndex = None
1012 profileIndex = None
1013
1013
1014 if dataOut.flagDataAsBlock:
1014 if dataOut.flagDataAsBlock:
1015
1015
1016 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1016 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1017 dataOut.flagNoData = False
1017 dataOut.flagNoData = False
1018
1018
1019 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1019 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1020
1020
1021 else:
1021 else:
1022
1022
1023 if self.__nTxs < 1:
1023 if self.__nTxs < 1:
1024
1024
1025 self.__appendProfile(dataOut, self.__nTxs)
1025 self.__appendProfile(dataOut, self.__nTxs)
1026 new_data = self.__getBuffer()
1026 new_data = self.__getBuffer()
1027
1027
1028 if new_data is not None:
1028 if new_data is not None:
1029 dataOut.data = new_data
1029 dataOut.data = new_data
1030 dataOut.flagNoData = False
1030 dataOut.flagNoData = False
1031
1031
1032 profileIndex = dataOut.profileIndex*nTxs
1032 profileIndex = dataOut.profileIndex*nTxs
1033
1033
1034 else:
1034 else:
1035 raise ValueError, "nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)"
1035 raise ValueError, "nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)"
1036
1036
1037 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1037 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1038
1038
1039 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1039 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1040
1040
1041 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1041 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1042
1042
1043 dataOut.profileIndex = profileIndex
1043 dataOut.profileIndex = profileIndex
1044
1044
1045 dataOut.ippSeconds /= self.__nTxs
1045 dataOut.ippSeconds /= self.__nTxs
1046
1046
1047 class SplitProfiles(Operation):
1047 class SplitProfiles(Operation):
1048
1048
1049 def __init__(self, **kwargs):
1049 def __init__(self, **kwargs):
1050
1050
1051 Operation.__init__(self, **kwargs)
1051 Operation.__init__(self, **kwargs)
1052
1052
1053 def run(self, dataOut, n):
1053 def run(self, dataOut, n):
1054
1054
1055 dataOut.flagNoData = True
1055 dataOut.flagNoData = True
1056 profileIndex = None
1056 profileIndex = None
1057
1057
1058 if dataOut.flagDataAsBlock:
1058 if dataOut.flagDataAsBlock:
1059
1059
1060 #nchannels, nprofiles, nsamples
1060 #nchannels, nprofiles, nsamples
1061 shape = dataOut.data.shape
1061 shape = dataOut.data.shape
1062
1062
1063 if shape[2] % n != 0:
1063 if shape[2] % n != 0:
1064 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[2])
1064 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[2])
1065
1065
1066 new_shape = shape[0], shape[1]*n, shape[2]/n
1066 new_shape = shape[0], shape[1]*n, shape[2]/n
1067
1067
1068 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1068 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1069 dataOut.flagNoData = False
1069 dataOut.flagNoData = False
1070
1070
1071 profileIndex = int(dataOut.nProfiles/n) - 1
1071 profileIndex = int(dataOut.nProfiles/n) - 1
1072
1072
1073 else:
1073 else:
1074
1074
1075 raise ValueError, "Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)"
1075 raise ValueError, "Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)"
1076
1076
1077 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1077 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1078
1078
1079 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1079 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1080
1080
1081 dataOut.nProfiles = int(dataOut.nProfiles*n)
1081 dataOut.nProfiles = int(dataOut.nProfiles*n)
1082
1082
1083 dataOut.profileIndex = profileIndex
1083 dataOut.profileIndex = profileIndex
1084
1084
1085 dataOut.ippSeconds /= n
1085 dataOut.ippSeconds /= n
1086
1086
1087 class CombineProfiles(Operation):
1087 class CombineProfiles(Operation):
1088
1088
1089 def __init__(self, **kwargs):
1089 def __init__(self, **kwargs):
1090
1090
1091 Operation.__init__(self, **kwargs)
1091 Operation.__init__(self, **kwargs)
1092
1092
1093 self.__remData = None
1093 self.__remData = None
1094 self.__profileIndex = 0
1094 self.__profileIndex = 0
1095
1095
1096 def run(self, dataOut, n):
1096 def run(self, dataOut, n):
1097
1097
1098 dataOut.flagNoData = True
1098 dataOut.flagNoData = True
1099 profileIndex = None
1099 profileIndex = None
1100
1100
1101 if dataOut.flagDataAsBlock:
1101 if dataOut.flagDataAsBlock:
1102
1102
1103 #nchannels, nprofiles, nsamples
1103 #nchannels, nprofiles, nsamples
1104 shape = dataOut.data.shape
1104 shape = dataOut.data.shape
1105 new_shape = shape[0], shape[1]/n, shape[2]*n
1105 new_shape = shape[0], shape[1]/n, shape[2]*n
1106
1106
1107 if shape[1] % n != 0:
1107 if shape[1] % n != 0:
1108 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[1])
1108 raise ValueError, "Could not split the data, n=%d has to be multiple of %d" %(n, shape[1])
1109
1109
1110 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1110 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1111 dataOut.flagNoData = False
1111 dataOut.flagNoData = False
1112
1112
1113 profileIndex = int(dataOut.nProfiles*n) - 1
1113 profileIndex = int(dataOut.nProfiles*n) - 1
1114
1114
1115 else:
1115 else:
1116
1116
1117 #nchannels, nsamples
1117 #nchannels, nsamples
1118 if self.__remData is None:
1118 if self.__remData is None:
1119 newData = dataOut.data
1119 newData = dataOut.data
1120 else:
1120 else:
1121 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1121 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1122
1122
1123 self.__profileIndex += 1
1123 self.__profileIndex += 1
1124
1124
1125 if self.__profileIndex < n:
1125 if self.__profileIndex < n:
1126 self.__remData = newData
1126 self.__remData = newData
1127 #continue
1127 #continue
1128 return
1128 return
1129
1129
1130 self.__profileIndex = 0
1130 self.__profileIndex = 0
1131 self.__remData = None
1131 self.__remData = None
1132
1132
1133 dataOut.data = newData
1133 dataOut.data = newData
1134 dataOut.flagNoData = False
1134 dataOut.flagNoData = False
1135
1135
1136 profileIndex = dataOut.profileIndex/n
1136 profileIndex = dataOut.profileIndex/n
1137
1137
1138
1138
1139 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1139 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1140
1140
1141 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1141 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1142
1142
1143 dataOut.nProfiles = int(dataOut.nProfiles/n)
1143 dataOut.nProfiles = int(dataOut.nProfiles/n)
1144
1144
1145 dataOut.profileIndex = profileIndex
1145 dataOut.profileIndex = profileIndex
1146
1146
1147 dataOut.ippSeconds *= n
1147 dataOut.ippSeconds *= n
1148
1148
1149 # import collections
1149 # import collections
1150 # from scipy.stats import mode
1150 # from scipy.stats import mode
1151 #
1151 #
1152 # class Synchronize(Operation):
1152 # class Synchronize(Operation):
1153 #
1153 #
1154 # isConfig = False
1154 # isConfig = False
1155 # __profIndex = 0
1155 # __profIndex = 0
1156 #
1156 #
1157 # def __init__(self, **kwargs):
1157 # def __init__(self, **kwargs):
1158 #
1158 #
1159 # Operation.__init__(self, **kwargs)
1159 # Operation.__init__(self, **kwargs)
1160 # # self.isConfig = False
1160 # # self.isConfig = False
1161 # self.__powBuffer = None
1161 # self.__powBuffer = None
1162 # self.__startIndex = 0
1162 # self.__startIndex = 0
1163 # self.__pulseFound = False
1163 # self.__pulseFound = False
1164 #
1164 #
1165 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1165 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1166 #
1166 #
1167 # #Read data
1167 # #Read data
1168 #
1168 #
1169 # powerdB = dataOut.getPower(channel = channel)
1169 # powerdB = dataOut.getPower(channel = channel)
1170 # noisedB = dataOut.getNoise(channel = channel)[0]
1170 # noisedB = dataOut.getNoise(channel = channel)[0]
1171 #
1171 #
1172 # self.__powBuffer.extend(powerdB.flatten())
1172 # self.__powBuffer.extend(powerdB.flatten())
1173 #
1173 #
1174 # dataArray = numpy.array(self.__powBuffer)
1174 # dataArray = numpy.array(self.__powBuffer)
1175 #
1175 #
1176 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1176 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1177 #
1177 #
1178 # maxValue = numpy.nanmax(filteredPower)
1178 # maxValue = numpy.nanmax(filteredPower)
1179 #
1179 #
1180 # if maxValue < noisedB + 10:
1180 # if maxValue < noisedB + 10:
1181 # #No se encuentra ningun pulso de transmision
1181 # #No se encuentra ningun pulso de transmision
1182 # return None
1182 # return None
1183 #
1183 #
1184 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1184 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1185 #
1185 #
1186 # if len(maxValuesIndex) < 2:
1186 # if len(maxValuesIndex) < 2:
1187 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1187 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1188 # return None
1188 # return None
1189 #
1189 #
1190 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1190 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1191 #
1191 #
1192 # #Seleccionar solo valores con un espaciamiento de nSamples
1192 # #Seleccionar solo valores con un espaciamiento de nSamples
1193 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1193 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1194 #
1194 #
1195 # if len(pulseIndex) < 2:
1195 # if len(pulseIndex) < 2:
1196 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1196 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1197 # return None
1197 # return None
1198 #
1198 #
1199 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1199 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1200 #
1200 #
1201 # #remover senales que se distancien menos de 10 unidades o muestras
1201 # #remover senales que se distancien menos de 10 unidades o muestras
1202 # #(No deberian existir IPP menor a 10 unidades)
1202 # #(No deberian existir IPP menor a 10 unidades)
1203 #
1203 #
1204 # realIndex = numpy.where(spacing > 10 )[0]
1204 # realIndex = numpy.where(spacing > 10 )[0]
1205 #
1205 #
1206 # if len(realIndex) < 2:
1206 # if len(realIndex) < 2:
1207 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1207 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1208 # return None
1208 # return None
1209 #
1209 #
1210 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1210 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1211 # realPulseIndex = pulseIndex[realIndex]
1211 # realPulseIndex = pulseIndex[realIndex]
1212 #
1212 #
1213 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1213 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1214 #
1214 #
1215 # print "IPP = %d samples" %period
1215 # print "IPP = %d samples" %period
1216 #
1216 #
1217 # self.__newNSamples = dataOut.nHeights #int(period)
1217 # self.__newNSamples = dataOut.nHeights #int(period)
1218 # self.__startIndex = int(realPulseIndex[0])
1218 # self.__startIndex = int(realPulseIndex[0])
1219 #
1219 #
1220 # return 1
1220 # return 1
1221 #
1221 #
1222 #
1222 #
1223 # def setup(self, nSamples, nChannels, buffer_size = 4):
1223 # def setup(self, nSamples, nChannels, buffer_size = 4):
1224 #
1224 #
1225 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1225 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1226 # maxlen = buffer_size*nSamples)
1226 # maxlen = buffer_size*nSamples)
1227 #
1227 #
1228 # bufferList = []
1228 # bufferList = []
1229 #
1229 #
1230 # for i in range(nChannels):
1230 # for i in range(nChannels):
1231 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1231 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1232 # maxlen = buffer_size*nSamples)
1232 # maxlen = buffer_size*nSamples)
1233 #
1233 #
1234 # bufferList.append(bufferByChannel)
1234 # bufferList.append(bufferByChannel)
1235 #
1235 #
1236 # self.__nSamples = nSamples
1236 # self.__nSamples = nSamples
1237 # self.__nChannels = nChannels
1237 # self.__nChannels = nChannels
1238 # self.__bufferList = bufferList
1238 # self.__bufferList = bufferList
1239 #
1239 #
1240 # def run(self, dataOut, channel = 0):
1240 # def run(self, dataOut, channel = 0):
1241 #
1241 #
1242 # if not self.isConfig:
1242 # if not self.isConfig:
1243 # nSamples = dataOut.nHeights
1243 # nSamples = dataOut.nHeights
1244 # nChannels = dataOut.nChannels
1244 # nChannels = dataOut.nChannels
1245 # self.setup(nSamples, nChannels)
1245 # self.setup(nSamples, nChannels)
1246 # self.isConfig = True
1246 # self.isConfig = True
1247 #
1247 #
1248 # #Append new data to internal buffer
1248 # #Append new data to internal buffer
1249 # for thisChannel in range(self.__nChannels):
1249 # for thisChannel in range(self.__nChannels):
1250 # bufferByChannel = self.__bufferList[thisChannel]
1250 # bufferByChannel = self.__bufferList[thisChannel]
1251 # bufferByChannel.extend(dataOut.data[thisChannel])
1251 # bufferByChannel.extend(dataOut.data[thisChannel])
1252 #
1252 #
1253 # if self.__pulseFound:
1253 # if self.__pulseFound:
1254 # self.__startIndex -= self.__nSamples
1254 # self.__startIndex -= self.__nSamples
1255 #
1255 #
1256 # #Finding Tx Pulse
1256 # #Finding Tx Pulse
1257 # if not self.__pulseFound:
1257 # if not self.__pulseFound:
1258 # indexFound = self.__findTxPulse(dataOut, channel)
1258 # indexFound = self.__findTxPulse(dataOut, channel)
1259 #
1259 #
1260 # if indexFound == None:
1260 # if indexFound == None:
1261 # dataOut.flagNoData = True
1261 # dataOut.flagNoData = True
1262 # return
1262 # return
1263 #
1263 #
1264 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1264 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1265 # self.__pulseFound = True
1265 # self.__pulseFound = True
1266 # self.__startIndex = indexFound
1266 # self.__startIndex = indexFound
1267 #
1267 #
1268 # #If pulse was found ...
1268 # #If pulse was found ...
1269 # for thisChannel in range(self.__nChannels):
1269 # for thisChannel in range(self.__nChannels):
1270 # bufferByChannel = self.__bufferList[thisChannel]
1270 # bufferByChannel = self.__bufferList[thisChannel]
1271 # #print self.__startIndex
1271 # #print self.__startIndex
1272 # x = numpy.array(bufferByChannel)
1272 # x = numpy.array(bufferByChannel)
1273 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1273 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1274 #
1274 #
1275 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1275 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1276 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1276 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1277 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1277 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1278 #
1278 #
1279 # dataOut.data = self.__arrayBuffer
1279 # dataOut.data = self.__arrayBuffer
1280 #
1280 #
1281 # self.__startIndex += self.__newNSamples
1281 # self.__startIndex += self.__newNSamples
1282 #
1282 #
1283 # return
1283 # return
Show file before | Show file after | Hide comments
@@ -1,97 +1,97
1 import argparse
1 import argparse
2
2
3 from schainpy.controller import Project, multiSchain
3 from schainpy.controller import Project, multiSchain
4
4
5 desc = "HF_EXAMPLE"
5 desc = "HF_EXAMPLE"
6
6
7 def fiber(cursor, skip, q, dt):
7 def fiber(cursor, skip, q, dt):
8
8
9 controllerObj = Project()
9 controllerObj = Project()
10
10
11 controllerObj.setup(id='191', name='test01', description=desc)
11 controllerObj.setup(id='191', name='test01', description=desc)
12
12
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
14 path='/home/nanosat/data/sp1_f0',
14 path='/home/nanosat/data/sp1_f0',
15 startDate=dt,
15 startDate=dt,
16 endDate=dt,
16 endDate=dt,
17 startTime="00:00:00",
17 startTime="00:00:00",
18 endTime="23:59:59",
18 endTime="23:59:59",
19 online=0,
19 online=0,
20 #set=1426485881,
20 #set=1426485881,
21 walk=1,
21 walk=1,
22 queue=q,
22 queue=q,
23 cursor=cursor,
23 cursor=cursor,
24 skip=skip,
24 skip=skip,
25 verbose=1
25 verbose=1
26 #timezone=-5*3600
26 #timezone=-5*3600
27 )
27 )
28
28
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
30 #
30 #
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
32 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
32 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
33
33
34 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
34 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
35 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
35 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
36
36
37 #
37 #
38 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
38 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
39 # opObj11.addParameter(name='id', value='1000', format='int')
39 # opObj11.addParameter(name='id', value='1000', format='int')
40 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
40 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
41 # opObj11.addParameter(name='channelList', value='0', format='intlist')
41 # opObj11.addParameter(name='channelList', value='0', format='intlist')
42 # opObj11.addParameter(name='zmin', value='-120', format='float')
42 # opObj11.addParameter(name='zmin', value='-120', format='float')
43 # opObj11.addParameter(name='zmax', value='-70', format='float')
43 # opObj11.addParameter(name='zmax', value='-70', format='float')
44 # opObj11.addParameter(name='save', value='1', format='int')
44 # opObj11.addParameter(name='save', value='1', format='int')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
46
46
47 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
47 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
48 # opObj11.addParameter(name='channelList', value='0', format='intList')
48 # opObj11.addParameter(name='channelList', value='0', format='intList')
49
49
50 # opObj11.addParameter(name='id', value='2000', format='int')
50 # opObj11.addParameter(name='id', value='2000', format='int')
51 # # opObj11.addParameter(name='colormap', value='0', format='bool')
51 # # opObj11.addParameter(name='colormap', value='0', format='bool')
52 # opObj11.addParameter(name='onlySNR', value='1', format='bool')
52 # opObj11.addParameter(name='onlySNR', value='1', format='bool')
53 # opObj11.addParameter(name='DOP', value='0', format='bool')
53 # opObj11.addParameter(name='DOP', value='0', format='bool')
54 # # opObj11.addParameter(name='showSNR', value='1', format='bool')
54 # # opObj11.addParameter(name='showSNR', value='1', format='bool')
55 # # opObj11.addParameter(name='SNRthresh', value='0', format='int')
55 # # opObj11.addParameter(name='SNRthresh', value='0', format='int')
56 # opObj11.addParameter(name='SNRmin', value='-10', format='int')
56 # opObj11.addParameter(name='SNRmin', value='-10', format='int')
57 # opObj11.addParameter(name='SNRmax', value='30', format='int')
57 # opObj11.addParameter(name='SNRmax', value='30', format='int')
58
58
59 # opObj11.addParameter(name='showSNR', value='1', format='int')
59 # opObj11.addParameter(name='showSNR', value='1', format='int')
60 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
60 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
61 # # opObj11.addParameter(name='xmin', value='0', format='float')
61 # # opObj11.addParameter(name='xmin', value='0', format='float')
62 # opObj11.addParameter(name='xmin', value='0', format='float')
62 # opObj11.addParameter(name='xmin', value='0', format='float')
63 # opObj11.addParameter(name='xmax', value='24', format='float')
63 # opObj11.addParameter(name='xmax', value='24', format='float')
64
64
65 # opObj11.addParameter(name='zmin', value='-110', format='float')
65 # opObj11.addParameter(name='zmin', value='-110', format='float')
66 # opObj11.addParameter(name='zmax', value='-70', format='float')
66 # opObj11.addParameter(name='zmax', value='-70', format='float')
67 # opObj11.addParameter(name='save', value='0', format='int')
67 # opObj11.addParameter(name='save', value='0', format='int')
68 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
68 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
69 #
69 #
70 opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
70 opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
71 opObj12.addParameter(name='zeromq', value=1, format='int')
71 opObj12.addParameter(name='zeromq', value=1, format='int')
72 opObj12.addParameter(name='verbose', value=0, format='bool')
72 opObj12.addParameter(name='verbose', value=0, format='bool')
73
73
74
74
75 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
75 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
76 # opObj13.addParameter(name='zeromq', value=1, format='int')
76 # opObj13.addParameter(name='zeromq', value=1, format='int')
77 # opObj13.addParameter(name='server', value="juanca", format='str')
77 # opObj13.addParameter(name='server', value="juanca", format='str')
78
78
79 opObj12.addParameter(name='delay', value=0, format='int')
79 opObj12.addParameter(name='delay', value=0, format='int')
80
80
81
81
82 # print "Escribiendo el archivo XML"
82 # print "Escribiendo el archivo XML"
83 # controllerObj.writeXml(filename)
83 # controllerObj.writeXml(filename)
84 # print "Leyendo el archivo XML"
84 # print "Leyendo el archivo XML"
85 # controllerObj.readXml(filename)
85 # controllerObj.readXml(filename)
86
86
87
87
88 # timeit.timeit('controllerObj.run()', number=2)
88 # timeit.timeit('controllerObj.run()', number=2)
89
89
90 controllerObj.start()
90 controllerObj.start()
91
91
92
92
93 if __name__ == '__main__':
93 if __name__ == '__main__':
94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
95 parser.add_argument('--nProcess', default=1, type=int)
95 parser.add_argument('--nProcess', default=1, type=int)
96 args = parser.parse_args()
96 args = parser.parse_args()
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/28')
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/26')
Show file before | Show file after | Hide comments
@@ -1,97 +1,94
1 import argparse
1 import argparse
2
2
3 from schainpy.controller import Project, multiSchain
3 from schainpy.controller import Project, multiSchain
4
4
5 desc = "HF_EXAMPLE"
5 desc = "HF_EXAMPLE"
6
6
7 def fiber(cursor, skip, q, dt):
7 def fiber(cursor, skip, q, dt):
8
8
9 controllerObj = Project()
9 controllerObj = Project()
10
10
11 controllerObj.setup(id='191', name='test01', description=desc)
11 controllerObj.setup(id='191', name='test01', description=desc)
12
12
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
13 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
14 path='/home/nanosat/data/julia',
14 path='/home/nanosat/data/julia',
15 startDate=dt,
15 startDate=dt,
16 endDate=dt,
16 endDate=dt,
17 startTime="00:00:00",
17 startTime="00:00:00",
18 endTime="23:59:59",
18 endTime="23:59:59",
19 online=0,
19 online=0,
20 #set=1426485881,
20 #set=1426485881,
21 delay=10,
21 delay=10,
22 walk=1,
22 walk=1,
23 queue=q,
23 queue=q,
24 cursor=cursor,
24 cursor=cursor,
25 skip=skip,
25 skip=skip,
26 #timezone=-5*3600
26 #timezone=-5*3600
27 )
27 )
28
28
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
29 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
30 #
30 #
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
31 procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
32 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
32 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
33
33
34 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
34 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
35 # opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
35 # opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
36
36
37 #
37 #
38 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
38 opObj11 = procUnitConfObj2.addOperation(name='RTIPlot', optype='other')
39 # opObj11.addParameter(name='id', value='1000', format='int')
39 opObj11.addParameter(name='id', value='1000', format='int')
40 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
40 opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
41 # opObj11.addParameter(name='channelList', value='0', format='intlist')
41 opObj11.addParameter(name='xmin', value='0', format='int')
42 # opObj11.addParameter(name='zmin', value='-120', format='float')
42 opObj11.addParameter(name='xmax', value='24', format='int')
43 # opObj11.addParameter(name='zmax', value='-70', format='float')
44 # opObj11.addParameter(name='save', value='1', format='int')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
46
43
47 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
44 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
48 # opObj11.addParameter(name='channelList', value='0', format='intList')
45 # opObj11.addParameter(name='channelList', value='0', format='intList')
49 #
46 #
50 # opObj11.addParameter(name='id', value='2000', format='int')
47 # opObj11.addParameter(name='id', value='2000', format='int')
51 # # opObj11.addParameter(name='colormap', value='0', format='bool')
48 # # opObj11.addParameter(name='colormap', value='0', format='bool')
52 # opObj11.addParameter(name='onlySNR', value='1', format='bool')
49 # opObj11.addParameter(name='onlySNR', value='1', format='bool')
53 # opObj11.addParameter(name='DOP', value='0', format='bool')
50 # opObj11.addParameter(name='DOP', value='0', format='bool')
54 # opObj11.addParameter(name='showSNR', value='1', format='bool')
51 # opObj11.addParameter(name='showSNR', value='1', format='bool')
55 # opObj11.addParameter(name='SNRthresh', value='0', format='int')
52 # opObj11.addParameter(name='SNRthresh', value='0', format='int')
56 # opObj11.addParameter(name='SNRmin', value='-10', format='int')
53 # opObj11.addParameter(name='SNRmin', value='-10', format='int')
57 # opObj11.addParameter(name='SNRmax', value='30', format='int')
54 # opObj11.addParameter(name='SNRmax', value='30', format='int')
58
55
59 # opObj11.addParameter(name='showSNR', value='1', format='int')
56 # opObj11.addParameter(name='showSNR', value='1', format='int')
60 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
57 # # opObj11.addParameter(name='channelList', value='0', format='intlist')
61 # # opObj11.addParameter(name='xmin', value='0', format='float')
58 # # opObj11.addParameter(name='xmin', value='0', format='float')
62 # opObj11.addParameter(name='xmin', value='0', format='float')
59 # opObj11.addParameter(name='xmin', value='0', format='float')
63 # opObj11.addParameter(name='xmax', value='24', format='float')
60 # opObj11.addParameter(name='xmax', value='24', format='float')
64
61
65 # opObj11.addParameter(name='zmin', value='-110', format='float')
62 # opObj11.addParameter(name='zmin', value='-110', format='float')
66 # opObj11.addParameter(name='zmax', value='-70', format='float')
63 # opObj11.addParameter(name='zmax', value='-70', format='float')
67 # opObj11.addParameter(name='save', value='0', format='int')
64 # opObj11.addParameter(name='save', value='0', format='int')
68 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
65 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
69 #
66 #
70 opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
67 # opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
71 opObj12.addParameter(name='zeromq', value=1, format='int')
68 # opObj12.addParameter(name='zeromq', value=1, format='int')
72 # opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
69 # opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
73
70
74
71
75 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
72 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
76 # opObj13.addParameter(name='zeromq', value=1, format='int')
73 # opObj13.addParameter(name='zeromq', value=1, format='int')
77 # opObj13.addParameter(name='server', value="juanca", format='str')
74 # opObj13.addParameter(name='server', value="juanca", format='str')
78
75
79 # opObj12.addParameter(name='delay', value=1, format='int')
76 # opObj12.addParameter(name='delay', value=1, format='int')
80
77
81
78
82 # print "Escribiendo el archivo XML"
79 # print "Escribiendo el archivo XML"
83 # controllerObj.writeXml(filename)
80 # controllerObj.writeXml(filename)
84 # print "Leyendo el archivo XML"
81 # print "Leyendo el archivo XML"
85 # controllerObj.readXml(filename)
82 # controllerObj.readXml(filename)
86
83
87
84
88 # timeit.timeit('controllerObj.run()', number=2)
85 # timeit.timeit('controllerObj.run()', number=2)
89
86
90 controllerObj.start()
87 controllerObj.start()
91
88
92
89
93 if __name__ == '__main__':
90 if __name__ == '__main__':
94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
91 parser = argparse.ArgumentParser(description='Set number of parallel processes')
95 parser.add_argument('--nProcess', default=1, type=int)
92 parser.add_argument('--nProcess', default=1, type=int)
96 args = parser.parse_args()
93 args = parser.parse_args()
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/19')
94 multiSchain(fiber, nProcess=args.nProcess, startDate='2016/08/19', endDate='2016/08/19')
Show file before | Show file after | Hide comments
@@ -1,59 +1,61
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 '''
2 '''
3 Created on Jul 7, 2014
3 Created on Jul 7, 2014
4
4
5 @author: roj-idl71
5 @author: roj-idl71
6 '''
6 '''
7 import os, sys
7 import os, sys
8
8
9 from schainpy.controller import Project
9 from schainpy.controller import Project
10
10
11 if __name__ == '__main__':
11 if __name__ == '__main__':
12 desc = "Segundo Test"
12 desc = "Segundo Test"
13
13
14 controllerObj = Project()
14 controllerObj = Project()
15 controllerObj.setup(id='191', name='test01', description=desc)
15 controllerObj.setup(id='191', name='test01', description=desc)
16
16
17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
18 proc1.addParameter(name='realtime', value='0', format='bool')
18 proc1.addParameter(name='realtime', value='0', format='bool')
19 proc1.addParameter(name='plottypes', value='rti,coh,phase', format='str')
19 proc1.addParameter(name='plottypes', value='snr', format='str')
20 proc1.addParameter(name='throttle', value='10', format='int')
20 proc1.addParameter(name='throttle', value='5', format='int')
21 proc1.addParameter(name='interactive', value='0', format='bool')
21 proc1.addParameter(name='interactive', value='0', format='bool')
22 # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
22 # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
23 ## TODO Agregar direccion de server de publicacion a graficos como variable
23 ## TODO Agregar direccion de server de publicacion a graficos como variable
24
24 op3 = proc1.addOperation(name='PlotSNRData', optype='other')
25 op1 = proc1.addOperation(name='PlotRTIData', optype='other')
25 op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
26 op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
26 op3.addParameter(name='zmax', value='30', format='int')
27 op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
27 op3.addParameter(name='zmin', value='-10', format='int')
28 op1.addParameter(name='show', value='0', format='bool')
28 op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
29 op1.addParameter(name='colormap', value='jet', format='str')
29 op3.addParameter(name='show', value='0', format='bool')
30 #
31 op2 = proc1.addOperation(name='PlotCOHData', optype='other')
32 op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
33 op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
34 op2.addParameter(name='colormap', value='jet', format='str')
35 op2.addParameter(name='show', value='0', format='bool')
36 # # #
37 op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
38 op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
39 op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
40 op6.addParameter(name='show', value='1', format='bool')
41 # #
42 # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
43 # # proc2.addParameter(name='server', value='juanca', format='str')
44 # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
45 # #
46 # op3 = proc1.addOperation(name='PlotSNRData', optype='other')
47 # op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
48 # op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
49 # op3.addParameter(name='show', value='0', format='bool')
50 # #
30 # #
51 # op4 = proc1.addOperation(name='PlotDOPData', optype='other')
31 # op4 = proc1.addOperation(name='PlotDOPData', optype='other')
52 # op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
32 # op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
53 # op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
33 # op4.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
54 # op4.addParameter(name='show', value='0', format='bool')
34 # op4.addParameter(name='show', value='0', format='bool')
55 # op4.addParameter(name='colormap', value='jet', format='str')
35 # op4.addParameter(name='colormap', value='jet', format='str')
56
36
37 # op1 = proc1.addOperation(name='PlotRTIData', optype='other')
38 # op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
39 # op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
40 # op1.addParameter(name='show', value='0', format='bool')
41 # op1.addParameter(name='colormap', value='jet', format='str')
42 # #
43 # op2 = proc1.addOperation(name='PlotCOHData', optype='other')
44 # op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
45 # op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
46 # op2.addParameter(name='colormap', value='jet', format='str')
47 # op2.addParameter(name='show', value='0', format='bool')
48 # # # #
49 # op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
50 # op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
51 # op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
52 # op6.addParameter(name='show', value='1', format='bool')
53 # #
54 # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
55 # # proc2.addParameter(name='server', value='juanca', format='str')
56 # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
57 # #
58
57
59
58
60
59 controllerObj.start()
61 controllerObj.start()
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="Segundo Test" id="191" name="test01"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/jchavez/jicamarca/jro_data/rawdata" /><Parameter format="date" id="191113" name="startDate" value="2010/10/28" /><Parameter format="date" id="191114" name="endDate" value="2017/10/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /><Parameter format="int" id="191119" name="walk" value="0" /></Operation><Operation id="19112" name="printNumberOfBlock" priority="2" type="self" /></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="DigitalRFWriter" priority="2" type="other"><Parameter format="str" id="191221" name="path" value="/home/jchavez/jicamarca/data_sink/" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="Segundo Test" id="191" name="test01"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/jchavez/jicamarca/jro_data/rawdata" /><Parameter format="date" id="191113" name="startDate" value="2010/10/28" /><Parameter format="date" id="191114" name="endDate" value="2017/10/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /><Parameter format="int" id="191119" name="walk" value="0" /></Operation><Operation id="19112" name="printNumberOfBlock" priority="2" type="self" /></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="DigitalRFWriter" priority="2" type="other"><Parameter format="str" id="191221" name="path" value="/home/jchavez/jicamarca/data_sink/" /></Operation></ProcUnit></Project>
General Comments 0
You need to be logged in to leave comments. Login now