##// END OF EJS Templates
schain
RSS
A new SendToServer Unit has been created to upload files to a remote server....
Miguel Valdez -
r573:e93e8802031b
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,27
1 '''
2 @author: roj-idl71
3 '''
4 #USED IN jroplot_spectra.py
5 RTI_CODE = 0 #Range time intensity (RTI).
6 SPEC_CODE = 1 #Spectra (and Cross-spectra) information.
7 CROSS_CODE = 2 #Cross-Correlation information.
8 COH_CODE = 3 #Coherence map.
9 BASE_CODE = 4 #Base lines graphic.
10 ROW_CODE = 5 #Row Spectra.
11 TOTAL_CODE = 6 #Total Power.
12 DRIFT_CODE = 7 #Drifts graphics.
13 HEIGHT_CODE = 8 #Height profile.
14 PHASE_CODE = 9 #Signal Phase.
15
16 POWER_CODE = 16
17 NOISE_CODE = 17
18 BEACON_CODE = 18
19
20 #USED IN jroplot_parameters.py
21
22 MOMENTS_CODE = 20
23 SKYMAP_CODE = 21
24 WIND_CODE = 22
25 PARMS_CODE = 23
26 SPECFIT_CODE = 24
27 EWDRIFT_CODE = 25
Show file before | Show file after | Hide comments
@@ -0,0 +1,464
1 # scp.py
2 # Copyright (C) 2008 James Bardin <j.bardin@gmail.com>
3
4 """
5 Utilities for sending files over ssh using the scp1 protocol.
6 """
7
8 __version__ = '0.10.0'
9
10 import locale
11 import os
12 import re
13 from socket import timeout as SocketTimeout
14
15
16 # this is quote from the shlex module, added in py3.3
17 _find_unsafe = re.compile(br'[^\w@%+=:,./~-]').search
18
19
20 def _sh_quote(s):
21 """Return a shell-escaped version of the string `s`."""
22 if not s:
23 return b""
24 if _find_unsafe(s) is None:
25 return s
26
27 # use single quotes, and put single quotes into double quotes
28 # the string $'b is then quoted as '$'"'"'b'
29 return b"'" + s.replace(b"'", b"'\"'\"'") + b"'"
30
31
32 # Unicode conversion functions; assume UTF-8
33
34 def asbytes(s):
35 """Turns unicode into bytes, if needed.
36 Assumes UTF-8.
37 """
38 if isinstance(s, bytes):
39 return s
40 else:
41 return s.encode('utf-8')
42
43
44 def asunicode(s):
45 """Turns bytes into unicode, if needed.
46 Uses UTF-8.
47 """
48 if isinstance(s, bytes):
49 return s.decode('utf-8', 'replace')
50 else:
51 return s
52
53
54 # os.path.sep is unicode on Python 3, no matter the platform
55 bytes_sep = asbytes(os.path.sep)
56
57
58 # Unicode conversion function for Windows
59 # Used to convert local paths if the local machine is Windows
60
61 def asunicode_win(s):
62 """Turns bytes into unicode, if needed.
63 """
64 if isinstance(s, bytes):
65 return s.decode(locale.getpreferredencoding())
66 else:
67 return s
68
69
70 class SCPClient(object):
71 """
72 An scp1 implementation, compatible with openssh scp.
73 Raises SCPException for all transport related errors. Local filesystem
74 and OS errors pass through.
75 Main public methods are .put and .get
76 The get method is controlled by the remote scp instance, and behaves
77 accordingly. This means that symlinks are resolved, and the transfer is
78 halted after too many levels of symlinks are detected.
79 The put method uses os.walk for recursion, and sends files accordingly.
80 Since scp doesn't support symlinks, we send file symlinks as the file
81 (matching scp behaviour), but we make no attempt at symlinked directories.
82 """
83 def __init__(self, transport, buff_size=16384, socket_timeout=5.0,
84 progress=None, sanitize=_sh_quote):
85 """
86 Create an scp1 client.
87 @param transport: an existing paramiko L{Transport}
88 @type transport: L{Transport}
89 @param buff_size: size of the scp send buffer.
90 @type buff_size: int
91 @param socket_timeout: channel socket timeout in seconds
92 @type socket_timeout: float
93 @param progress: callback - called with (filename, size, sent) during
94 transfers
95 @param sanitize: function - called with filename, should return
96 safe or escaped string. Uses _sh_quote by default.
97 @type progress: function(string, int, int)
98 """
99 self.transport = transport
100 self.buff_size = buff_size
101 self.socket_timeout = socket_timeout
102 self.channel = None
103 self.preserve_times = False
104 self._progress = progress
105 self._recv_dir = b''
106 self._rename = False
107 self._utime = None
108 self.sanitize = sanitize
109 self._dirtimes = {}
110
111 def __enter__(self):
112 self.channel = self._open()
113 return self
114
115 def __exit__(self, type, value, traceback):
116 self.close()
117
118 def put(self, files, remote_path=b'.',
119 recursive=False, preserve_times=False):
120 """
121 Transfer files to remote host.
122 @param files: A single path, or a list of paths to be transfered.
123 recursive must be True to transfer directories.
124 @type files: string OR list of strings
125 @param remote_path: path in which to receive the files on the remote
126 host. defaults to '.'
127 @type remote_path: str
128 @param recursive: transfer files and directories recursively
129 @type recursive: bool
130 @param preserve_times: preserve mtime and atime of transfered files
131 and directories.
132 @type preserve_times: bool
133 """
134 self.preserve_times = preserve_times
135 self.channel = self._open()
136 self._pushed = 0
137 self.channel.settimeout(self.socket_timeout)
138 scp_command = (b'scp -t ', b'scp -r -t ')[recursive]
139 self.channel.exec_command(scp_command +
140 self.sanitize(asbytes(remote_path)))
141 self._recv_confirm()
142
143 if not isinstance(files, (list, tuple)):
144 files = [files]
145
146 if recursive:
147 self._send_recursive(files)
148 else:
149 self._send_files(files)
150
151 self.close()
152
153 def get(self, remote_path, local_path='',
154 recursive=False, preserve_times=False):
155 """
156 Transfer files from remote host to localhost
157 @param remote_path: path to retreive from remote host. since this is
158 evaluated by scp on the remote host, shell wildcards and
159 environment variables may be used.
160 @type remote_path: str
161 @param local_path: path in which to receive files locally
162 @type local_path: str
163 @param recursive: transfer files and directories recursively
164 @type recursive: bool
165 @param preserve_times: preserve mtime and atime of transfered files
166 and directories.
167 @type preserve_times: bool
168 """
169 if not isinstance(remote_path, (list, tuple)):
170 remote_path = [remote_path]
171 remote_path = [self.sanitize(asbytes(r)) for r in remote_path]
172 self._recv_dir = local_path or os.getcwd()
173 self._rename = (len(remote_path) == 1 and
174 not os.path.isdir(os.path.abspath(local_path)))
175 if len(remote_path) > 1:
176 if not os.path.exists(self._recv_dir):
177 raise SCPException("Local path '%s' does not exist" %
178 asunicode(self._recv_dir))
179 elif not os.path.isdir(self._recv_dir):
180 raise SCPException("Local path '%s' is not a directory" %
181 asunicode(self._recv_dir))
182 rcsv = (b'', b' -r')[recursive]
183 prsv = (b'', b' -p')[preserve_times]
184 self.channel = self._open()
185 self._pushed = 0
186 self.channel.settimeout(self.socket_timeout)
187 self.channel.exec_command(b"scp" +
188 rcsv +
189 prsv +
190 b" -f " +
191 b' '.join(remote_path))
192 self._recv_all()
193 self.close()
194
195 def _open(self):
196 """open a scp channel"""
197 if self.channel is None:
198 self.channel = self.transport.open_session()
199
200 return self.channel
201
202 def close(self):
203 """close scp channel"""
204 if self.channel is not None:
205 self.channel.close()
206 self.channel = None
207
208 def _read_stats(self, name):
209 """return just the file stats needed for scp"""
210 if os.name == 'nt':
211 name = asunicode(name)
212 stats = os.stat(name)
213 mode = oct(stats.st_mode)[-4:]
214 size = stats.st_size
215 atime = int(stats.st_atime)
216 mtime = int(stats.st_mtime)
217 return (mode, size, mtime, atime)
218
219 def _send_files(self, files):
220 for name in files:
221 basename = asbytes(os.path.basename(name))
222 (mode, size, mtime, atime) = self._read_stats(name)
223 if self.preserve_times:
224 self._send_time(mtime, atime)
225 file_hdl = open(name, 'rb')
226
227 # The protocol can't handle \n in the filename.
228 # Quote them as the control sequence \^J for now,
229 # which is how openssh handles it.
230 self.channel.sendall(("C%s %d " % (mode, size)).encode('ascii') +
231 basename.replace(b'\n', b'\\^J') + b"\n")
232 self._recv_confirm()
233 file_pos = 0
234 if self._progress:
235 if size == 0:
236 # avoid divide-by-zero
237 self._progress(basename, 1, 1)
238 else:
239 self._progress(basename, size, 0)
240 buff_size = self.buff_size
241 chan = self.channel
242 while file_pos < size:
243 chan.sendall(file_hdl.read(buff_size))
244 file_pos = file_hdl.tell()
245 if self._progress:
246 self._progress(basename, size, file_pos)
247 chan.sendall('\x00')
248 file_hdl.close()
249 self._recv_confirm()
250
251 def _chdir(self, from_dir, to_dir):
252 # Pop until we're one level up from our next push.
253 # Push *once* into to_dir.
254 # This is dependent on the depth-first traversal from os.walk
255
256 # add path.sep to each when checking the prefix, so we can use
257 # path.dirname after
258 common = os.path.commonprefix([from_dir + bytes_sep,
259 to_dir + bytes_sep])
260 # now take the dirname, since commonprefix is character based,
261 # and we either have a seperator, or a partial name
262 common = os.path.dirname(common)
263 cur_dir = from_dir.rstrip(bytes_sep)
264 while cur_dir != common:
265 cur_dir = os.path.split(cur_dir)[0]
266 self._send_popd()
267 # now we're in our common base directory, so on
268 self._send_pushd(to_dir)
269
270 def _send_recursive(self, files):
271 for base in files:
272 if not os.path.isdir(base):
273 # filename mixed into the bunch
274 self._send_files([base])
275 continue
276 last_dir = asbytes(base)
277 for root, dirs, fls in os.walk(base):
278 self._chdir(last_dir, asbytes(root))
279 self._send_files([os.path.join(root, f) for f in fls])
280 last_dir = asbytes(root)
281 # back out of the directory
282 while self._pushed > 0:
283 self._send_popd()
284
285 def _send_pushd(self, directory):
286 (mode, size, mtime, atime) = self._read_stats(directory)
287 basename = asbytes(os.path.basename(directory))
288 if self.preserve_times:
289 self._send_time(mtime, atime)
290 self.channel.sendall(('D%s 0 ' % mode).encode('ascii') +
291 basename.replace(b'\n', b'\\^J') + b'\n')
292 self._recv_confirm()
293 self._pushed += 1
294
295 def _send_popd(self):
296 self.channel.sendall('E\n')
297 self._recv_confirm()
298 self._pushed -= 1
299
300 def _send_time(self, mtime, atime):
301 self.channel.sendall(('T%d 0 %d 0\n' % (mtime, atime)).encode('ascii'))
302 self._recv_confirm()
303
304 def _recv_confirm(self):
305 # read scp response
306 msg = b''
307 try:
308 msg = self.channel.recv(512)
309 except SocketTimeout:
310 raise SCPException('Timout waiting for scp response')
311 # slice off the first byte, so this compare will work in py2 and py3
312 if msg and msg[0:1] == b'\x00':
313 return
314 elif msg and msg[0:1] == b'\x01':
315 raise SCPException(asunicode(msg[1:]))
316 elif self.channel.recv_stderr_ready():
317 msg = self.channel.recv_stderr(512)
318 raise SCPException(asunicode(msg))
319 elif not msg:
320 raise SCPException('No response from server')
321 else:
322 raise SCPException('Invalid response from server', msg)
323
324 def _recv_all(self):
325 # loop over scp commands, and receive as necessary
326 command = {b'C': self._recv_file,
327 b'T': self._set_time,
328 b'D': self._recv_pushd,
329 b'E': self._recv_popd}
330 while not self.channel.closed:
331 # wait for command as long as we're open
332 self.channel.sendall('\x00')
333 msg = self.channel.recv(1024)
334 if not msg: # chan closed while recving
335 break
336 assert msg[-1:] == b'\n'
337 msg = msg[:-1]
338 code = msg[0:1]
339 try:
340 command[code](msg[1:])
341 except KeyError:
342 raise SCPException(asunicode(msg[1:]))
343 # directory times can't be set until we're done writing files
344 self._set_dirtimes()
345
346 def _set_time(self, cmd):
347 try:
348 times = cmd.split(b' ')
349 mtime = int(times[0])
350 atime = int(times[2]) or mtime
351 except:
352 self.channel.send(b'\x01')
353 raise SCPException('Bad time format')
354 # save for later
355 self._utime = (atime, mtime)
356
357 def _recv_file(self, cmd):
358 chan = self.channel
359 parts = cmd.strip().split(b' ', 2)
360
361 try:
362 mode = int(parts[0], 8)
363 size = int(parts[1])
364 if self._rename:
365 path = self._recv_dir
366 self._rename = False
367 elif os.name == 'nt':
368 path = os.path.join(asunicode_win(self._recv_dir),
369 parts[2].decode('utf-8'))
370 else:
371 path = os.path.join(asbytes(self._recv_dir),
372 parts[2])
373 except:
374 chan.send('\x01')
375 chan.close()
376 raise SCPException('Bad file format')
377
378 try:
379 file_hdl = open(path, 'wb')
380 except IOError as e:
381 chan.send(b'\x01' + str(e).encode('utf-8'))
382 chan.close()
383 raise
384
385 if self._progress:
386 if size == 0:
387 # avoid divide-by-zero
388 self._progress(path, 1, 1)
389 else:
390 self._progress(path, size, 0)
391 buff_size = self.buff_size
392 pos = 0
393 chan.send(b'\x00')
394 try:
395 while pos < size:
396 # we have to make sure we don't read the final byte
397 if size - pos <= buff_size:
398 buff_size = size - pos
399 file_hdl.write(chan.recv(buff_size))
400 pos = file_hdl.tell()
401 if self._progress:
402 self._progress(path, size, pos)
403
404 msg = chan.recv(512)
405 if msg and msg[0:1] != b'\x00':
406 raise SCPException(asunicode(msg[1:]))
407 except SocketTimeout:
408 chan.close()
409 raise SCPException('Error receiving, socket.timeout')
410
411 file_hdl.truncate()
412 try:
413 os.utime(path, self._utime)
414 self._utime = None
415 os.chmod(path, mode)
416 # should we notify the other end?
417 finally:
418 file_hdl.close()
419 # '\x00' confirmation sent in _recv_all
420
421 def _recv_pushd(self, cmd):
422 parts = cmd.split(b' ', 2)
423 try:
424 mode = int(parts[0], 8)
425 if self._rename:
426 path = self._recv_dir
427 self._rename = False
428 elif os.name == 'nt':
429 path = os.path.join(asunicode_win(self._recv_dir),
430 parts[2].decode('utf-8'))
431 else:
432 path = os.path.join(asbytes(self._recv_dir),
433 parts[2])
434 except:
435 self.channel.send(b'\x01')
436 raise SCPException('Bad directory format')
437 try:
438 if not os.path.exists(path):
439 os.mkdir(path, mode)
440 elif os.path.isdir(path):
441 os.chmod(path, mode)
442 else:
443 raise SCPException('%s: Not a directory' % path)
444 self._dirtimes[path] = (self._utime)
445 self._utime = None
446 self._recv_dir = path
447 except (OSError, SCPException) as e:
448 self.channel.send(b'\x01' + asbytes(str(e)))
449 raise
450
451 def _recv_popd(self, *cmd):
452 self._recv_dir = os.path.split(self._recv_dir)[0]
453
454 def _set_dirtimes(self):
455 try:
456 for d in self._dirtimes:
457 os.utime(d, self._dirtimes[d])
458 finally:
459 self._dirtimes = {}
460
461
462 class SCPException(Exception):
463 """SCP exception class"""
464 pass No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,844 +1,855
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5 from xml.etree.ElementTree import Element, SubElement
6 6 from xml.etree import ElementTree as ET
7 7 from xml.dom import minidom
8 8
9 9 #import datetime
10 10 from model import *
11 11
12 12 import ast
13 13
14 14 def prettify(elem):
15 15 """Return a pretty-printed XML string for the Element.
16 16 """
17 17 rough_string = ET.tostring(elem, 'utf-8')
18 18 reparsed = minidom.parseString(rough_string)
19 19 return reparsed.toprettyxml(indent=" ")
20 20
21 21 class ParameterConf():
22 22
23 23 id = None
24 24 name = None
25 25 value = None
26 26 format = None
27 27
28 28 __formated_value = None
29 29
30 30 ELEMENTNAME = 'Parameter'
31 31
32 32 def __init__(self):
33 33
34 34 self.format = 'str'
35 35
36 36 def getElementName(self):
37 37
38 38 return self.ELEMENTNAME
39 39
40 40 def getValue(self):
41 41
42 42 if self.__formated_value != None:
43 43
44 44 return self.__formated_value
45 45
46 46 value = self.value
47 47
48 48 if self.format == 'bool':
49 49 value = int(value)
50 50
51 51 if self.format == 'list':
52 52 strList = value.split(',')
53 53
54 54 self.__formated_value = strList
55 55
56 56 return self.__formated_value
57 57
58 58 if self.format == 'intlist':
59 59 """
60 60 Example:
61 61 value = (0,1,2)
62 62 """
63 63 strList = value.split(',')
64 64 intList = [int(x) for x in strList]
65 65
66 66 self.__formated_value = intList
67 67
68 68 return self.__formated_value
69 69
70 70 if self.format == 'floatlist':
71 71 """
72 72 Example:
73 73 value = (0.5, 1.4, 2.7)
74 74 """
75 75 strList = value.split(',')
76 76 floatList = [float(x) for x in strList]
77 77
78 78 self.__formated_value = floatList
79 79
80 80 return self.__formated_value
81 81
82 82 if self.format == 'date':
83 83 strList = value.split('/')
84 84 intList = [int(x) for x in strList]
85 85 date = datetime.date(intList[0], intList[1], intList[2])
86 86
87 87 self.__formated_value = date
88 88
89 89 return self.__formated_value
90 90
91 91 if self.format == 'time':
92 92 strList = value.split(':')
93 93 intList = [int(x) for x in strList]
94 94 time = datetime.time(intList[0], intList[1], intList[2])
95 95
96 96 self.__formated_value = time
97 97
98 98 return self.__formated_value
99 99
100 100 if self.format == 'pairslist':
101 101 """
102 102 Example:
103 103 value = (0,1),(1,2)
104 104 """
105 105
106 106 value = value.replace('(', '')
107 107 value = value.replace(')', '')
108 108
109 109 strList = value.split(',')
110 110 intList = [int(item) for item in strList]
111 111 pairList = []
112 112 for i in range(len(intList)/2):
113 113 pairList.append((intList[i*2], intList[i*2 + 1]))
114 114
115 115 self.__formated_value = pairList
116 116
117 117 return self.__formated_value
118 118
119 119 if self.format == 'multilist':
120 120 """
121 121 Example:
122 122 value = (0,1,2),(3,4,5)
123 123 """
124 124 multiList = ast.literal_eval(value)
125 125
126 126 self.__formated_value = multiList
127 127
128 128 return self.__formated_value
129 129
130 130 format_func = eval(self.format)
131 131
132 132 self.__formated_value = format_func(value)
133 133
134 134 return self.__formated_value
135 135
136 136 def setup(self, id, name, value, format='str'):
137 137
138 138 self.id = id
139 139 self.name = name
140 140 self.value = str(value)
141 141 self.format = str.lower(format)
142 142
143 143 def makeXml(self, opElement):
144 144
145 145 parmElement = SubElement(opElement, self.ELEMENTNAME)
146 146 parmElement.set('id', str(self.id))
147 147 parmElement.set('name', self.name)
148 148 parmElement.set('value', self.value)
149 149 parmElement.set('format', self.format)
150 150
151 151 def readXml(self, parmElement):
152 152
153 153 self.id = parmElement.get('id')
154 154 self.name = parmElement.get('name')
155 155 self.value = parmElement.get('value')
156 156 self.format = str.lower(parmElement.get('format'))
157 157
158 158 #Compatible with old signal chain version
159 159 if self.format == 'int' and self.name == 'idfigure':
160 160 self.name = 'id'
161 161
162 162 def printattr(self):
163 163
164 164 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
165 165
166 166 class OperationConf():
167 167
168 168 id = None
169 169 name = None
170 170 priority = None
171 171 type = None
172 172
173 173 parmConfObjList = []
174 174
175 175 ELEMENTNAME = 'Operation'
176 176
177 177 def __init__(self):
178 178
179 179 self.id = 0
180 180 self.name = None
181 181 self.priority = None
182 182 self.type = 'self'
183 183
184 184
185 185 def __getNewId(self):
186 186
187 187 return int(self.id)*10 + len(self.parmConfObjList) + 1
188 188
189 189 def getElementName(self):
190 190
191 191 return self.ELEMENTNAME
192 192
193 193 def getParameterObjList(self):
194 194
195 195 return self.parmConfObjList
196 196
197 197 def setup(self, id, name, priority, type):
198 198
199 199 self.id = id
200 200 self.name = name
201 201 self.type = type
202 202 self.priority = priority
203 203
204 204 self.parmConfObjList = []
205 205
206 206 def addParameter(self, name, value, format='str'):
207 207
208 208 id = self.__getNewId()
209 209
210 210 parmConfObj = ParameterConf()
211 211 parmConfObj.setup(id, name, value, format)
212 212
213 213 self.parmConfObjList.append(parmConfObj)
214 214
215 215 return parmConfObj
216 216
217 217 def makeXml(self, upElement):
218 218
219 219 opElement = SubElement(upElement, self.ELEMENTNAME)
220 220 opElement.set('id', str(self.id))
221 221 opElement.set('name', self.name)
222 222 opElement.set('type', self.type)
223 223 opElement.set('priority', str(self.priority))
224 224
225 225 for parmConfObj in self.parmConfObjList:
226 226 parmConfObj.makeXml(opElement)
227 227
228 228 def readXml(self, opElement):
229 229
230 230 self.id = opElement.get('id')
231 231 self.name = opElement.get('name')
232 232 self.type = opElement.get('type')
233 233 self.priority = opElement.get('priority')
234 234
235 235 #Compatible with old signal chain version
236 236 #Use of 'run' method instead 'init'
237 237 if self.type == 'self' and self.name == 'init':
238 238 self.name = 'run'
239 239
240 240 self.parmConfObjList = []
241 241
242 242 parmElementList = opElement.getiterator(ParameterConf().getElementName())
243 243
244 244 for parmElement in parmElementList:
245 245 parmConfObj = ParameterConf()
246 246 parmConfObj.readXml(parmElement)
247 247
248 248 #Compatible with old signal chain version
249 249 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
250 250 if self.type != 'self' and self.name == 'Plot':
251 251 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
252 252 self.name = parmConfObj.value
253 253 continue
254 254
255 255 self.parmConfObjList.append(parmConfObj)
256 256
257 257 def printattr(self):
258 258
259 259 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
260 260 self.id,
261 261 self.name,
262 262 self.type,
263 263 self.priority)
264 264
265 265 for parmConfObj in self.parmConfObjList:
266 266 parmConfObj.printattr()
267 267
268 268 def createObject(self):
269 269
270 270 if self.type == 'self':
271 271 raise ValueError, "This operation type cannot be created"
272 272
273 273 if self.type == 'external' or self.type == 'other':
274 274 className = eval(self.name)
275 275 opObj = className()
276 276
277 277 return opObj
278 278
279 279 class ProcUnitConf():
280 280
281 281 id = None
282 282 name = None
283 283 datatype = None
284 284 inputId = None
285 285
286 286 opConfObjList = []
287 287
288 288 procUnitObj = None
289 289 opObjList = []
290 290
291 291 ELEMENTNAME = 'ProcUnit'
292 292
293 293 def __init__(self):
294 294
295 295 self.id = None
296 296 self.datatype = None
297 297 self.name = None
298 298 self.inputId = None
299 299
300 300 self.opConfObjList = []
301 301
302 302 self.procUnitObj = None
303 303 self.opObjDict = {}
304 304
305 305 def __getPriority(self):
306 306
307 307 return len(self.opConfObjList)+1
308 308
309 309 def __getNewId(self):
310 310
311 311 return int(self.id)*10 + len(self.opConfObjList) + 1
312 312
313 313 def getElementName(self):
314 314
315 315 return self.ELEMENTNAME
316 316
317 317 def getId(self):
318 318
319 319 return str(self.id)
320 320
321 321 def getInputId(self):
322 322
323 323 return str(self.inputId)
324 324
325 325 def getOperationObjList(self):
326 326
327 327 return self.opConfObjList
328 328
329 329 def getProcUnitObj(self):
330 330
331 331 return self.procUnitObj
332 332
333 333 def setup(self, id, name, datatype, inputId):
334 334
335 335 self.id = id
336 336 self.name = name
337 337 self.datatype = datatype
338 338 self.inputId = inputId
339 339
340 340 self.opConfObjList = []
341 341
342 342 self.addOperation(name='run', optype='self')
343 343
344 344 def addParameter(self, **kwargs):
345 345
346 346 opObj = self.opConfObjList[0]
347 347
348 348 opObj.addParameter(**kwargs)
349 349
350 350 return opObj
351 351
352 352 def addOperation(self, name, optype='self'):
353 353
354 354 id = self.__getNewId()
355 355 priority = self.__getPriority()
356 356
357 357 opConfObj = OperationConf()
358 358 opConfObj.setup(id, name=name, priority=priority, type=optype)
359 359
360 360 self.opConfObjList.append(opConfObj)
361 361
362 362 return opConfObj
363 363
364 364 def makeXml(self, procUnitElement):
365 365
366 366 upElement = SubElement(procUnitElement, self.ELEMENTNAME)
367 367 upElement.set('id', str(self.id))
368 368 upElement.set('name', self.name)
369 369 upElement.set('datatype', self.datatype)
370 370 upElement.set('inputId', str(self.inputId))
371 371
372 372 for opConfObj in self.opConfObjList:
373 373 opConfObj.makeXml(upElement)
374 374
375 375 def readXml(self, upElement):
376 376
377 377 self.id = upElement.get('id')
378 378 self.name = upElement.get('name')
379 379 self.datatype = upElement.get('datatype')
380 380 self.inputId = upElement.get('inputId')
381
382 #Compatible with old signal chain version
383 if self.ELEMENTNAME == ReadUnitConf().getElementName():
384 if 'Reader' not in self.name:
385 self.name += 'Reader'
386
387 if self.ELEMENTNAME == ProcUnitConf().getElementName():
388 if 'Proc' not in self.name:
389 self.name += 'Proc'
390 381
391 382 self.opConfObjList = []
392 383
393 384 opElementList = upElement.getiterator(OperationConf().getElementName())
394 385
395 386 for opElement in opElementList:
396 387 opConfObj = OperationConf()
397 388 opConfObj.readXml(opElement)
398 389 self.opConfObjList.append(opConfObj)
399 390
400 391 def printattr(self):
401 392
402 393 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
403 394 self.id,
404 395 self.name,
405 396 self.datatype,
406 397 self.inputId)
407 398
408 399 for opConfObj in self.opConfObjList:
409 400 opConfObj.printattr()
410 401
411 402 def createObjects(self):
412 403
413 404 className = eval(self.name)
414 405 procUnitObj = className()
415 406
416 407 for opConfObj in self.opConfObjList:
417 408
418 409 if opConfObj.type == 'self':
419 410 continue
420 411
421 412 opObj = opConfObj.createObject()
422 413
423 414 self.opObjDict[opConfObj.id] = opObj
424 415 procUnitObj.addOperation(opObj, opConfObj.id)
425 416
426 417 self.procUnitObj = procUnitObj
427 418
428 419 return procUnitObj
429 420
430 421 def run(self):
431 422
432 423 finalSts = False
433 424
434 425 for opConfObj in self.opConfObjList:
435 426
436 427 kwargs = {}
437 428 for parmConfObj in opConfObj.getParameterObjList():
438 429 kwargs[parmConfObj.name] = parmConfObj.getValue()
439 430
440 431 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
441 432 sts = self.procUnitObj.call(opType = opConfObj.type,
442 433 opName = opConfObj.name,
443 434 opId = opConfObj.id,
444 435 **kwargs)
445 436 finalSts = finalSts or sts
446 437
447 438 return finalSts
448
439
440 def close(self):
441
442 self.procUnitObj.close()
443
444 return
445
449 446 class ReadUnitConf(ProcUnitConf):
450 447
451 448 path = None
452 449 startDate = None
453 450 endDate = None
454 451 startTime = None
455 452 endTime = None
456 453
457 454 ELEMENTNAME = 'ReadUnit'
458 455
459 456 def __init__(self):
460 457
461 458 self.id = None
462 459 self.datatype = None
463 460 self.name = None
464 461 self.inputId = 0
465 462
466 463 self.opConfObjList = []
467 464 self.opObjList = []
468 465
469 466 def getElementName(self):
470 467
471 468 return self.ELEMENTNAME
472 469
473 470 def setup(self, id, name, datatype, path="", startDate="", endDate="", startTime="", endTime="", **kwargs):
474 471
475 472 self.id = id
476 473 self.name = name
477 474 self.datatype = datatype
478 475
479 476 self.path = path
480 477 self.startDate = startDate
481 478 self.endDate = endDate
482 479 self.startTime = startTime
483 480 self.endTime = endTime
484 481
485 482 self.addRunOperation(**kwargs)
486 483
487 484 def addRunOperation(self, **kwargs):
488 485
489 486 opObj = self.addOperation(name = 'run', optype = 'self')
490 487
491 488 opObj.addParameter(name='path' , value=self.path, format='str')
492 489 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
493 490 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
494 491 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
495 492 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
496 493
497 494 for key, value in kwargs.items():
498 495 opObj.addParameter(name=key, value=value, format=type(value).__name__)
499 496
500 497 return opObj
501 498
502 499
503 500 class Project():
504 501
505 502 id = None
506 503 name = None
507 504 description = None
508 505 # readUnitConfObjList = None
509 506 procUnitConfObjDict = None
510 507
511 508 ELEMENTNAME = 'Project'
512 509
513 510 def __init__(self):
514 511
515 512 self.id = None
516 513 self.name = None
517 514 self.description = None
518 515
519 516 # self.readUnitConfObjList = []
520 517 self.procUnitConfObjDict = {}
521 518
522 519 def __getNewId(self):
523 520
524 521 id = int(self.id)*10 + len(self.procUnitConfObjDict) + 1
525 522
526 523 return str(id)
527 524
528 525 def getElementName(self):
529 526
530 527 return self.ELEMENTNAME
531 528
532 529 def setup(self, id, name, description):
533 530
534 531 self.id = id
535 532 self.name = name
536 533 self.description = description
537 534
538 535 def addReadUnit(self, datatype=None, name=None, **kwargs):
539 536
540 537 #Compatible with old signal chain version
541 538 if datatype==None and name==None:
542 539 raise ValueError, "datatype or name should be defined"
543 540
544 541 if name==None:
545 542 if 'Reader' in datatype:
546 543 name = datatype
547 544 else:
548 545 name = '%sReader' %(datatype)
549 546
550 547 if datatype==None:
551 548 datatype = name.replace('Reader','')
552 549
553 550 id = self.__getNewId()
554 551
555 552 readUnitConfObj = ReadUnitConf()
556 553 readUnitConfObj.setup(id, name, datatype, **kwargs)
557 554
558 555 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
559 556
560 557 return readUnitConfObj
561 558
562 def addProcUnit(self, inputId, datatype=None, name=None):
559 def addProcUnit(self, inputId=0, datatype=None, name=None):
563 560
564 561 #Compatible with old signal chain version
565 562 if datatype==None and name==None:
566 563 raise ValueError, "datatype or name should be defined"
567 564
568 565 if name==None:
569 566 if 'Proc' in datatype:
570 567 name = datatype
571 568 else:
572 569 name = '%sProc' %(datatype)
573 570
574 571 if datatype==None:
575 572 datatype = name.replace('Proc','')
576 573
577 574 id = self.__getNewId()
578 575
579 576 procUnitConfObj = ProcUnitConf()
580 577 procUnitConfObj.setup(id, name, datatype, inputId)
581 578
582 579 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
583 580
584 581 return procUnitConfObj
585 582
586 583 def makeXml(self):
587 584
588 585 projectElement = Element('Project')
589 586 projectElement.set('id', str(self.id))
590 587 projectElement.set('name', self.name)
591 588 projectElement.set('description', self.description)
592 589
593 590 # for readUnitConfObj in self.readUnitConfObjList:
594 591 # readUnitConfObj.makeXml(projectElement)
595 592
596 593 for procUnitConfObj in self.procUnitConfObjDict.values():
597 594 procUnitConfObj.makeXml(projectElement)
598 595
599 596 self.projectElement = projectElement
600 597
601 598 def writeXml(self, filename):
602 599
603 600 self.makeXml()
604 601
605 602 #print prettify(self.projectElement)
606 603
607 604 ElementTree(self.projectElement).write(filename, method='xml')
608 605
609 606 def readXml(self, filename):
610 607
611 608 #tree = ET.parse(filename)
612 609 self.projectElement = None
613 610 # self.readUnitConfObjList = []
614 611 self.procUnitConfObjDict = {}
615 612
616 613 self.projectElement = ElementTree().parse(filename)
617 614
618 615 self.project = self.projectElement.tag
619 616
620 617 self.id = self.projectElement.get('id')
621 618 self.name = self.projectElement.get('name')
622 619 self.description = self.projectElement.get('description')
623 620
624 621 readUnitElementList = self.projectElement.getiterator(ReadUnitConf().getElementName())
625 622
626 623 for readUnitElement in readUnitElementList:
627 624 readUnitConfObj = ReadUnitConf()
628 625 readUnitConfObj.readXml(readUnitElement)
629 626
630 627 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
631 628
632 629 procUnitElementList = self.projectElement.getiterator(ProcUnitConf().getElementName())
633 630
634 631 for procUnitElement in procUnitElementList:
635 632 procUnitConfObj = ProcUnitConf()
636 633 procUnitConfObj.readXml(procUnitElement)
637 634
638 635 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
639 636
640 637 def printattr(self):
641 638
642 639 print "Project[%s]: name = %s, description = %s" %(self.id,
643 640 self.name,
644 641 self.description)
645 642
646 643 # for readUnitConfObj in self.readUnitConfObjList:
647 644 # readUnitConfObj.printattr()
648 645
649 646 for procUnitConfObj in self.procUnitConfObjDict.values():
650 647 procUnitConfObj.printattr()
651 648
652 649 def createObjects(self):
653 650
654 651 # for readUnitConfObj in self.readUnitConfObjList:
655 652 # readUnitConfObj.createObjects()
656 653
657 654 for procUnitConfObj in self.procUnitConfObjDict.values():
658 655 procUnitConfObj.createObjects()
659 656
660 657 def __connect(self, objIN, thisObj):
661 658
662 659 thisObj.setInput(objIN.getOutputObj())
663 660
664 661 def connectObjects(self):
665 662
666 663 for thisPUConfObj in self.procUnitConfObjDict.values():
667 664
668 665 inputId = thisPUConfObj.getInputId()
669 666
670 667 if int(inputId) == 0:
671 668 continue
672 669
673 670 #Get input object
674 671 puConfINObj = self.procUnitConfObjDict[inputId]
675 672 puObjIN = puConfINObj.getProcUnitObj()
676 673
677 674 #Get current object
678 675 thisPUObj = thisPUConfObj.getProcUnitObj()
679 676
680 677 self.__connect(puObjIN, thisPUObj)
681 678
682 679 def run(self):
683 680
684 681 # for readUnitConfObj in self.readUnitConfObjList:
685 682 # readUnitConfObj.run()
686 683 print
687 684 print "*"*40
688 685 print " Starting SIGNAL CHAIN PROCESSING "
689 686 print "*"*40
690 687 print
691 688
692 689 keyList = self.procUnitConfObjDict.keys()
693 690 keyList.sort()
694 691
695 692 while(True):
696 693
697 694 finalSts = False
698 695
699 696 for procKey in keyList:
700 697 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
701 698
702 699 procUnitConfObj = self.procUnitConfObjDict[procKey]
703 700 sts = procUnitConfObj.run()
704 701 finalSts = finalSts or sts
705 702
706 703 #If every process unit finished so end process
707 704 if not(finalSts):
708 705 print "Every process unit have finished"
709 706 break
710
707
708 #Closing every process
709 for procKey in keyList:
710 procUnitConfObj = self.procUnitConfObjDict[procKey]
711 procUnitConfObj.close()
712
713 def start(self, filename):
714
715 self.writeXml(filename)
716 self.readXml(filename)
717
718 self.createObjects()
719 self.connectObjects()
720 self.run()
721
711 722 if __name__ == '__main__':
712 723
713 724 desc = "Segundo Test"
714 725 filename = "schain.xml"
715 726
716 727 controllerObj = Project()
717 728
718 729 controllerObj.setup(id = '191', name='test01', description=desc)
719 730
720 731 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
721 732 path='data/rawdata/',
722 733 startDate='2011/01/01',
723 734 endDate='2012/12/31',
724 735 startTime='00:00:00',
725 736 endTime='23:59:59',
726 737 online=1,
727 738 walk=1)
728 739
729 740 # opObj00 = readUnitConfObj.addOperation(name='printInfo')
730 741
731 742 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
732 743
733 744 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
734 745 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
735 746
736 747 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
737 748 opObj10.addParameter(name='minHei', value='90', format='float')
738 749 opObj10.addParameter(name='maxHei', value='180', format='float')
739 750
740 751 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
741 752 opObj12.addParameter(name='n', value='10', format='int')
742 753
743 754 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
744 755 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
745 756 # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
746 757
747 758
748 759 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
749 760 opObj11.addParameter(name='idfigure', value='1', format='int')
750 761 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
751 762 opObj11.addParameter(name='zmin', value='40', format='int')
752 763 opObj11.addParameter(name='zmax', value='90', format='int')
753 764 opObj11.addParameter(name='showprofile', value='1', format='int')
754 765
755 766 # opObj11 = procUnitConfObj1.addOperation(name='CrossSpectraPlot', optype='external')
756 767 # opObj11.addParameter(name='idfigure', value='2', format='int')
757 768 # opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
758 769 # opObj11.addParameter(name='zmin', value='40', format='int')
759 770 # opObj11.addParameter(name='zmax', value='90', format='int')
760 771
761 772
762 773 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
763 774 #
764 775 # opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='external')
765 776 # opObj12.addParameter(name='n', value='2', format='int')
766 777 # opObj12.addParameter(name='overlapping', value='1', format='int')
767 778 #
768 779 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
769 780 # procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
770 781 #
771 782 # opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='external')
772 783 # opObj11.addParameter(name='idfigure', value='2', format='int')
773 784 # opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
774 785 # opObj11.addParameter(name='zmin', value='40', format='int')
775 786 # opObj11.addParameter(name='zmax', value='90', format='int')
776 787 # opObj11.addParameter(name='showprofile', value='1', format='int')
777 788
778 789 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
779 790 # opObj11.addParameter(name='idfigure', value='10', format='int')
780 791 # opObj11.addParameter(name='wintitle', value='RTI', format='str')
781 792 ## opObj11.addParameter(name='xmin', value='21', format='float')
782 793 ## opObj11.addParameter(name='xmax', value='22', format='float')
783 794 # opObj11.addParameter(name='zmin', value='40', format='int')
784 795 # opObj11.addParameter(name='zmax', value='90', format='int')
785 796 # opObj11.addParameter(name='showprofile', value='1', format='int')
786 797 # opObj11.addParameter(name='timerange', value=str(60), format='int')
787 798
788 799 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
789 800 # opObj10.addParameter(name='channelList', value='0,2,4,6', format='intlist')
790 801 #
791 802 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
792 803 # opObj12.addParameter(name='n', value='2', format='int')
793 804 #
794 805 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
795 806 # opObj11.addParameter(name='idfigure', value='2', format='int')
796 807 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
797 808 # opObj11.addParameter(name='zmin', value='70', format='int')
798 809 # opObj11.addParameter(name='zmax', value='90', format='int')
799 810 #
800 811 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
801 812 # opObj10.addParameter(name='channelList', value='2,6', format='intlist')
802 813 #
803 814 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
804 815 # opObj12.addParameter(name='n', value='2', format='int')
805 816 #
806 817 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
807 818 # opObj11.addParameter(name='idfigure', value='3', format='int')
808 819 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
809 820 # opObj11.addParameter(name='zmin', value='70', format='int')
810 821 # opObj11.addParameter(name='zmax', value='90', format='int')
811 822
812 823
813 824 # opObj12 = procUnitConfObj1.addOperation(name='decoder')
814 825 # opObj12.addParameter(name='ncode', value='2', format='int')
815 826 # opObj12.addParameter(name='nbauds', value='8', format='int')
816 827 # opObj12.addParameter(name='code0', value='001110011', format='int')
817 828 # opObj12.addParameter(name='code1', value='001110011', format='int')
818 829
819 830
820 831
821 832 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj1.getId())
822 833 #
823 834 # opObj21 = procUnitConfObj2.addOperation(name='IncohInt', optype='external')
824 835 # opObj21.addParameter(name='n', value='2', format='int')
825 836 #
826 837 # opObj11 = procUnitConfObj2.addOperation(name='SpectraPlot', optype='external')
827 838 # opObj11.addParameter(name='idfigure', value='4', format='int')
828 839 # opObj11.addParameter(name='wintitle', value='SpectraPlot OBJ 2', format='str')
829 840 # opObj11.addParameter(name='zmin', value='70', format='int')
830 841 # opObj11.addParameter(name='zmax', value='90', format='int')
831 842
832 843 print "Escribiendo el archivo XML"
833 844
834 845 controllerObj.writeXml(filename)
835 846
836 847 print "Leyendo el archivo XML"
837 848 controllerObj.readXml(filename)
838 849 #controllerObj.printattr()
839 850
840 851 controllerObj.createObjects()
841 852 controllerObj.connectObjects()
842 853 controllerObj.run()
843 854
844 855 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,608 +1,650
1 1 import os
2 2 import numpy
3 3 import time, datetime
4 4 import mpldriver
5 5
6 6
7 7 import Queue
8 8 import threading
9 9
10 10 def isRealtime(utcdatatime):
11 11 utcnow = time.mktime(time.localtime())
12 12 delta = abs(utcnow - utcdatatime) # abs
13 13 if delta >= 30.:
14 14 return False
15 15 return True
16 16
17 17
18 18
19 19
20 20 class Figure:
21 21
22 22 __driver = mpldriver
23 23 __isConfigThread = False
24 24 fig = None
25 25
26 26 id = None
27 27 wintitle = None
28 28 width = None
29 29 height = None
30 30 nplots = None
31 31 timerange = None
32 32
33 33 axesObjList = []
34 34
35 35 WIDTH = None
36 36 HEIGHT = None
37 37 PREFIX = 'fig'
38 38
39 39 xmin = None
40 40 xmax = None
41 41
42 counter_imagwr = 0
43
44 figfile = None
45
42 46 def __init__(self):
43 47
44 48 raise ValueError, "This method is not implemented"
45 49
46 50 def __del__(self):
47 51
48 52 self.__driver.closeFigure(True)
49 53
50 54 def getFilename(self, name, ext='.png'):
51 55
52 56 path = '%s%03d' %(self.PREFIX, self.id)
53 57 filename = '%s_%s%s' %(self.PREFIX, name, ext)
54 58 return os.path.join(path, filename)
55 59
56 60 def getAxesObjList(self):
57 61
58 62 return self.axesObjList
59 63
60 64 def getSubplots(self):
61 65
62 66 raise ValueError, "Abstract method: This method should be defined"
63 67
64 68 def getScreenDim(self, widthplot, heightplot):
65 69
66 70 nrow, ncol = self.getSubplots()
67 71
68 72 widthscreen = widthplot*ncol
69 73 heightscreen = heightplot*nrow
70 74
71 75 return widthscreen, heightscreen
72 76
73 77 def getTimeLim(self, x, xmin=None, xmax=None, timerange=None):
74 78
75 79 if self.xmin != None and self.xmax != None:
76 80 if timerange == None:
77 81 timerange = self.xmax - self.xmin
78 82 xmin = self.xmin + timerange
79 83 xmax = self.xmax + timerange
80 84
81 85 return xmin, xmax
82 86
83 87 if timerange == None and (xmin==None or xmax==None):
84 88 raise ValueError, "timerange or xmin+xmax should be defined"
85 89
86 90 if timerange != None:
87 91 txmin = x[0] - x[0] % min(timerange/10, 10*60)
88 92 else:
89 93 txmin = x[0] - x[0] % 10*60
90 94
91 95 thisdatetime = datetime.datetime.utcfromtimestamp(txmin)
92 96 thisdate = datetime.datetime.combine(thisdatetime.date(), datetime.time(0,0,0))
93 97
94 98 if timerange != None:
95 99 xmin = (thisdatetime - thisdate).seconds/(60*60.)
96 100 xmax = xmin + timerange/(60*60.)
97 101
98 102 mindt = thisdate + datetime.timedelta(hours=xmin) - datetime.timedelta(seconds=time.timezone)
99 103 xmin_sec = time.mktime(mindt.timetuple())
100 104
101 105 maxdt = thisdate + datetime.timedelta(hours=xmax) - datetime.timedelta(seconds=time.timezone)
102 106 xmax_sec = time.mktime(maxdt.timetuple())
103 107
104 108 return xmin_sec, xmax_sec
105 109
106 110
107 111
108 112
109 113
110 114 # if timerange != None:
111 115 # txmin = x[0] - x[0]%timerange
112 116 # else:
113 117 # txmin = numpy.min(x)
114 118 #
115 119 # thisdatetime = datetime.datetime.utcfromtimestamp(txmin)
116 120 # thisdate = datetime.datetime.combine(thisdatetime.date(), datetime.time(0,0,0))
117 121 #
118 122 # ####################################################
119 123 # #If the x is out of xrange
120 124 # if xmax != None:
121 125 # if xmax < (thisdatetime - thisdate).seconds/(60*60.):
122 126 # xmin = None
123 127 # xmax = None
124 128 #
125 129 # if xmin == None:
126 130 # td = thisdatetime - thisdate
127 131 # xmin = td.seconds/(60*60.)
128 132 #
129 133 # if xmax == None:
130 134 # xmax = xmin + self.timerange/(60*60.)
131 135 #
132 136 # mindt = thisdate + datetime.timedelta(hours=xmin) - datetime.timedelta(seconds=time.timezone)
133 137 # tmin = time.mktime(mindt.timetuple())
134 138 #
135 139 # maxdt = thisdate + datetime.timedelta(hours=xmax) - datetime.timedelta(seconds=time.timezone)
136 140 # tmax = time.mktime(maxdt.timetuple())
137 141 #
138 142 # #self.timerange = tmax - tmin
139 143 #
140 144 # return tmin, tmax
141 145
142 146 def init(self, id, nplots, wintitle):
143 147
144 148 raise ValueError, "This method has been replaced with createFigure"
145 149
146 150 def createFigure(self, id, wintitle, widthplot=None, heightplot=None, show=True):
147 151
148 152 """
149 153 Crea la figura de acuerdo al driver y parametros seleccionados seleccionados.
150 154 Las dimensiones de la pantalla es calculada a partir de los atributos self.WIDTH
151 155 y self.HEIGHT y el numero de subplots (nrow, ncol)
152 156
153 157 Input:
154 158 id : Los parametros necesarios son
155 159 wintitle :
156 160
157 161 """
158 162
159 163 if widthplot == None:
160 164 widthplot = self.WIDTH
161 165
162 166 if heightplot == None:
163 167 heightplot = self.HEIGHT
164 168
165 169 self.id = id
166 170
167 171 self.wintitle = wintitle
168 172
169 173 self.widthscreen, self.heightscreen = self.getScreenDim(widthplot, heightplot)
170 174
171 175 self.fig = self.__driver.createFigure(id=self.id,
172 176 wintitle=self.wintitle,
173 177 width=self.widthscreen,
174 178 height=self.heightscreen,
175 179 show=show)
176 180
177 181 self.axesObjList = []
182 self.counter_imagwr = 0
178 183
179 184
180 185 def setDriver(self, driver=mpldriver):
181 186
182 187 self.__driver = driver
183 188
184 189 def setTitle(self, title):
185 190
186 191 self.__driver.setTitle(self.fig, title)
187 192
188 193 def setWinTitle(self, title):
189 194
190 195 self.__driver.setWinTitle(self.fig, title=title)
191 196
192 197 def setTextFromAxes(self, text):
193 198
194 199 raise ValueError, "Este metodo ha sido reemplazaado con el metodo setText de la clase Axes"
195 200
196 201 def makeAxes(self, nrow, ncol, xpos, ypos, colspan, rowspan):
197 202
198 203 raise ValueError, "Este metodo ha sido reemplazaado con el metodo addAxes"
199 204
200 205 def addAxes(self, *args):
201 206 """
202 207
203 208 Input:
204 209 *args : Los parametros necesarios son
205 210 nrow, ncol, xpos, ypos, colspan, rowspan
206 211 """
207 212
208 213 axesObj = Axes(self.fig, *args)
209 214 self.axesObjList.append(axesObj)
210 215
211 216 def saveFigure(self, figpath, figfile, *args):
212 217
213 218 filename = os.path.join(figpath, figfile)
214 219
215 220 fullpath = os.path.split(filename)[0]
216 221
217 222 if not os.path.exists(fullpath):
218 223 subpath = os.path.split(fullpath)[0]
219 224
220 225 if not os.path.exists(subpath):
221 226 os.mkdir(subpath)
222 227
223 228 os.mkdir(fullpath)
224 229
225 230 self.__driver.saveFigure(self.fig, filename, *args)
226 231
227
228
232 def save(self, figpath, figfile=None, save=True, ftp=False, wr_period=1, thisDatetime=None, update_figfile=True):
233
234 if not save:
235 return
236
237 if figfile == None:
238
239 if not thisDatetime:
240 raise ValueError, "Saving figure: figfile or thisDatetime should be defined"
241 return
242
243 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
244 figfile = self.getFilename(name = str_datetime)
245
246 if self.figfile == None:
247 self.figfile = figfile
248
249 if update_figfile:
250 self.figfile = figfile
251
252 self.counter_imagwr += 1
253
254 if self.counter_imagwr<wr_period:
255 return
256
257 # store png plot to local folder
258 self.saveFigure(figpath, self.figfile)
259 self.counter_imagwr = 0
260
261 if not ftp:
262 return
263
264 if not thisDatetime:
265 return
266
267 # store png plot to FTP server according to RT-Web format
268 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
269 ftp_filename = os.path.join(figpath, name)
270 self.saveFigure(figpath, ftp_filename)
229 271
230 272 def getNameToFtp(self, thisDatetime, FTP_WEI, EXP_CODE, SUB_EXP_CODE, PLOT_CODE, PLOT_POS):
231 273 YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
232 274 DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
233 275 FTP_WEI = '%2.2d'%FTP_WEI
234 276 EXP_CODE = '%3.3d'%EXP_CODE
235 277 SUB_EXP_CODE = '%2.2d'%SUB_EXP_CODE
236 278 PLOT_CODE = '%2.2d'%PLOT_CODE
237 279 PLOT_POS = '%2.2d'%PLOT_POS
238 280 name = YEAR_STR + DOY_STR + FTP_WEI + EXP_CODE + SUB_EXP_CODE + PLOT_CODE + PLOT_POS
239 281 return name
240 282
241 283 def draw(self):
242 284
243 285 self.__driver.draw(self.fig)
244 286
245 287 def run(self):
246 288
247 289 raise ValueError, "This method is not implemented"
248 290
249 291 def close(self):
250 292
251 293 self.__driver.show(True)
252 294
253 295 axesList = property(getAxesObjList)
254 296
255 297
256 298 class Axes:
257 299
258 300 __driver = mpldriver
259 301 fig = None
260 302 ax = None
261 303 plot = None
262 304 __missing = 1E30
263 305 __firsttime = None
264 306
265 307 __showprofile = False
266 308
267 309 xmin = None
268 310 xmax = None
269 311 ymin = None
270 312 ymax = None
271 313 zmin = None
272 314 zmax = None
273 315
274 316 x_buffer = None
275 317 z_buffer = None
276 318
277 319 decimationx = None
278 320 decimationy = None
279 321
280 322 __MAXNUMX = 300
281 323 __MAXNUMY = 150
282 324
283 325 def __init__(self, *args):
284 326
285 327 """
286 328
287 329 Input:
288 330 *args : Los parametros necesarios son
289 331 fig, nrow, ncol, xpos, ypos, colspan, rowspan
290 332 """
291 333
292 334 ax = self.__driver.createAxes(*args)
293 335 self.fig = args[0]
294 336 self.ax = ax
295 337 self.plot = None
296 338
297 339 self.__firsttime = True
298 340 self.idlineList = []
299 341
300 342 self.x_buffer = numpy.array([])
301 343 self.z_buffer = numpy.array([])
302 344
303 345 def setText(self, text):
304 346
305 347 self.__driver.setAxesText(self.ax, text)
306 348
307 349 def setXAxisAsTime(self):
308 350 pass
309 351
310 352 def pline(self, x, y,
311 353 xmin=None, xmax=None,
312 354 ymin=None, ymax=None,
313 355 xlabel='', ylabel='',
314 356 title='',
315 357 **kwargs):
316 358
317 359 """
318 360
319 361 Input:
320 362 x :
321 363 y :
322 364 xmin :
323 365 xmax :
324 366 ymin :
325 367 ymax :
326 368 xlabel :
327 369 ylabel :
328 370 title :
329 371 **kwargs : Los parametros aceptados son
330 372
331 373 ticksize
332 374 ytick_visible
333 375 """
334 376
335 377 if self.__firsttime:
336 378
337 379 if xmin == None: xmin = numpy.nanmin(x)
338 380 if xmax == None: xmax = numpy.nanmax(x)
339 381 if ymin == None: ymin = numpy.nanmin(y)
340 382 if ymax == None: ymax = numpy.nanmax(y)
341 383
342 384 self.plot = self.__driver.createPline(self.ax, x, y,
343 385 xmin, xmax,
344 386 ymin, ymax,
345 387 xlabel=xlabel,
346 388 ylabel=ylabel,
347 389 title=title,
348 390 **kwargs)
349 391
350 392 self.idlineList.append(0)
351 393 self.__firsttime = False
352 394 return
353 395
354 396 self.__driver.pline(self.plot, x, y, xlabel=xlabel,
355 397 ylabel=ylabel,
356 398 title=title)
357 399
358 400 def addpline(self, x, y, idline, **kwargs):
359 401 lines = self.ax.lines
360 402
361 403 if idline in self.idlineList:
362 404 self.__driver.set_linedata(self.ax, x, y, idline)
363 405
364 406 if idline not in(self.idlineList):
365 407 self.__driver.addpline(self.ax, x, y, **kwargs)
366 408 self.idlineList.append(idline)
367 409
368 410 return
369 411
370 412 def pmultiline(self, x, y,
371 413 xmin=None, xmax=None,
372 414 ymin=None, ymax=None,
373 415 xlabel='', ylabel='',
374 416 title='',
375 417 **kwargs):
376 418
377 419 if self.__firsttime:
378 420
379 421 if xmin == None: xmin = numpy.nanmin(x)
380 422 if xmax == None: xmax = numpy.nanmax(x)
381 423 if ymin == None: ymin = numpy.nanmin(y)
382 424 if ymax == None: ymax = numpy.nanmax(y)
383 425
384 426 self.plot = self.__driver.createPmultiline(self.ax, x, y,
385 427 xmin, xmax,
386 428 ymin, ymax,
387 429 xlabel=xlabel,
388 430 ylabel=ylabel,
389 431 title=title,
390 432 **kwargs)
391 433 self.__firsttime = False
392 434 return
393 435
394 436 self.__driver.pmultiline(self.plot, x, y, xlabel=xlabel,
395 437 ylabel=ylabel,
396 438 title=title)
397 439
398 440 def pmultilineyaxis(self, x, y,
399 441 xmin=None, xmax=None,
400 442 ymin=None, ymax=None,
401 443 xlabel='', ylabel='',
402 444 title='',
403 445 **kwargs):
404 446
405 447 if self.__firsttime:
406 448
407 449 if xmin == None: xmin = numpy.nanmin(x)
408 450 if xmax == None: xmax = numpy.nanmax(x)
409 451 if ymin == None: ymin = numpy.nanmin(y)
410 452 if ymax == None: ymax = numpy.nanmax(y)
411 453
412 454 self.plot = self.__driver.createPmultilineYAxis(self.ax, x, y,
413 455 xmin, xmax,
414 456 ymin, ymax,
415 457 xlabel=xlabel,
416 458 ylabel=ylabel,
417 459 title=title,
418 460 **kwargs)
419 461 if self.xmin == None: self.xmin = xmin
420 462 if self.xmax == None: self.xmax = xmax
421 463 if self.ymin == None: self.ymin = ymin
422 464 if self.ymax == None: self.ymax = ymax
423 465
424 466 self.__firsttime = False
425 467 return
426 468
427 469 self.__driver.pmultilineyaxis(self.plot, x, y, xlabel=xlabel,
428 470 ylabel=ylabel,
429 471 title=title)
430 472
431 473 def pcolor(self, x, y, z,
432 474 xmin=None, xmax=None,
433 475 ymin=None, ymax=None,
434 476 zmin=None, zmax=None,
435 477 xlabel='', ylabel='',
436 478 title='', rti = False, colormap='jet',
437 479 **kwargs):
438 480
439 481 """
440 482 Input:
441 483 x :
442 484 y :
443 485 x :
444 486 xmin :
445 487 xmax :
446 488 ymin :
447 489 ymax :
448 490 zmin :
449 491 zmax :
450 492 xlabel :
451 493 ylabel :
452 494 title :
453 495 **kwargs : Los parametros aceptados son
454 496 ticksize=9,
455 497 cblabel=''
456 498 rti = True or False
457 499 """
458 500
459 501 if self.__firsttime:
460 502
461 503 if xmin == None: xmin = numpy.nanmin(x)
462 504 if xmax == None: xmax = numpy.nanmax(x)
463 505 if ymin == None: ymin = numpy.nanmin(y)
464 506 if ymax == None: ymax = numpy.nanmax(y)
465 507 if zmin == None: zmin = numpy.nanmin(z)
466 508 if zmax == None: zmax = numpy.nanmax(z)
467 509
468 510
469 511 self.plot = self.__driver.createPcolor(self.ax, x, y, z,
470 512 xmin, xmax,
471 513 ymin, ymax,
472 514 zmin, zmax,
473 515 xlabel=xlabel,
474 516 ylabel=ylabel,
475 517 title=title,
476 518 colormap=colormap,
477 519 **kwargs)
478 520
479 521 if self.xmin == None: self.xmin = xmin
480 522 if self.xmax == None: self.xmax = xmax
481 523 if self.ymin == None: self.ymin = ymin
482 524 if self.ymax == None: self.ymax = ymax
483 525 if self.zmin == None: self.zmin = zmin
484 526 if self.zmax == None: self.zmax = zmax
485 527
486 528 self.__firsttime = False
487 529 return
488 530
489 531 if rti:
490 532 self.__driver.addpcolor(self.ax, x, y, z, self.zmin, self.zmax,
491 533 xlabel=xlabel,
492 534 ylabel=ylabel,
493 535 title=title,
494 536 colormap=colormap)
495 537 return
496 538
497 539 self.__driver.pcolor(self.plot, z,
498 540 xlabel=xlabel,
499 541 ylabel=ylabel,
500 542 title=title)
501 543
502 544 def pcolorbuffer(self, x, y, z,
503 545 xmin=None, xmax=None,
504 546 ymin=None, ymax=None,
505 547 zmin=None, zmax=None,
506 548 xlabel='', ylabel='',
507 549 title='', rti = True, colormap='jet',
508 550 maxNumX = None, maxNumY = None,
509 551 **kwargs):
510 552
511 553 if maxNumX == None:
512 554 maxNumX = self.__MAXNUMX
513 555
514 556 if maxNumY == None:
515 557 maxNumY = self.__MAXNUMY
516 558
517 559 if self.__firsttime:
518 560 self.z_buffer = z
519 561 self.x_buffer = numpy.hstack((self.x_buffer, x))
520 562
521 563 if xmin == None: xmin = numpy.nanmin(x)
522 564 if xmax == None: xmax = numpy.nanmax(x)
523 565 if ymin == None: ymin = numpy.nanmin(y)
524 566 if ymax == None: ymax = numpy.nanmax(y)
525 567 if zmin == None: zmin = numpy.nanmin(z)
526 568 if zmax == None: zmax = numpy.nanmax(z)
527 569
528 570
529 571 self.plot = self.__driver.createPcolor(self.ax, self.x_buffer, y, z,
530 572 xmin, xmax,
531 573 ymin, ymax,
532 574 zmin, zmax,
533 575 xlabel=xlabel,
534 576 ylabel=ylabel,
535 577 title=title,
536 578 colormap=colormap,
537 579 **kwargs)
538 580
539 581 if self.xmin == None: self.xmin = xmin
540 582 if self.xmax == None: self.xmax = xmax
541 583 if self.ymin == None: self.ymin = ymin
542 584 if self.ymax == None: self.ymax = ymax
543 585 if self.zmin == None: self.zmin = zmin
544 586 if self.zmax == None: self.zmax = zmax
545 587
546 588 self.__firsttime = False
547 589 return
548 590
549 591 self.x_buffer = numpy.hstack((self.x_buffer, x[-1]))
550 592 self.z_buffer = numpy.hstack((self.z_buffer, z))
551 593
552 594 if self.decimationx == None:
553 595 deltax = float(self.xmax - self.xmin)/maxNumX
554 596 deltay = float(self.ymax - self.ymin)/maxNumY
555 597
556 598 resolutionx = self.x_buffer[2]-self.x_buffer[0]
557 599 resolutiony = y[1]-y[0]
558 600
559 601 self.decimationx = numpy.ceil(deltax / resolutionx)
560 602 self.decimationy = numpy.ceil(deltay / resolutiony)
561 603
562 604 z_buffer = self.z_buffer.reshape(-1,len(y))
563 605
564 606 x_buffer = self.x_buffer[::self.decimationx]
565 607 y_buffer = y[::self.decimationy]
566 608 z_buffer = z_buffer[::self.decimationx, ::self.decimationy]
567 609 #===================================================
568 610
569 611 x_buffer, y_buffer, z_buffer = self.__fillGaps(x_buffer, y_buffer, z_buffer)
570 612
571 613 self.__driver.addpcolorbuffer(self.ax, x_buffer, y_buffer, z_buffer, self.zmin, self.zmax,
572 614 xlabel=xlabel,
573 615 ylabel=ylabel,
574 616 title=title,
575 617 colormap=colormap)
576 618
577 619 def polar(self, x, y,
578 620 title='', xlabel='',ylabel='',**kwargs):
579 621
580 622 if self.__firsttime:
581 623 self.plot = self.__driver.createPolar(self.ax, x, y, title = title, xlabel = xlabel, ylabel = ylabel)
582 624 self.__firsttime = False
583 625 self.x_buffer = x
584 626 self.y_buffer = y
585 627 return
586 628
587 629 self.x_buffer = numpy.hstack((self.x_buffer,x))
588 630 self.y_buffer = numpy.hstack((self.y_buffer,y))
589 631 self.__driver.polar(self.plot, self.x_buffer, self.y_buffer, xlabel=xlabel,
590 632 ylabel=ylabel,
591 633 title=title)
592 634
593 635 def __fillGaps(self, x_buffer, y_buffer, z_buffer):
594 636
595 637 deltas = x_buffer[1:] - x_buffer[0:-1]
596 638 x_median = numpy.median(deltas)
597 639
598 640 index = numpy.where(deltas >= 2*x_median)
599 641
600 642 if len(index[0]) != 0:
601 643 z_buffer[index[0],::] = self.__missing
602 644 z_buffer = numpy.ma.masked_inside(z_buffer,0.99*self.__missing,1.01*self.__missing)
603 645
604 646 return x_buffer, y_buffer, z_buffer
605 647
606 648
607 649
608 650 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,198 +1,188
1 1 import os
2 2 import datetime
3 3 import numpy
4 4 import copy
5 5
6 6 from figure import Figure, isRealtime
7 7
8 8 class CorrelationPlot(Figure):
9 9
10 10 isConfig = None
11 11 __nsubplots = None
12 12
13 13 WIDTHPROF = None
14 14 HEIGHTPROF = None
15 15 PREFIX = 'corr'
16 16
17 17 def __init__(self):
18 18
19 19 self.isConfig = False
20 20 self.__nsubplots = 1
21 21
22 22 self.WIDTH = 280
23 23 self.HEIGHT = 250
24 24 self.WIDTHPROF = 120
25 25 self.HEIGHTPROF = 0
26 26 self.counter_imagwr = 0
27 27
28 28 self.PLOT_CODE = 1
29 29 self.FTP_WEI = None
30 30 self.EXP_CODE = None
31 31 self.SUB_EXP_CODE = None
32 32 self.PLOT_POS = None
33 33
34 34 def getSubplots(self):
35 35
36 36 ncol = int(numpy.sqrt(self.nplots)+0.9)
37 37 nrow = int(self.nplots*1./ncol + 0.9)
38 38
39 39 return nrow, ncol
40 40
41 41 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
42 42
43 43 showprofile = False
44 44 self.__showprofile = showprofile
45 45 self.nplots = nplots
46 46
47 47 ncolspan = 1
48 48 colspan = 1
49 49 if showprofile:
50 50 ncolspan = 3
51 51 colspan = 2
52 52 self.__nsubplots = 2
53 53
54 54 self.createFigure(id = id,
55 55 wintitle = wintitle,
56 56 widthplot = self.WIDTH + self.WIDTHPROF,
57 57 heightplot = self.HEIGHT + self.HEIGHTPROF,
58 58 show=show)
59 59
60 60 nrow, ncol = self.getSubplots()
61 61
62 62 counter = 0
63 63 for y in range(nrow):
64 64 for x in range(ncol):
65 65
66 66 if counter >= self.nplots:
67 67 break
68 68
69 69 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
70 70
71 71 if showprofile:
72 72 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
73 73
74 74 counter += 1
75 75
76 76 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
77 77 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
78 78 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
79 79 server=None, folder=None, username=None, password=None,
80 80 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
81 81
82 82 """
83 83
84 84 Input:
85 85 dataOut :
86 86 id :
87 87 wintitle :
88 88 channelList :
89 89 showProfile :
90 90 xmin : None,
91 91 xmax : None,
92 92 ymin : None,
93 93 ymax : None,
94 94 zmin : None,
95 95 zmax : None
96 96 """
97 97
98 98 if dataOut.flagNoData:
99 99 return None
100 100
101 101 if realtime:
102 102 if not(isRealtime(utcdatatime = dataOut.utctime)):
103 103 print 'Skipping this plot function'
104 104 return
105 105
106 106 if channelList == None:
107 107 channelIndexList = dataOut.channelIndexList
108 108 else:
109 109 channelIndexList = []
110 110 for channel in channelList:
111 111 if channel not in dataOut.channelList:
112 112 raise ValueError, "Channel %d is not in dataOut.channelList"
113 113 channelIndexList.append(dataOut.channelList.index(channel))
114 114
115 115 factor = dataOut.normFactor
116 116 lenfactor = factor.shape[1]
117 117 x = dataOut.getLagTRange(1)
118 118 y = dataOut.getHeiRange()
119 119
120 120 z = copy.copy(dataOut.data_corr[:,:,0,:])
121 121 for i in range(dataOut.data_corr.shape[0]):
122 122 z[i,:,:] = z[i,:,:]/factor[i,:]
123 123 zdB = numpy.abs(z)
124 124
125 125 avg = numpy.average(z, axis=1)
126 126 # avg = numpy.nanmean(z, axis=1)
127 127 # noise = dataOut.noise/factor
128 128
129 129 #thisDatetime = dataOut.datatime
130 130 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
131 131 title = wintitle + " Correlation"
132 132 xlabel = "Lag T (s)"
133 133 ylabel = "Range (Km)"
134 134
135 135 if not self.isConfig:
136 136
137 137 nplots = dataOut.data_corr.shape[0]
138 138
139 139 self.setup(id=id,
140 140 nplots=nplots,
141 141 wintitle=wintitle,
142 142 showprofile=showprofile,
143 143 show=show)
144 144
145 145 if xmin == None: xmin = numpy.nanmin(x)
146 146 if xmax == None: xmax = numpy.nanmax(x)
147 147 if ymin == None: ymin = numpy.nanmin(y)
148 148 if ymax == None: ymax = numpy.nanmax(y)
149 149 if zmin == None: zmin = 0
150 150 if zmax == None: zmax = 1
151 151
152 152 self.FTP_WEI = ftp_wei
153 153 self.EXP_CODE = exp_code
154 154 self.SUB_EXP_CODE = sub_exp_code
155 155 self.PLOT_POS = plot_pos
156 156
157 157 self.isConfig = True
158 158
159 159 self.setWinTitle(title)
160 160
161 161 for i in range(self.nplots):
162 162 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
163 163 title = "Channel %d and %d: : %s" %(dataOut.pairsList[i][0]+1,dataOut.pairsList[i][1]+1 , str_datetime)
164 164 axes = self.axesList[i*self.__nsubplots]
165 165 axes.pcolor(x, y, zdB[i,:,:],
166 166 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
167 167 xlabel=xlabel, ylabel=ylabel, title=title,
168 168 ticksize=9, cblabel='')
169 169
170 170 # if self.__showprofile:
171 171 # axes = self.axesList[i*self.__nsubplots +1]
172 172 # axes.pline(avgdB[i], y,
173 173 # xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
174 174 # xlabel='dB', ylabel='', title='',
175 175 # ytick_visible=False,
176 176 # grid='x')
177 177 #
178 178 # noiseline = numpy.repeat(noisedB[i], len(y))
179 179 # axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
180 180
181 181 self.draw()
182 182
183 if save:
184
185 if figfile == None:
186 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
187 figfile = self.getFilename(name = str_datetime)
188
189 self.counter_imagwr += 1
190 if (self.counter_imagwr>=wr_period):
191 # store png plot to local folder
192 self.saveFigure(figpath, figfile)
193 # store png plot to FTP server according to RT-Web format
194 if ftp:
195 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
196 ftp_filename = os.path.join(figpath, name)
197 self.saveFigure(figpath, ftp_filename)
198 self.counter_imagwr = 0
183 self.save(figpath=figpath,
184 figfile=figfile,
185 save=save,
186 ftp=ftp,
187 wr_period=wr_period,
188 thisDatetime=thisDatetime)
Show file before | Show file after | Hide comments
@@ -1,328 +1,312
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from figure import Figure, isRealtime
11 11
12 12 class SpectraHeisScope(Figure):
13 13
14 14
15 15 isConfig = None
16 16 __nsubplots = None
17 17
18 18 WIDTHPROF = None
19 19 HEIGHTPROF = None
20 20 PREFIX = 'spc'
21 21
22 22 def __init__(self):
23 23
24 24 self.isConfig = False
25 25 self.__nsubplots = 1
26 26
27 27 self.WIDTH = 230
28 28 self.HEIGHT = 250
29 29 self.WIDTHPROF = 120
30 30 self.HEIGHTPROF = 0
31 31 self.counter_imagwr = 0
32 32
33 33 def getSubplots(self):
34 34
35 35 ncol = int(numpy.sqrt(self.nplots)+0.9)
36 36 nrow = int(self.nplots*1./ncol + 0.9)
37 37
38 38 return nrow, ncol
39 39
40 40 def setup(self, id, nplots, wintitle, show):
41 41
42 42 showprofile = False
43 43 self.__showprofile = showprofile
44 44 self.nplots = nplots
45 45
46 46 ncolspan = 1
47 47 colspan = 1
48 48 if showprofile:
49 49 ncolspan = 3
50 50 colspan = 2
51 51 self.__nsubplots = 2
52 52
53 53 self.createFigure(id = id,
54 54 wintitle = wintitle,
55 55 widthplot = self.WIDTH + self.WIDTHPROF,
56 56 heightplot = self.HEIGHT + self.HEIGHTPROF,
57 57 show = show)
58 58
59 59 nrow, ncol = self.getSubplots()
60 60
61 61 counter = 0
62 62 for y in range(nrow):
63 63 for x in range(ncol):
64 64
65 65 if counter >= self.nplots:
66 66 break
67 67
68 68 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
69 69
70 70 if showprofile:
71 71 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
72 72
73 73 counter += 1
74 74
75 75
76 76 def run(self, dataOut, id, wintitle="", channelList=None,
77 77 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
78 78 figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
79 79 server=None, folder=None, username=None, password=None):
80 80
81 81 """
82 82
83 83 Input:
84 84 dataOut :
85 85 id :
86 86 wintitle :
87 87 channelList :
88 88 xmin : None,
89 89 xmax : None,
90 90 ymin : None,
91 91 ymax : None,
92 92 """
93 93
94 94 if dataOut.realtime:
95 95 if not(isRealtime(utcdatatime = dataOut.utctime)):
96 96 print 'Skipping this plot function'
97 97 return
98 98
99 99 if channelList == None:
100 100 channelIndexList = dataOut.channelIndexList
101 101 else:
102 102 channelIndexList = []
103 103 for channel in channelList:
104 104 if channel not in dataOut.channelList:
105 105 raise ValueError, "Channel %d is not in dataOut.channelList"
106 106 channelIndexList.append(dataOut.channelList.index(channel))
107 107
108 108 # x = dataOut.heightList
109 109 c = 3E8
110 110 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
111 111 #deberia cambiar para el caso de 1Mhz y 100KHz
112 112 x = numpy.arange(-1*dataOut.nHeights/2.,dataOut.nHeights/2.)*(c/(2*deltaHeight*dataOut.nHeights*1000))
113 113 #para 1Mhz descomentar la siguiente linea
114 114 #x= x/(10000.0)
115 115 # y = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
116 116 # y = y.real
117 117 factor = dataOut.normFactor
118 118 data = dataOut.data_spc / factor
119 119 datadB = 10.*numpy.log10(data)
120 120 y = datadB
121 121
122 122 #thisDatetime = dataOut.datatime
123 123 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
124 124 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
125 125 xlabel = ""
126 126 #para 1Mhz descomentar la siguiente linea
127 127 #xlabel = "Frequency x 10000"
128 128 ylabel = "Intensity (dB)"
129 129
130 130 if not self.isConfig:
131 131 nplots = len(channelIndexList)
132 132
133 133 self.setup(id=id,
134 134 nplots=nplots,
135 135 wintitle=wintitle,
136 136 show=show)
137 137
138 138 if xmin == None: xmin = numpy.nanmin(x)
139 139 if xmax == None: xmax = numpy.nanmax(x)
140 140 if ymin == None: ymin = numpy.nanmin(y)
141 141 if ymax == None: ymax = numpy.nanmax(y)
142 142
143 143 self.isConfig = True
144 144
145 145 self.setWinTitle(title)
146 146
147 147 for i in range(len(self.axesList)):
148 148 ychannel = y[i,:]
149 149 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
150 150 title = "Channel %d: %4.2fdB: %s" %(i, numpy.max(ychannel), str_datetime)
151 151 axes = self.axesList[i]
152 152 axes.pline(x, ychannel,
153 153 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
154 154 xlabel=xlabel, ylabel=ylabel, title=title, grid='both')
155 155
156 156
157 157 self.draw()
158 158
159 if save:
160
161 if figfile == None:
162 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
163 figfile = self.getFilename(name = str_datetime)
164
165 self.counter_imagwr += 1
166 if (self.counter_imagwr>=wr_period):
167 # store png plot to local folder
168 self.saveFigure(figpath, figfile)
169 # store png plot to FTP server according to RT-Web format
170 #name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
171 #ftp_filename = os.path.join(figpath, name)
172 #self.saveFigure(figpath, ftp_filename)
173 self.counter_imagwr = 0
159 self.save(figpath=figpath,
160 figfile=figfile,
161 save=save,
162 ftp=ftp,
163 wr_period=wr_period,
164 thisDatetime=thisDatetime)
174 165
175 166 class RTIfromSpectraHeis(Figure):
176 167
177 168 isConfig = None
178 169 __nsubplots = None
179 170
180 171 PREFIX = 'rtinoise'
181 172
182 173 def __init__(self):
183 174
184 175 self.timerange = 24*60*60
185 176 self.isConfig = False
186 177 self.__nsubplots = 1
187 178
188 179 self.WIDTH = 820
189 180 self.HEIGHT = 200
190 181 self.WIDTHPROF = 120
191 182 self.HEIGHTPROF = 0
192 183 self.counter_imagwr = 0
193 184 self.xdata = None
194 185 self.ydata = None
195 186 self.figfile = None
196 187
197 188 def getSubplots(self):
198 189
199 190 ncol = 1
200 191 nrow = 1
201 192
202 193 return nrow, ncol
203 194
204 195 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
205 196
206 197 self.__showprofile = showprofile
207 198 self.nplots = nplots
208 199
209 200 ncolspan = 7
210 201 colspan = 6
211 202 self.__nsubplots = 2
212 203
213 204 self.createFigure(id = id,
214 205 wintitle = wintitle,
215 206 widthplot = self.WIDTH+self.WIDTHPROF,
216 207 heightplot = self.HEIGHT+self.HEIGHTPROF,
217 208 show = show)
218 209
219 210 nrow, ncol = self.getSubplots()
220 211
221 212 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
222 213
223 214
224 215 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
225 216 xmin=None, xmax=None, ymin=None, ymax=None,
226 217 timerange=None,
227 218 save=False, figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
228 219 server=None, folder=None, username=None, password=None):
229 220
230 221 if channelList == None:
231 222 channelIndexList = dataOut.channelIndexList
232 223 channelList = dataOut.channelList
233 224 else:
234 225 channelIndexList = []
235 226 for channel in channelList:
236 227 if channel not in dataOut.channelList:
237 228 raise ValueError, "Channel %d is not in dataOut.channelList"
238 229 channelIndexList.append(dataOut.channelList.index(channel))
239 230
240 231 if timerange != None:
241 232 self.timerange = timerange
242 233
243 234 tmin = None
244 235 tmax = None
245 236 x = dataOut.getTimeRange()
246 237 y = dataOut.getHeiRange()
247 238
248 239 factor = dataOut.normFactor
249 240 data = dataOut.data_spc / factor
250 241 data = numpy.average(data,axis=1)
251 242 datadB = 10*numpy.log10(data)
252 243
253 244 # factor = dataOut.normFactor
254 245 # noise = dataOut.getNoise()/factor
255 246 # noisedB = 10*numpy.log10(noise)
256 247
257 248 #thisDatetime = dataOut.datatime
258 249 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
259 250 title = wintitle + " RTI: %s" %(thisDatetime.strftime("%d-%b-%Y"))
260 251 xlabel = "Local Time"
261 252 ylabel = "Intensity (dB)"
262 253
263 254 if not self.isConfig:
264 255
265 256 nplots = 1
266 257
267 258 self.setup(id=id,
268 259 nplots=nplots,
269 260 wintitle=wintitle,
270 261 showprofile=showprofile,
271 262 show=show)
272 263
273 264 tmin, tmax = self.getTimeLim(x, xmin, xmax)
274 265 if ymin == None: ymin = numpy.nanmin(datadB)
275 266 if ymax == None: ymax = numpy.nanmax(datadB)
276 267
277 268 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
278 269 self.isConfig = True
279 270 self.figfile = figfile
280 271 self.xdata = numpy.array([])
281 272 self.ydata = numpy.array([])
282 273
283 274 self.setWinTitle(title)
284 275
285 276
286 277 # title = "RTI %s" %(thisDatetime.strftime("%d-%b-%Y"))
287 278 title = "RTI - %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
288 279
289 280 legendlabels = ["channel %d"%idchannel for idchannel in channelList]
290 281 axes = self.axesList[0]
291 282
292 283 self.xdata = numpy.hstack((self.xdata, x[0:1]))
293 284
294 285 if len(self.ydata)==0:
295 286 self.ydata = datadB[channelIndexList].reshape(-1,1)
296 287 else:
297 288 self.ydata = numpy.hstack((self.ydata, datadB[channelIndexList].reshape(-1,1)))
298 289
299 290
300 291 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
301 292 xmin=tmin, xmax=tmax, ymin=ymin, ymax=ymax,
302 293 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='.', markersize=8, linestyle="solid", grid='both',
303 294 XAxisAsTime=True
304 295 )
305 296
306 297 self.draw()
307 298
308 299 if x[1] >= self.axesList[0].xmax:
309 300 self.counter_imagwr = wr_period
310 301 del self.xdata
311 302 del self.ydata
312 303 self.__isConfig = False
313
314 if save:
304 self.figfile = None
315 305
316 if self.figfile == None:
317 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
318 self.figfile = self.getFilename(name = str_datetime)
319
320 self.counter_imagwr += 1
321 if (self.counter_imagwr>=wr_period):
322 # store png plot to local folder
323 self.saveFigure(figpath, self.figfile)
324 # store png plot to FTP server according to RT-Web format
325 #name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
326 #ftp_filename = os.path.join(figpath, name)
327 #self.saveFigure(figpath, ftp_filename)
328 self.counter_imagwr = 0
306 self.save(figpath=figpath,
307 figfile=figfile,
308 save=save,
309 ftp=ftp,
310 wr_period=wr_period,
311 thisDatetime=thisDatetime,
312 update_figfile=False)
Show file before | Show file after | Hide comments
@@ -1,1208 +1,1148
1 1 import os
2 2 import datetime
3 3 import numpy
4 4
5 5 from figure import Figure, isRealtime
6 from plotting_codes import *
6 7
7 8 class MomentsPlot(Figure):
8 9
9 10 isConfig = None
10 11 __nsubplots = None
11 12
12 13 WIDTHPROF = None
13 14 HEIGHTPROF = None
14 15 PREFIX = 'prm'
15 16
16 17 def __init__(self):
17 18
18 19 self.isConfig = False
19 20 self.__nsubplots = 1
20 21
21 22 self.WIDTH = 280
22 23 self.HEIGHT = 250
23 24 self.WIDTHPROF = 120
24 25 self.HEIGHTPROF = 0
25 26 self.counter_imagwr = 0
26 27
27 self.PLOT_CODE = 1
28 self.PLOT_CODE = MOMENTS_CODE
29
28 30 self.FTP_WEI = None
29 31 self.EXP_CODE = None
30 32 self.SUB_EXP_CODE = None
31 33 self.PLOT_POS = None
32 34
33 35 def getSubplots(self):
34 36
35 37 ncol = int(numpy.sqrt(self.nplots)+0.9)
36 38 nrow = int(self.nplots*1./ncol + 0.9)
37 39
38 40 return nrow, ncol
39 41
40 42 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
41 43
42 44 self.__showprofile = showprofile
43 45 self.nplots = nplots
44 46
45 47 ncolspan = 1
46 48 colspan = 1
47 49 if showprofile:
48 50 ncolspan = 3
49 51 colspan = 2
50 52 self.__nsubplots = 2
51 53
52 54 self.createFigure(id = id,
53 55 wintitle = wintitle,
54 56 widthplot = self.WIDTH + self.WIDTHPROF,
55 57 heightplot = self.HEIGHT + self.HEIGHTPROF,
56 58 show=show)
57 59
58 60 nrow, ncol = self.getSubplots()
59 61
60 62 counter = 0
61 63 for y in range(nrow):
62 64 for x in range(ncol):
63 65
64 66 if counter >= self.nplots:
65 67 break
66 68
67 69 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
68 70
69 71 if showprofile:
70 72 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
71 73
72 74 counter += 1
73 75
74 76 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
75 77 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
76 78 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
77 79 server=None, folder=None, username=None, password=None,
78 80 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
79 81
80 82 """
81 83
82 84 Input:
83 85 dataOut :
84 86 id :
85 87 wintitle :
86 88 channelList :
87 89 showProfile :
88 90 xmin : None,
89 91 xmax : None,
90 92 ymin : None,
91 93 ymax : None,
92 94 zmin : None,
93 95 zmax : None
94 96 """
95 97
96 98 if dataOut.flagNoData:
97 99 return None
98 100
99 101 if realtime:
100 102 if not(isRealtime(utcdatatime = dataOut.utctime)):
101 103 print 'Skipping this plot function'
102 104 return
103 105
104 106 if channelList == None:
105 107 channelIndexList = dataOut.channelIndexList
106 108 else:
107 109 channelIndexList = []
108 110 for channel in channelList:
109 111 if channel not in dataOut.channelList:
110 112 raise ValueError, "Channel %d is not in dataOut.channelList"
111 113 channelIndexList.append(dataOut.channelList.index(channel))
112 114
113 115 factor = dataOut.normFactor
114 116 x = dataOut.abscissaList
115 117 y = dataOut.heightList
116 118
117 119 z = dataOut.data_pre[channelIndexList,:,:]/factor
118 120 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
119 121 avg = numpy.average(z, axis=1)
120 122 noise = dataOut.noise/factor
121 123
122 124 zdB = 10*numpy.log10(z)
123 125 avgdB = 10*numpy.log10(avg)
124 126 noisedB = 10*numpy.log10(noise)
125 127
126 128 #thisDatetime = dataOut.datatime
127 129 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
128 130 title = wintitle + " Parameters"
129 131 xlabel = "Velocity (m/s)"
130 132 ylabel = "Range (Km)"
131 133
132 134 if not self.isConfig:
133 135
134 136 nplots = len(channelIndexList)
135 137
136 138 self.setup(id=id,
137 139 nplots=nplots,
138 140 wintitle=wintitle,
139 141 showprofile=showprofile,
140 142 show=show)
141 143
142 144 if xmin == None: xmin = numpy.nanmin(x)
143 145 if xmax == None: xmax = numpy.nanmax(x)
144 146 if ymin == None: ymin = numpy.nanmin(y)
145 147 if ymax == None: ymax = numpy.nanmax(y)
146 148 if zmin == None: zmin = numpy.nanmin(avgdB)*0.9
147 149 if zmax == None: zmax = numpy.nanmax(avgdB)*0.9
148 150
149 151 self.FTP_WEI = ftp_wei
150 152 self.EXP_CODE = exp_code
151 153 self.SUB_EXP_CODE = sub_exp_code
152 154 self.PLOT_POS = plot_pos
153 155
154 156 self.isConfig = True
155 157
156 158 self.setWinTitle(title)
157 159
158 160 for i in range(self.nplots):
159 161 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
160 162 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i]+1, noisedB[i], str_datetime)
161 163 axes = self.axesList[i*self.__nsubplots]
162 164 axes.pcolor(x, y, zdB[i,:,:],
163 165 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
164 166 xlabel=xlabel, ylabel=ylabel, title=title,
165 167 ticksize=9, cblabel='')
166 168 #Mean Line
167 169 mean = dataOut.data_param[i, 1, :]
168 170 axes.addpline(mean, y, idline=0, color="black", linestyle="solid", lw=1)
169 171
170 172 if self.__showprofile:
171 173 axes = self.axesList[i*self.__nsubplots +1]
172 174 axes.pline(avgdB[i], y,
173 175 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
174 176 xlabel='dB', ylabel='', title='',
175 177 ytick_visible=False,
176 178 grid='x')
177 179
178 180 noiseline = numpy.repeat(noisedB[i], len(y))
179 181 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
180 182
181 183 self.draw()
182
183 if save:
184
185 if figfile == None:
186 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
187 figfile = self.getFilename(name = str_datetime)
188 184
189 self.counter_imagwr += 1
190 if (self.counter_imagwr>=wr_period):
191 # store png plot to local folder
192 self.saveFigure(figpath, figfile)
193 self.counter_imagwr = 0
194 # store png plot to FTP server according to RT-Web format
195 if ftp:
196 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
197 ftp_filename = os.path.join(figpath, name)
198 self.saveFigure(figpath, ftp_filename)
185 self.save(figpath=figpath,
186 figfile=figfile,
187 save=save,
188 ftp=ftp,
189 wr_period=wr_period,
190 thisDatetime=thisDatetime)
199 191
200 192
201 193
202 194 class SkyMapPlot(Figure):
203 195
204 196 __isConfig = None
205 197 __nsubplots = None
206 198
207 199 WIDTHPROF = None
208 200 HEIGHTPROF = None
209 201 PREFIX = 'prm'
210 202
211 203 def __init__(self):
212 204
213 205 self.__isConfig = False
214 206 self.__nsubplots = 1
215 207
216 208 # self.WIDTH = 280
217 209 # self.HEIGHT = 250
218 210 self.WIDTH = 600
219 211 self.HEIGHT = 600
220 212 self.WIDTHPROF = 120
221 213 self.HEIGHTPROF = 0
222 214 self.counter_imagwr = 0
223 215
224 self.PLOT_CODE = 1
216 self.PLOT_CODE = SKYMAP_CODE
217
225 218 self.FTP_WEI = None
226 219 self.EXP_CODE = None
227 220 self.SUB_EXP_CODE = None
228 221 self.PLOT_POS = None
229 222
230 223 def getSubplots(self):
231 224
232 225 ncol = int(numpy.sqrt(self.nplots)+0.9)
233 226 nrow = int(self.nplots*1./ncol + 0.9)
234 227
235 228 return nrow, ncol
236 229
237 230 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
238 231
239 232 self.__showprofile = showprofile
240 233 self.nplots = nplots
241 234
242 235 ncolspan = 1
243 236 colspan = 1
244 237
245 238 self.createFigure(id = id,
246 239 wintitle = wintitle,
247 240 widthplot = self.WIDTH, #+ self.WIDTHPROF,
248 241 heightplot = self.HEIGHT,# + self.HEIGHTPROF,
249 242 show=show)
250 243
251 244 nrow, ncol = 1,1
252 245 counter = 0
253 246 x = 0
254 247 y = 0
255 248 self.addAxes(1, 1, 0, 0, 1, 1, True)
256 249
257 250 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
258 251 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
259 252 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
260 253 server=None, folder=None, username=None, password=None,
261 254 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
262 255
263 256 """
264 257
265 258 Input:
266 259 dataOut :
267 260 id :
268 261 wintitle :
269 262 channelList :
270 263 showProfile :
271 264 xmin : None,
272 265 xmax : None,
273 266 ymin : None,
274 267 ymax : None,
275 268 zmin : None,
276 269 zmax : None
277 270 """
278 271
279 272 arrayParameters = dataOut.data_param
280 273 error = arrayParameters[:,-1]
281 274 indValid = numpy.where(error == 0)[0]
282 275 finalMeteor = arrayParameters[indValid,:]
283 276 finalAzimuth = finalMeteor[:,4]
284 277 finalZenith = finalMeteor[:,5]
285 278
286 279 x = finalAzimuth*numpy.pi/180
287 280 y = finalZenith
288 281
289 282
290 283 #thisDatetime = dataOut.datatime
291 284 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
292 285 title = wintitle + " Parameters"
293 286 xlabel = "Zonal Zenith Angle (deg) "
294 287 ylabel = "Meridional Zenith Angle (deg)"
295 288
296 289 if not self.__isConfig:
297 290
298 291 nplots = 1
299 292
300 293 self.setup(id=id,
301 294 nplots=nplots,
302 295 wintitle=wintitle,
303 296 showprofile=showprofile,
304 297 show=show)
305 298
306 299 self.FTP_WEI = ftp_wei
307 300 self.EXP_CODE = exp_code
308 301 self.SUB_EXP_CODE = sub_exp_code
309 302 self.PLOT_POS = plot_pos
310 303 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
311 304 self.firstdate = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
312 305 self.__isConfig = True
313 306
314 307 self.setWinTitle(title)
315 308
316 309 i = 0
317 310 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
318 311
319 312 axes = self.axesList[i*self.__nsubplots]
320 313 nevents = axes.x_buffer.shape[0] + x.shape[0]
321 314 title = "Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n" %(self.firstdate,str_datetime,nevents)
322 315 axes.polar(x, y,
323 316 title=title, xlabel=xlabel, ylabel=ylabel,
324 317 ticksize=9, cblabel='')
325 318
326 319 self.draw()
327
328 if save:
329 320
330 self.counter_imagwr += 1
331 if (self.counter_imagwr==wr_period):
332
333 if figfile == None:
334 figfile = self.getFilename(name = self.name)
335
336 self.saveFigure(figpath, figfile)
337 self.counter_imagwr = 0
338
339 if ftp:
340 #provisionalmente envia archivos en el formato de la web en tiempo real
341 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
342 path = '%s%03d' %(self.PREFIX, self.id)
343 ftp_file = os.path.join(path,'ftp','%s.png'%name)
344 self.saveFigure(figpath, ftp_file)
345 ftp_filename = os.path.join(figpath,ftp_file)
346
347
348 try:
349 self.sendByFTP(ftp_filename, server, folder, username, password)
350 except:
351 self.counter_imagwr = 0
352 raise ValueError, 'Error FTP'
353
354
355
321 self.save(figpath=figpath,
322 figfile=figfile,
323 save=save,
324 ftp=ftp,
325 wr_period=wr_period,
326 thisDatetime=thisDatetime)
356 327
357 328 class WindProfilerPlot(Figure):
358 329
359 330 __isConfig = None
360 331 __nsubplots = None
361 332
362 333 WIDTHPROF = None
363 334 HEIGHTPROF = None
364 335 PREFIX = 'wind'
365 336
366 337 def __init__(self):
367 338
368 339 self.timerange = 2*60*60
369 340 self.__isConfig = False
370 341 self.__nsubplots = 1
371 342
372 343 self.WIDTH = 800
373 344 self.HEIGHT = 150
374 345 self.WIDTHPROF = 120
375 346 self.HEIGHTPROF = 0
376 347 self.counter_imagwr = 0
377 348
378 self.PLOT_CODE = 0
349 self.PLOT_CODE = WIND_CODE
350
379 351 self.FTP_WEI = None
380 352 self.EXP_CODE = None
381 353 self.SUB_EXP_CODE = None
382 354 self.PLOT_POS = None
383 355 self.tmin = None
384 356 self.tmax = None
385 357
386 358 self.xmin = None
387 359 self.xmax = None
388 360
389 361 self.figfile = None
390 362
391 363 def getSubplots(self):
392 364
393 365 ncol = 1
394 366 nrow = self.nplots
395 367
396 368 return nrow, ncol
397 369
398 370 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
399 371
400 372 self.__showprofile = showprofile
401 373 self.nplots = nplots
402 374
403 375 ncolspan = 1
404 376 colspan = 1
405 377
406 378 self.createFigure(id = id,
407 379 wintitle = wintitle,
408 380 widthplot = self.WIDTH + self.WIDTHPROF,
409 381 heightplot = self.HEIGHT + self.HEIGHTPROF,
410 382 show=show)
411 383
412 384 nrow, ncol = self.getSubplots()
413 385
414 386 counter = 0
415 387 for y in range(nrow):
416 388 if counter >= self.nplots:
417 389 break
418 390
419 391 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
420 392 counter += 1
421 393
422 394 def run(self, dataOut, id, wintitle="", channelList=None,
423 395 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
424 396 zmax_ver = None, zmin_ver = None, SNRmin = None, SNRmax = None,
425 397 timerange=None, SNRthresh = None,
426 398 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
427 399 server=None, folder=None, username=None, password=None,
428 400 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
429 401 """
430 402
431 403 Input:
432 404 dataOut :
433 405 id :
434 406 wintitle :
435 407 channelList :
436 408 showProfile :
437 409 xmin : None,
438 410 xmax : None,
439 411 ymin : None,
440 412 ymax : None,
441 413 zmin : None,
442 414 zmax : None
443 415 """
444 416
445 417 if channelList == None:
446 418 channelIndexList = dataOut.channelIndexList
447 419 else:
448 420 channelIndexList = []
449 421 for channel in channelList:
450 422 if channel not in dataOut.channelList:
451 423 raise ValueError, "Channel %d is not in dataOut.channelList"
452 424 channelIndexList.append(dataOut.channelList.index(channel))
453 425
454 426 if timerange != None:
455 427 self.timerange = timerange
456 428
457 429 tmin = None
458 430 tmax = None
459 431
460 432 x = dataOut.getTimeRange1()
461 433 # y = dataOut.heightList
462 434 y = dataOut.heightList
463 435
464 436 z = dataOut.data_output.copy()
465 437 nplots = z.shape[0] #Number of wind dimensions estimated
466 438 nplotsw = nplots
467 439
468 440 #If there is a SNR function defined
469 441 if dataOut.data_SNR != None:
470 442 nplots += 1
471 443 SNR = dataOut.data_SNR
472 444 SNRavg = numpy.average(SNR, axis=0)
473 445
474 446 SNRdB = 10*numpy.log10(SNR)
475 447 SNRavgdB = 10*numpy.log10(SNRavg)
476 448
477 449 if SNRthresh == None: SNRthresh = -5.0
478 450 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
479 451
480 452 for i in range(nplotsw):
481 453 z[i,ind] = numpy.nan
482 454
483 455
484 456 showprofile = False
485 457 # thisDatetime = dataOut.datatime
486 458 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
487 459 title = wintitle + "Wind"
488 460 xlabel = ""
489 461 ylabel = "Range (Km)"
490 462
491 463 if not self.__isConfig:
492 464
493 465 self.setup(id=id,
494 466 nplots=nplots,
495 467 wintitle=wintitle,
496 468 showprofile=showprofile,
497 469 show=show)
498 470
499 471 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
500 472
501 473 if ymin == None: ymin = numpy.nanmin(y)
502 474 if ymax == None: ymax = numpy.nanmax(y)
503 475
504 476 if zmax == None: zmax = numpy.nanmax(abs(z[range(2),:]))
505 477 #if numpy.isnan(zmax): zmax = 50
506 478 if zmin == None: zmin = -zmax
507 479
508 480 if nplotsw == 3:
509 481 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
510 482 if zmin_ver == None: zmin_ver = -zmax_ver
511 483
512 484 if dataOut.data_SNR != None:
513 485 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
514 486 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
515 487
516 488 self.FTP_WEI = ftp_wei
517 489 self.EXP_CODE = exp_code
518 490 self.SUB_EXP_CODE = sub_exp_code
519 491 self.PLOT_POS = plot_pos
520 492
521 493 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
522 494 self.__isConfig = True
523 495
524 496
525 497 self.setWinTitle(title)
526 498
527 499 if ((self.xmax - x[1]) < (x[1]-x[0])):
528 500 x[1] = self.xmax
529 501
530 502 strWind = ['Zonal', 'Meridional', 'Vertical']
531 503 strCb = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
532 504 zmaxVector = [zmax, zmax, zmax_ver]
533 505 zminVector = [zmin, zmin, zmin_ver]
534 506 windFactor = [1,1,100]
535 507
536 508 for i in range(nplotsw):
537 509
538 510 title = "%s Wind: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
539 511 axes = self.axesList[i*self.__nsubplots]
540 512
541 513 z1 = z[i,:].reshape((1,-1))*windFactor[i]
542 514
543 515 axes.pcolorbuffer(x, y, z1,
544 516 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
545 517 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
546 518 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="RdBu_r" )
547 519
548 520 if dataOut.data_SNR != None:
549 521 i += 1
550 522 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
551 523 axes = self.axesList[i*self.__nsubplots]
552 524
553 525 SNRavgdB = SNRavgdB.reshape((1,-1))
554 526
555 527 axes.pcolorbuffer(x, y, SNRavgdB,
556 528 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
557 529 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
558 530 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
559 531
560 532 self.draw()
561 533
562 if save:
563
564 if self.figfile == None:
565 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
566 self.figfile = self.getFilename(name = str_datetime)
567
568 self.counter_imagwr += 1
569
570 if (self.counter_imagwr>=wr_period):
571 # store png plot to local folder
572 self.saveFigure(figpath, self.figfile)
573 self.counter_imagwr = 0
574
575 if ftp:
576 # store png plot to FTP server according to RT-Web format
577 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
578 ftp_filename = os.path.join(figpath, name)
579 self.saveFigure(figpath, ftp_filename)
580
581
582
583 534 if x[1] >= self.axesList[0].xmax:
584 535 self.counter_imagwr = wr_period
585 536 self.__isConfig = False
586 537 self.figfile = None
587 538
539 self.save(figpath=figpath,
540 figfile=figfile,
541 save=save,
542 ftp=ftp,
543 wr_period=wr_period,
544 thisDatetime=thisDatetime,
545 update_figfile=False)
546
588 547
589 548 class ParametersPlot(Figure):
590 549
591 550 __isConfig = None
592 551 __nsubplots = None
593 552
594 553 WIDTHPROF = None
595 554 HEIGHTPROF = None
596 555 PREFIX = 'prm'
597 556
598 557 def __init__(self):
599 558
600 559 self.timerange = 2*60*60
601 560 self.__isConfig = False
602 561 self.__nsubplots = 1
603 562
604 563 self.WIDTH = 800
605 564 self.HEIGHT = 150
606 565 self.WIDTHPROF = 120
607 566 self.HEIGHTPROF = 0
608 567 self.counter_imagwr = 0
609 568
610 self.PLOT_CODE = 0
569 self.PLOT_CODE = PARMS_CODE
570
611 571 self.FTP_WEI = None
612 572 self.EXP_CODE = None
613 573 self.SUB_EXP_CODE = None
614 574 self.PLOT_POS = None
615 575 self.tmin = None
616 576 self.tmax = None
617 577
618 578 self.xmin = None
619 579 self.xmax = None
620 580
621 581 self.figfile = None
622 582
623 583 def getSubplots(self):
624 584
625 585 ncol = 1
626 586 nrow = self.nplots
627 587
628 588 return nrow, ncol
629 589
630 590 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
631 591
632 592 self.__showprofile = showprofile
633 593 self.nplots = nplots
634 594
635 595 ncolspan = 1
636 596 colspan = 1
637 597
638 598 self.createFigure(id = id,
639 599 wintitle = wintitle,
640 600 widthplot = self.WIDTH + self.WIDTHPROF,
641 601 heightplot = self.HEIGHT + self.HEIGHTPROF,
642 602 show=show)
643 603
644 604 nrow, ncol = self.getSubplots()
645 605
646 606 counter = 0
647 607 for y in range(nrow):
648 608 for x in range(ncol):
649 609
650 610 if counter >= self.nplots:
651 611 break
652 612
653 613 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
654 614
655 615 if showprofile:
656 616 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
657 617
658 618 counter += 1
659 619
660 620 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
661 621 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
662 622 parameterIndex = None, onlyPositive = False,
663 623 SNRthresh = -numpy.inf, SNR = True, SNRmin = None, SNRmax = None,
664 624 zlabel = "", parameterName = "", parameterObject = "data_param",
665 625 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
666 626 server=None, folder=None, username=None, password=None,
667 627 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
668 628
669 629 """
670 630
671 631 Input:
672 632 dataOut :
673 633 id :
674 634 wintitle :
675 635 channelList :
676 636 showProfile :
677 637 xmin : None,
678 638 xmax : None,
679 639 ymin : None,
680 640 ymax : None,
681 641 zmin : None,
682 642 zmax : None
683 643 """
684 644
685 645 data_param = getattr(dataOut, parameterObject)
686 646
687 647 if channelList == None:
688 648 channelIndexList = numpy.arange(data_param.shape[0])
689 649 else:
690 650 channelIndexList = numpy.array(channelList)
691 651
692 652 nchan = len(channelIndexList) #Number of channels being plotted
693 653
694 654 if timerange != None:
695 655 self.timerange = timerange
696 656
697 657 #tmin = None
698 658 #tmax = None
699 659 if parameterIndex == None:
700 660 parameterIndex = 1
701 661 x = dataOut.getTimeRange1()
702 662 y = dataOut.heightList
703 663 z = data_param[channelIndexList,parameterIndex,:].copy()
704 664
705 665 zRange = dataOut.abscissaList
706 666 nplots = z.shape[0] #Number of wind dimensions estimated
707 667 # thisDatetime = dataOut.datatime
708 668
709 669 if dataOut.data_SNR != None:
710 670 SNRarray = dataOut.data_SNR[channelIndexList,:]
711 671 SNRdB = 10*numpy.log10(SNRarray)
712 672 # SNRavgdB = 10*numpy.log10(SNRavg)
713 673 ind = numpy.where(SNRdB < 10**(SNRthresh/10))
714 674 z[ind] = numpy.nan
715 675
716 676 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
717 677 title = wintitle + " Parameters Plot" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
718 678 xlabel = ""
719 679 ylabel = "Range (Km)"
720 680
721 681 if SNR: nplots = 2*nplots
722 682
723 683 if onlyPositive:
724 684 colormap = "jet"
725 685 zmin = 0
726 686 else: colormap = "RdBu_r"
727 687
728 688 if not self.__isConfig:
729 689
730 690 self.setup(id=id,
731 691 nplots=nplots,
732 692 wintitle=wintitle,
733 693 showprofile=showprofile,
734 694 show=show)
735 695
736 696 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
737 697
738 698 if ymin == None: ymin = numpy.nanmin(y)
739 699 if ymax == None: ymax = numpy.nanmax(y)
740 700 if zmin == None: zmin = numpy.nanmin(zRange)
741 701 if zmax == None: zmax = numpy.nanmax(zRange)
742 702
743 703 if SNR != None:
744 704 if SNRmin == None: SNRmin = numpy.nanmin(SNRdB)
745 705 if SNRmax == None: SNRmax = numpy.nanmax(SNRdB)
746 706
747 707 self.FTP_WEI = ftp_wei
748 708 self.EXP_CODE = exp_code
749 709 self.SUB_EXP_CODE = sub_exp_code
750 710 self.PLOT_POS = plot_pos
751 711
752 712 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
753 713 self.__isConfig = True
754 714 self.figfile = figfile
755 715
756 716 self.setWinTitle(title)
757 717
758 718 if ((self.xmax - x[1]) < (x[1]-x[0])):
759 719 x[1] = self.xmax
760 720
761 721 for i in range(nchan):
762 722 if SNR: j = 2*i
763 723 else: j = i
764 724
765 725 title = "%s Channel %d: %s" %(parameterName, channelIndexList[i]+1, thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
766 726
767 727 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
768 728 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
769 729 axes = self.axesList[j*self.__nsubplots]
770 730 z1 = z[i,:].reshape((1,-1))
771 731 axes.pcolorbuffer(x, y, z1,
772 732 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
773 733 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap=colormap,
774 734 ticksize=9, cblabel=zlabel, cbsize="1%")
775 735
776 736 if SNR:
777 737 title = "Channel %d Signal Noise Ratio (SNR): %s" %(channelIndexList[i]+1, thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
778 738 axes = self.axesList[(j + 1)*self.__nsubplots]
779 739 z1 = SNRdB[i,:].reshape((1,-1))
780 740 axes.pcolorbuffer(x, y, z1,
781 741 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
782 742 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,colormap="jet",
783 743 ticksize=9, cblabel=zlabel, cbsize="1%")
784 744
785 745
786 746
787 747 self.draw()
788 748
789 if save:
790
791 if self.figfile == None:
792 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
793 self.figfile = self.getFilename(name = str_datetime)
794
795 self.counter_imagwr += 1
796
797 if (self.counter_imagwr>=wr_period):
798 # store png plot to local folder
799 self.saveFigure(figpath, self.figfile)
800 self.counter_imagwr = 0
801
802 if ftp:
803 # store png plot to FTP server according to RT-Web format
804 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
805 ftp_filename = os.path.join(figpath, name)
806 self.saveFigure(figpath, ftp_filename)
807
808 749 if x[1] >= self.axesList[0].xmax:
809 750 self.counter_imagwr = wr_period
810 751 self.__isConfig = False
811 752 self.figfile = None
812 753
813
754 self.save(figpath=figpath,
755 figfile=figfile,
756 save=save,
757 ftp=ftp,
758 wr_period=wr_period,
759 thisDatetime=thisDatetime,
760 update_figfile=False)
761
814 762 class SpectralFittingPlot(Figure):
815 763
816 764 __isConfig = None
817 765 __nsubplots = None
818 766
819 767 WIDTHPROF = None
820 768 HEIGHTPROF = None
821 769 PREFIX = 'prm'
822 770
823 771
824 772 N = None
825 773 ippSeconds = None
826 774
827 775 def __init__(self):
828 776 self.__isConfig = False
829 777 self.__nsubplots = 1
830 778
779 self.PLOT_CODE = SPECFIT_CODE
780
831 781 self.WIDTH = 450
832 782 self.HEIGHT = 250
833 783 self.WIDTHPROF = 0
834 784 self.HEIGHTPROF = 0
835 785
836 786 def getSubplots(self):
837 787
838 788 ncol = int(numpy.sqrt(self.nplots)+0.9)
839 789 nrow = int(self.nplots*1./ncol + 0.9)
840 790
841 791 return nrow, ncol
842 792
843 793 def setup(self, id, nplots, wintitle, showprofile=False, show=True):
844 794
845 795 showprofile = False
846 796 self.__showprofile = showprofile
847 797 self.nplots = nplots
848 798
849 799 ncolspan = 5
850 800 colspan = 4
851 801 if showprofile:
852 802 ncolspan = 5
853 803 colspan = 4
854 804 self.__nsubplots = 2
855 805
856 806 self.createFigure(id = id,
857 807 wintitle = wintitle,
858 808 widthplot = self.WIDTH + self.WIDTHPROF,
859 809 heightplot = self.HEIGHT + self.HEIGHTPROF,
860 810 show=show)
861 811
862 812 nrow, ncol = self.getSubplots()
863 813
864 814 counter = 0
865 815 for y in range(nrow):
866 816 for x in range(ncol):
867 817
868 818 if counter >= self.nplots:
869 819 break
870 820
871 821 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
872 822
873 823 if showprofile:
874 824 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
875 825
876 826 counter += 1
877 827
878 828 def run(self, dataOut, id, cutHeight=None, fit=False, wintitle="", channelList=None, showprofile=True,
879 829 xmin=None, xmax=None, ymin=None, ymax=None,
880 830 save=False, figpath='./', figfile=None, show=True):
881 831
882 832 """
883 833
884 834 Input:
885 835 dataOut :
886 836 id :
887 837 wintitle :
888 838 channelList :
889 839 showProfile :
890 840 xmin : None,
891 841 xmax : None,
892 842 zmin : None,
893 843 zmax : None
894 844 """
895 845
896 846 if cutHeight==None:
897 847 h=270
898 848 heightindex = numpy.abs(cutHeight - dataOut.heightList).argmin()
899 849 cutHeight = dataOut.heightList[heightindex]
900 850
901 851 factor = dataOut.normFactor
902 852 x = dataOut.abscissaList[:-1]
903 853 #y = dataOut.getHeiRange()
904 854
905 855 z = dataOut.data_pre[:,:,heightindex]/factor
906 856 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
907 857 avg = numpy.average(z, axis=1)
908 858 listChannels = z.shape[0]
909 859
910 860 #Reconstruct Function
911 861 if fit==True:
912 862 groupArray = dataOut.groupList
913 863 listChannels = groupArray.reshape((groupArray.size))
914 864 listChannels.sort()
915 865 spcFitLine = numpy.zeros(z.shape)
916 866 constants = dataOut.constants
917 867
918 868 nGroups = groupArray.shape[0]
919 869 nChannels = groupArray.shape[1]
920 870 nProfiles = z.shape[1]
921 871
922 872 for f in range(nGroups):
923 873 groupChann = groupArray[f,:]
924 874 p = dataOut.data_param[f,:,heightindex]
925 875 # p = numpy.array([ 89.343967,0.14036615,0.17086219,18.89835291,1.58388365,1.55099167])
926 876 fitLineAux = dataOut.library.modelFunction(p, constants)*nProfiles
927 877 fitLineAux = fitLineAux.reshape((nChannels,nProfiles))
928 878 spcFitLine[groupChann,:] = fitLineAux
929 879 # spcFitLine = spcFitLine/factor
930 880
931 881 z = z[listChannels,:]
932 882 spcFitLine = spcFitLine[listChannels,:]
933 883 spcFitLinedB = 10*numpy.log10(spcFitLine)
934 884
935 885 zdB = 10*numpy.log10(z)
936 886 #thisDatetime = dataOut.datatime
937 887 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
938 888 title = wintitle + " Doppler Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
939 889 xlabel = "Velocity (m/s)"
940 890 ylabel = "Spectrum"
941 891
942 892 if not self.__isConfig:
943 893
944 894 nplots = listChannels.size
945 895
946 896 self.setup(id=id,
947 897 nplots=nplots,
948 898 wintitle=wintitle,
949 899 showprofile=showprofile,
950 900 show=show)
951 901
952 902 if xmin == None: xmin = numpy.nanmin(x)
953 903 if xmax == None: xmax = numpy.nanmax(x)
954 904 if ymin == None: ymin = numpy.nanmin(zdB)
955 905 if ymax == None: ymax = numpy.nanmax(zdB)+2
956 906
957 907 self.__isConfig = True
958 908
959 909 self.setWinTitle(title)
960 910 for i in range(self.nplots):
961 911 # title = "Channel %d: %4.2fdB" %(dataOut.channelList[i]+1, noisedB[i])
962 912 title = "Height %4.1f km\nChannel %d:" %(cutHeight, listChannels[i]+1)
963 913 axes = self.axesList[i*self.__nsubplots]
964 914 if fit == False:
965 915 axes.pline(x, zdB[i,:],
966 916 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
967 917 xlabel=xlabel, ylabel=ylabel, title=title
968 918 )
969 919 if fit == True:
970 920 fitline=spcFitLinedB[i,:]
971 921 y=numpy.vstack([zdB[i,:],fitline] )
972 922 legendlabels=['Data','Fitting']
973 923 axes.pmultilineyaxis(x, y,
974 924 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
975 925 xlabel=xlabel, ylabel=ylabel, title=title,
976 926 legendlabels=legendlabels, marker=None,
977 927 linestyle='solid', grid='both')
978 928
979 929 self.draw()
980
981 if save:
982 date = thisDatetime.strftime("%Y%m%d_%H%M%S")
983 if figfile == None:
984 figfile = self.getFilename(name = date)
985
986 self.saveFigure(figpath, figfile)
930
931 self.save(figpath=figpath,
932 figfile=figfile,
933 save=save,
934 ftp=ftp,
935 wr_period=wr_period,
936 thisDatetime=thisDatetime)
987 937
988 938
989 939 class EWDriftsPlot(Figure):
990 940
991 941 __isConfig = None
992 942 __nsubplots = None
993 943
994 944 WIDTHPROF = None
995 945 HEIGHTPROF = None
996 946 PREFIX = 'drift'
997 947
998 948 def __init__(self):
999 949
1000 950 self.timerange = 2*60*60
1001 951 self.isConfig = False
1002 952 self.__nsubplots = 1
1003 953
1004 954 self.WIDTH = 800
1005 955 self.HEIGHT = 150
1006 956 self.WIDTHPROF = 120
1007 957 self.HEIGHTPROF = 0
1008 958 self.counter_imagwr = 0
1009 959
1010 self.PLOT_CODE = 0
960 self.PLOT_CODE = EWDRIFT_CODE
961
1011 962 self.FTP_WEI = None
1012 963 self.EXP_CODE = None
1013 964 self.SUB_EXP_CODE = None
1014 965 self.PLOT_POS = None
1015 966 self.tmin = None
1016 967 self.tmax = None
1017 968
1018 969 self.xmin = None
1019 970 self.xmax = None
1020 971
1021 972 self.figfile = None
1022 973
1023 974 def getSubplots(self):
1024 975
1025 976 ncol = 1
1026 977 nrow = self.nplots
1027 978
1028 979 return nrow, ncol
1029 980
1030 981 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1031 982
1032 983 self.__showprofile = showprofile
1033 984 self.nplots = nplots
1034 985
1035 986 ncolspan = 1
1036 987 colspan = 1
1037 988
1038 989 self.createFigure(id = id,
1039 990 wintitle = wintitle,
1040 991 widthplot = self.WIDTH + self.WIDTHPROF,
1041 992 heightplot = self.HEIGHT + self.HEIGHTPROF,
1042 993 show=show)
1043 994
1044 995 nrow, ncol = self.getSubplots()
1045 996
1046 997 counter = 0
1047 998 for y in range(nrow):
1048 999 if counter >= self.nplots:
1049 1000 break
1050 1001
1051 1002 self.addAxes(nrow, ncol*ncolspan, y, 0, colspan, 1)
1052 1003 counter += 1
1053 1004
1054 1005 def run(self, dataOut, id, wintitle="", channelList=None,
1055 1006 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
1056 1007 zmaxVertical = None, zminVertical = None, zmaxZonal = None, zminZonal = None,
1057 1008 timerange=None, SNRthresh = -numpy.inf, SNRmin = None, SNRmax = None, SNR_1 = False,
1058 1009 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
1059 1010 server=None, folder=None, username=None, password=None,
1060 1011 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1061 1012 """
1062 1013
1063 1014 Input:
1064 1015 dataOut :
1065 1016 id :
1066 1017 wintitle :
1067 1018 channelList :
1068 1019 showProfile :
1069 1020 xmin : None,
1070 1021 xmax : None,
1071 1022 ymin : None,
1072 1023 ymax : None,
1073 1024 zmin : None,
1074 1025 zmax : None
1075 1026 """
1076 1027
1077 1028 if timerange != None:
1078 1029 self.timerange = timerange
1079 1030
1080 1031 tmin = None
1081 1032 tmax = None
1082 1033
1083 1034 x = dataOut.getTimeRange1()
1084 1035 # y = dataOut.heightList
1085 1036 y = dataOut.heightList
1086 1037
1087 1038 z = dataOut.data_output
1088 1039 nplots = z.shape[0] #Number of wind dimensions estimated
1089 1040 nplotsw = nplots
1090 1041
1091 1042 #If there is a SNR function defined
1092 1043 if dataOut.data_SNR != None:
1093 1044 nplots += 1
1094 1045 SNR = dataOut.data_SNR
1095 1046
1096 1047 if SNR_1:
1097 1048 SNR += 1
1098 1049
1099 1050 SNRavg = numpy.average(SNR, axis=0)
1100 1051
1101 1052 SNRdB = 10*numpy.log10(SNR)
1102 1053 SNRavgdB = 10*numpy.log10(SNRavg)
1103 1054
1104 1055 ind = numpy.where(SNRavg < 10**(SNRthresh/10))[0]
1105 1056
1106 1057 for i in range(nplotsw):
1107 1058 z[i,ind] = numpy.nan
1108 1059
1109 1060
1110 1061 showprofile = False
1111 1062 # thisDatetime = dataOut.datatime
1112 1063 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1113 1064 title = wintitle + " EW Drifts"
1114 1065 xlabel = ""
1115 1066 ylabel = "Height (Km)"
1116 1067
1117 1068 if not self.__isConfig:
1118 1069
1119 1070 self.setup(id=id,
1120 1071 nplots=nplots,
1121 1072 wintitle=wintitle,
1122 1073 showprofile=showprofile,
1123 1074 show=show)
1124 1075
1125 1076 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1126 1077
1127 1078 if ymin == None: ymin = numpy.nanmin(y)
1128 1079 if ymax == None: ymax = numpy.nanmax(y)
1129 1080
1130 1081 if zmaxZonal == None: zmaxZonal = numpy.nanmax(abs(z[0,:]))
1131 1082 if zminZonal == None: zminZonal = -zmaxZonal
1132 1083 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1133 1084 if zminVertical == None: zminVertical = -zmaxVertical
1134 1085
1135 1086 if dataOut.data_SNR != None:
1136 1087 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1137 1088 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1138 1089
1139 1090 self.FTP_WEI = ftp_wei
1140 1091 self.EXP_CODE = exp_code
1141 1092 self.SUB_EXP_CODE = sub_exp_code
1142 1093 self.PLOT_POS = plot_pos
1143 1094
1144 1095 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1145 1096 self.__isConfig = True
1146 1097
1147 1098
1148 1099 self.setWinTitle(title)
1149 1100
1150 1101 if ((self.xmax - x[1]) < (x[1]-x[0])):
1151 1102 x[1] = self.xmax
1152 1103
1153 1104 strWind = ['Zonal','Vertical']
1154 1105 strCb = 'Velocity (m/s)'
1155 1106 zmaxVector = [zmaxZonal, zmaxVertical]
1156 1107 zminVector = [zminZonal, zminVertical]
1157 1108
1158 1109 for i in range(nplotsw):
1159 1110
1160 1111 title = "%s Drifts: %s" %(strWind[i], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1161 1112 axes = self.axesList[i*self.__nsubplots]
1162 1113
1163 1114 z1 = z[i,:].reshape((1,-1))
1164 1115
1165 1116 axes.pcolorbuffer(x, y, z1,
1166 1117 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zminVector[i], zmax=zmaxVector[i],
1167 1118 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1168 1119 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1169 1120
1170 1121 if dataOut.data_SNR != None:
1171 1122 i += 1
1172 1123 if SNR_1:
1173 1124 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1174 1125 else:
1175 1126 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1176 1127 axes = self.axesList[i*self.__nsubplots]
1177 1128 SNRavgdB = SNRavgdB.reshape((1,-1))
1178 1129
1179 1130 axes.pcolorbuffer(x, y, SNRavgdB,
1180 1131 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=SNRmin, zmax=SNRmax,
1181 1132 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1182 1133 ticksize=9, cblabel='', cbsize="1%", colormap="jet")
1183 1134
1184 1135 self.draw()
1185 1136
1186 if save:
1187
1188 if self.figfile == None:
1189 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1190 self.figfile = self.getFilename(name = str_datetime)
1191
1192 self.counter_imagwr += 1
1193
1194 if (self.counter_imagwr>=wr_period):
1195 # store png plot to local folder
1196 self.saveFigure(figpath, self.figfile)
1197 self.counter_imagwr = 0
1198
1199 if ftp:
1200 # store png plot to FTP server according to RT-Web format
1201 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
1202 ftp_filename = os.path.join(figpath, name)
1203 self.saveFigure(figpath, ftp_filename)
1204
1205 1137 if x[1] >= self.axesList[0].xmax:
1206 1138 self.counter_imagwr = wr_period
1207 1139 self.__isConfig = False
1208 self.figfile = None No newline at end of file
1140 self.figfile = None
1141
1142 self.save(figpath=figpath,
1143 figfile=figfile,
1144 save=save,
1145 ftp=ftp,
1146 wr_period=wr_period,
1147 thisDatetime=thisDatetime,
1148 update_figfile=False) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1375 +1,1314
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from figure import Figure, isRealtime
11 from plotting_codes import *
11 12
12 13 class SpectraPlot(Figure):
13 14
14 15 isConfig = None
15 16 __nsubplots = None
16 17
17 18 WIDTHPROF = None
18 19 HEIGHTPROF = None
19 20 PREFIX = 'spc'
20 21
21 22 def __init__(self):
22 23
23 24 self.isConfig = False
24 25 self.__nsubplots = 1
25 26
26 27 self.WIDTH = 280
27 28 self.HEIGHT = 250
28 29 self.WIDTHPROF = 120
29 30 self.HEIGHTPROF = 0
30 31 self.counter_imagwr = 0
31 32
32 self.PLOT_CODE = 1
33 self.PLOT_CODE = SPEC_CODE
34
33 35 self.FTP_WEI = None
34 36 self.EXP_CODE = None
35 37 self.SUB_EXP_CODE = None
36 38 self.PLOT_POS = None
37 39
38 40 self.__xfilter_ena = False
39 41 self.__yfilter_ena = False
40 42
41 43 def getSubplots(self):
42 44
43 45 ncol = int(numpy.sqrt(self.nplots)+0.9)
44 46 nrow = int(self.nplots*1./ncol + 0.9)
45 47
46 48 return nrow, ncol
47 49
48 50 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
49 51
50 52 self.__showprofile = showprofile
51 53 self.nplots = nplots
52 54
53 55 ncolspan = 1
54 56 colspan = 1
55 57 if showprofile:
56 58 ncolspan = 3
57 59 colspan = 2
58 60 self.__nsubplots = 2
59 61
60 62 self.createFigure(id = id,
61 63 wintitle = wintitle,
62 64 widthplot = self.WIDTH + self.WIDTHPROF,
63 65 heightplot = self.HEIGHT + self.HEIGHTPROF,
64 66 show=show)
65 67
66 68 nrow, ncol = self.getSubplots()
67 69
68 70 counter = 0
69 71 for y in range(nrow):
70 72 for x in range(ncol):
71 73
72 74 if counter >= self.nplots:
73 75 break
74 76
75 77 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
76 78
77 79 if showprofile:
78 80 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
79 81
80 82 counter += 1
81 83
82 84 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
83 85 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
84 86 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
85 87 server=None, folder=None, username=None, password=None,
86 88 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False):
87 89
88 90 """
89 91
90 92 Input:
91 93 dataOut :
92 94 id :
93 95 wintitle :
94 96 channelList :
95 97 showProfile :
96 98 xmin : None,
97 99 xmax : None,
98 100 ymin : None,
99 101 ymax : None,
100 102 zmin : None,
101 103 zmax : None
102 104 """
103 105
104 106 if realtime:
105 107 if not(isRealtime(utcdatatime = dataOut.utctime)):
106 108 print 'Skipping this plot function'
107 109 return
108 110
109 111 if channelList == None:
110 112 channelIndexList = dataOut.channelIndexList
111 113 else:
112 114 channelIndexList = []
113 115 for channel in channelList:
114 116 if channel not in dataOut.channelList:
115 117 raise ValueError, "Channel %d is not in dataOut.channelList"
116 118 channelIndexList.append(dataOut.channelList.index(channel))
117 119
118 120 factor = dataOut.normFactor
119 121
120 122 x = dataOut.getVelRange(1)
121 123 y = dataOut.getHeiRange()
122 124
123 125 z = dataOut.data_spc[channelIndexList,:,:]/factor
124 126 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
125 127 zdB = 10*numpy.log10(z)
126 128
127 129 avg = numpy.nanmean(z, axis=1)
128 130 avgdB = 10*numpy.log10(avg)
129 131
130 132 noise = dataOut.getNoise()/factor
131 133 noisedB = 10*numpy.log10(noise)
132 134
133 135 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
134 136 title = wintitle + " Spectra"
135 137 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
136 138 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
137 139
138 140 xlabel = "Velocity (m/s)"
139 141 ylabel = "Range (Km)"
140 142
141 143 if not self.isConfig:
142 144
143 145 nplots = len(channelIndexList)
144 146
145 147 self.setup(id=id,
146 148 nplots=nplots,
147 149 wintitle=wintitle,
148 150 showprofile=showprofile,
149 151 show=show)
150 152
151 153 if xmin == None: xmin = numpy.nanmin(x)
152 154 if xmax == None: xmax = numpy.nanmax(x)
153 155 if ymin == None: ymin = numpy.nanmin(y)
154 156 if ymax == None: ymax = numpy.nanmax(y)
155 157 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
156 158 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
157 159
158 160 self.FTP_WEI = ftp_wei
159 161 self.EXP_CODE = exp_code
160 162 self.SUB_EXP_CODE = sub_exp_code
161 163 self.PLOT_POS = plot_pos
162 164
163 165 self.isConfig = True
164 166
165 167 self.setWinTitle(title)
166 168
167 169 for i in range(self.nplots):
168 170 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
169 171 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[i]+1, noisedB[i], str_datetime)
170 172 if len(dataOut.beam.codeList) != 0:
171 173 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[i]+1, noisedB[i], dataOut.beam.azimuthList[i], dataOut.beam.zenithList[i], str_datetime)
172 174
173 175 axes = self.axesList[i*self.__nsubplots]
174 176 axes.pcolor(x, y, zdB[i,:,:],
175 177 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
176 178 xlabel=xlabel, ylabel=ylabel, title=title,
177 179 ticksize=9, cblabel='')
178 180
179 181 if self.__showprofile:
180 182 axes = self.axesList[i*self.__nsubplots +1]
181 183 axes.pline(avgdB[i,:], y,
182 184 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
183 185 xlabel='dB', ylabel='', title='',
184 186 ytick_visible=False,
185 187 grid='x')
186 188
187 189 noiseline = numpy.repeat(noisedB[i], len(y))
188 190 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
189 191
190 192 self.draw()
191
192 if save:
193
194 if figfile == None:
195 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
196 figfile = self.getFilename(name = str_datetime)
197 name = str_datetime
198 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
199 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
200 figfile = self.getFilename(name)
201
202 self.counter_imagwr += 1
203
204 if (self.counter_imagwr>=wr_period):
205 # store png plot to local folder
206 self.saveFigure(figpath, figfile)
207 self.counter_imagwr = 0
193
194 if figfile == None:
195 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
196 name = str_datetime
197 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
198 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
199 figfile = self.getFilename(name)
208 200
209 if ftp:
210 # store png plot to FTP server according to RT-Web format
211 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
212 ftp_filename = os.path.join(figpath, name)
213 self.saveFigure(figpath, ftp_filename)
214
215
201 self.save(figpath=figpath,
202 figfile=figfile,
203 save=save,
204 ftp=ftp,
205 wr_period=wr_period,
206 thisDatetime=thisDatetime)
216 207
217 208 class CrossSpectraPlot(Figure):
218 209
219 210 isConfig = None
220 211 __nsubplots = None
221 212
222 213 WIDTH = None
223 214 HEIGHT = None
224 215 WIDTHPROF = None
225 216 HEIGHTPROF = None
226 217 PREFIX = 'cspc'
227 218
228 219 def __init__(self):
229 220
230 221 self.isConfig = False
231 222 self.__nsubplots = 4
232 223 self.counter_imagwr = 0
233 224 self.WIDTH = 250
234 225 self.HEIGHT = 250
235 226 self.WIDTHPROF = 0
236 227 self.HEIGHTPROF = 0
237 228
238 self.PLOT_CODE = 1
229 self.PLOT_CODE = CROSS_CODE
239 230 self.FTP_WEI = None
240 231 self.EXP_CODE = None
241 232 self.SUB_EXP_CODE = None
242 233 self.PLOT_POS = None
243 234
244 235 def getSubplots(self):
245 236
246 237 ncol = 4
247 238 nrow = self.nplots
248 239
249 240 return nrow, ncol
250 241
251 242 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
252 243
253 244 self.__showprofile = showprofile
254 245 self.nplots = nplots
255 246
256 247 ncolspan = 1
257 248 colspan = 1
258 249
259 250 self.createFigure(id = id,
260 251 wintitle = wintitle,
261 252 widthplot = self.WIDTH + self.WIDTHPROF,
262 253 heightplot = self.HEIGHT + self.HEIGHTPROF,
263 254 show=True)
264 255
265 256 nrow, ncol = self.getSubplots()
266 257
267 258 counter = 0
268 259 for y in range(nrow):
269 260 for x in range(ncol):
270 261 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
271 262
272 263 counter += 1
273 264
274 265 def run(self, dataOut, id, wintitle="", pairsList=None,
275 266 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
276 267 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
277 268 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
278 269 server=None, folder=None, username=None, password=None,
279 270 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
280 271
281 272 """
282 273
283 274 Input:
284 275 dataOut :
285 276 id :
286 277 wintitle :
287 278 channelList :
288 279 showProfile :
289 280 xmin : None,
290 281 xmax : None,
291 282 ymin : None,
292 283 ymax : None,
293 284 zmin : None,
294 285 zmax : None
295 286 """
296 287
297 288 if pairsList == None:
298 289 pairsIndexList = dataOut.pairsIndexList
299 290 else:
300 291 pairsIndexList = []
301 292 for pair in pairsList:
302 293 if pair not in dataOut.pairsList:
303 294 raise ValueError, "Pair %s is not in dataOut.pairsList" %str(pair)
304 295 pairsIndexList.append(dataOut.pairsList.index(pair))
305 296
306 297 if pairsIndexList == []:
307 298 return
308 299
309 300 if len(pairsIndexList) > 4:
310 301 pairsIndexList = pairsIndexList[0:4]
311 302 factor = dataOut.normFactor
312 303 x = dataOut.getVelRange(1)
313 304 y = dataOut.getHeiRange()
314 305 z = dataOut.data_spc[:,:,:]/factor
315 306 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
316 307
317 308 noise = dataOut.noise/factor
318 309
319 310 zdB = 10*numpy.log10(z)
320 311 noisedB = 10*numpy.log10(noise)
321 312
322 313
323 314 #thisDatetime = dataOut.datatime
324 315 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
325 316 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
326 317 xlabel = "Velocity (m/s)"
327 318 ylabel = "Range (Km)"
328 319
329 320 if not self.isConfig:
330 321
331 322 nplots = len(pairsIndexList)
332 323
333 324 self.setup(id=id,
334 325 nplots=nplots,
335 326 wintitle=wintitle,
336 327 showprofile=False,
337 328 show=show)
338 329
339 330 avg = numpy.abs(numpy.average(z, axis=1))
340 331 avgdB = 10*numpy.log10(avg)
341 332
342 333 if xmin == None: xmin = numpy.nanmin(x)
343 334 if xmax == None: xmax = numpy.nanmax(x)
344 335 if ymin == None: ymin = numpy.nanmin(y)
345 336 if ymax == None: ymax = numpy.nanmax(y)
346 337 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
347 338 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
348 339
349 340 self.FTP_WEI = ftp_wei
350 341 self.EXP_CODE = exp_code
351 342 self.SUB_EXP_CODE = sub_exp_code
352 343 self.PLOT_POS = plot_pos
353 344
354 345 self.isConfig = True
355 346
356 347 self.setWinTitle(title)
357 348
358 349 for i in range(self.nplots):
359 350 pair = dataOut.pairsList[pairsIndexList[i]]
360 351 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
361 352 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[pair[0]], str_datetime)
362 353 zdB = 10.*numpy.log10(dataOut.data_spc[pair[0],:,:]/factor)
363 354 axes0 = self.axesList[i*self.__nsubplots]
364 355 axes0.pcolor(x, y, zdB,
365 356 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
366 357 xlabel=xlabel, ylabel=ylabel, title=title,
367 358 ticksize=9, colormap=power_cmap, cblabel='')
368 359
369 360 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[pair[1]], str_datetime)
370 361 zdB = 10.*numpy.log10(dataOut.data_spc[pair[1],:,:]/factor)
371 362 axes0 = self.axesList[i*self.__nsubplots+1]
372 363 axes0.pcolor(x, y, zdB,
373 364 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
374 365 xlabel=xlabel, ylabel=ylabel, title=title,
375 366 ticksize=9, colormap=power_cmap, cblabel='')
376 367
377 368 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[pair[0],:,:]*dataOut.data_spc[pair[1],:,:])
378 369 coherence = numpy.abs(coherenceComplex)
379 370 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
380 371 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
381 372
382 373 title = "Coherence %d%d" %(pair[0], pair[1])
383 374 axes0 = self.axesList[i*self.__nsubplots+2]
384 375 axes0.pcolor(x, y, coherence,
385 376 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=0, zmax=1,
386 377 xlabel=xlabel, ylabel=ylabel, title=title,
387 378 ticksize=9, colormap=coherence_cmap, cblabel='')
388 379
389 380 title = "Phase %d%d" %(pair[0], pair[1])
390 381 axes0 = self.axesList[i*self.__nsubplots+3]
391 382 axes0.pcolor(x, y, phase,
392 383 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=-180, zmax=180,
393 384 xlabel=xlabel, ylabel=ylabel, title=title,
394 385 ticksize=9, colormap=phase_cmap, cblabel='')
395 386
396 387
397 388
398 389 self.draw()
399 390
400 if save != '':
401
402 if figfile == None:
403 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
404 figfile = self.getFilename(name = str_datetime)
405
406 self.counter_imagwr += 1
407
408 if (self.counter_imagwr>=wr_period):
409 # store png plot to local folder
410 self.saveFigure(figpath, figfile)
411 self.counter_imagwr = 0
412
413 if ftp:
414 # store png plot to FTP server according to RT-Web format
415 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
416 ftp_filename = os.path.join(figpath, name)
417 self.saveFigure(figpath, ftp_filename)
391 self.save(figpath=figpath,
392 figfile=figfile,
393 save=save,
394 ftp=ftp,
395 wr_period=wr_period,
396 thisDatetime=thisDatetime)
418 397
419 398
420 399 class RTIPlot(Figure):
421 400
422 401 __isConfig = None
423 402 __nsubplots = None
424 403
425 404 WIDTHPROF = None
426 405 HEIGHTPROF = None
427 406 PREFIX = 'rti'
428 407
429 408 def __init__(self):
430 409
431 410 self.timerange = 2*60*60
432 411 self.__isConfig = False
433 412 self.__nsubplots = 1
434 413
435 414 self.WIDTH = 800
436 415 self.HEIGHT = 150
437 416 self.WIDTHPROF = 120
438 417 self.HEIGHTPROF = 0
439 418 self.counter_imagwr = 0
440 419
441 self.PLOT_CODE = 0
420 self.PLOT_CODE = RTI_CODE
421
442 422 self.FTP_WEI = None
443 423 self.EXP_CODE = None
444 424 self.SUB_EXP_CODE = None
445 425 self.PLOT_POS = None
446 426 self.tmin = None
447 427 self.tmax = None
448 428
449 429 self.xmin = None
450 430 self.xmax = None
451 431
452 432 self.figfile = None
453 433
454 434 def getSubplots(self):
455 435
456 436 ncol = 1
457 437 nrow = self.nplots
458 438
459 439 return nrow, ncol
460 440
461 441 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
462 442
463 443 self.__showprofile = showprofile
464 444 self.nplots = nplots
465 445
466 446 ncolspan = 1
467 447 colspan = 1
468 448 if showprofile:
469 449 ncolspan = 7
470 450 colspan = 6
471 451 self.__nsubplots = 2
472 452
473 453 self.createFigure(id = id,
474 454 wintitle = wintitle,
475 455 widthplot = self.WIDTH + self.WIDTHPROF,
476 456 heightplot = self.HEIGHT + self.HEIGHTPROF,
477 457 show=show)
478 458
479 459 nrow, ncol = self.getSubplots()
480 460
481 461 counter = 0
482 462 for y in range(nrow):
483 463 for x in range(ncol):
484 464
485 465 if counter >= self.nplots:
486 466 break
487 467
488 468 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
489 469
490 470 if showprofile:
491 471 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
492 472
493 473 counter += 1
494 474
495 475 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
496 476 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
497 477 timerange=None,
498 478 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
499 479 server=None, folder=None, username=None, password=None,
500 480 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
501 481
502 482 """
503 483
504 484 Input:
505 485 dataOut :
506 486 id :
507 487 wintitle :
508 488 channelList :
509 489 showProfile :
510 490 xmin : None,
511 491 xmax : None,
512 492 ymin : None,
513 493 ymax : None,
514 494 zmin : None,
515 495 zmax : None
516 496 """
517 497
518 498 if channelList == None:
519 499 channelIndexList = dataOut.channelIndexList
520 500 else:
521 501 channelIndexList = []
522 502 for channel in channelList:
523 503 if channel not in dataOut.channelList:
524 504 raise ValueError, "Channel %d is not in dataOut.channelList"
525 505 channelIndexList.append(dataOut.channelList.index(channel))
526 506
527 507 # if timerange != None:
528 508 # self.timerange = timerange
529 509
530 510 #tmin = None
531 511 #tmax = None
532 512 factor = dataOut.normFactor
533 513 x = dataOut.getTimeRange()
534 514 y = dataOut.getHeiRange()
535 515
536 516 z = dataOut.data_spc[channelIndexList,:,:]/factor
537 517 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
538 518 avg = numpy.average(z, axis=1)
539 519
540 520 avgdB = 10.*numpy.log10(avg)
541 521
542 522
543 523 # thisDatetime = dataOut.datatime
544 524 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
545 525 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
546 526 xlabel = ""
547 527 ylabel = "Range (Km)"
548 528
549 529 if not self.__isConfig:
550 530
551 531 nplots = len(channelIndexList)
552 532
553 533 self.setup(id=id,
554 534 nplots=nplots,
555 535 wintitle=wintitle,
556 536 showprofile=showprofile,
557 537 show=show)
558 538
559 539 if timerange != None:
560 540 self.timerange = timerange
561 541
562 542 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
563 543
564 544 noise = dataOut.noise/factor
565 545 noisedB = 10*numpy.log10(noise)
566 546
567 547 if ymin == None: ymin = numpy.nanmin(y)
568 548 if ymax == None: ymax = numpy.nanmax(y)
569 549 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
570 550 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
571 551
572 552 self.FTP_WEI = ftp_wei
573 553 self.EXP_CODE = exp_code
574 554 self.SUB_EXP_CODE = sub_exp_code
575 555 self.PLOT_POS = plot_pos
576 556
577 557 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
578 558 self.__isConfig = True
579 559 self.figfile = figfile
580 560
581 561 self.setWinTitle(title)
582 562
583 563 if ((self.xmax - x[1]) < (x[1]-x[0])):
584 564 x[1] = self.xmax
585 565
586 566 for i in range(self.nplots):
587 567 title = "Channel %d: %s" %(dataOut.channelList[i]+1, thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
588 568 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
589 569 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
590 570 axes = self.axesList[i*self.__nsubplots]
591 571 zdB = avgdB[i].reshape((1,-1))
592 572 axes.pcolorbuffer(x, y, zdB,
593 573 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
594 574 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
595 575 ticksize=9, cblabel='', cbsize="1%")
596 576
597 577 if self.__showprofile:
598 578 axes = self.axesList[i*self.__nsubplots +1]
599 579 axes.pline(avgdB[i], y,
600 580 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
601 581 xlabel='dB', ylabel='', title='',
602 582 ytick_visible=False,
603 583 grid='x')
604 584
605 585 self.draw()
606
607 if save:
608 586
609 if self.figfile == None:
610 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
611 self.figfile = self.getFilename(name = str_datetime)
612
613 self.counter_imagwr += 1
614
615 if (self.counter_imagwr>=wr_period):
616 # store png plot to local folder
617 self.saveFigure(figpath, self.figfile)
618 self.counter_imagwr = 0
619
620 if ftp:
621 # store png plot to FTP server according to RT-Web format
622 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
623 ftp_filename = os.path.join(figpath, name)
624 self.saveFigure(figpath, ftp_filename)
625
626 587 if x[1] >= self.axesList[0].xmax:
627 588 self.counter_imagwr = wr_period
628 589 self.__isConfig = False
629 590 self.figfile = None
591
592 self.save(figpath=figpath,
593 figfile=figfile,
594 save=save,
595 ftp=ftp,
596 wr_period=wr_period,
597 thisDatetime=thisDatetime,
598 update_figfile=False)
630 599
631 600 class CoherenceMap(Figure):
632 601 isConfig = None
633 602 __nsubplots = None
634 603
635 604 WIDTHPROF = None
636 605 HEIGHTPROF = None
637 606 PREFIX = 'cmap'
638 607
639 608 def __init__(self):
640 609 self.timerange = 2*60*60
641 610 self.isConfig = False
642 611 self.__nsubplots = 1
643 612
644 613 self.WIDTH = 800
645 614 self.HEIGHT = 150
646 615 self.WIDTHPROF = 120
647 616 self.HEIGHTPROF = 0
648 617 self.counter_imagwr = 0
649 618
650 self.PLOT_CODE = 3
619 self.PLOT_CODE = COH_CODE
620
651 621 self.FTP_WEI = None
652 622 self.EXP_CODE = None
653 623 self.SUB_EXP_CODE = None
654 624 self.PLOT_POS = None
655 625 self.counter_imagwr = 0
656 626
657 627 self.xmin = None
658 628 self.xmax = None
659 629
660 630 def getSubplots(self):
661 631 ncol = 1
662 632 nrow = self.nplots*2
663 633
664 634 return nrow, ncol
665 635
666 636 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
667 637 self.__showprofile = showprofile
668 638 self.nplots = nplots
669 639
670 640 ncolspan = 1
671 641 colspan = 1
672 642 if showprofile:
673 643 ncolspan = 7
674 644 colspan = 6
675 645 self.__nsubplots = 2
676 646
677 647 self.createFigure(id = id,
678 648 wintitle = wintitle,
679 649 widthplot = self.WIDTH + self.WIDTHPROF,
680 650 heightplot = self.HEIGHT + self.HEIGHTPROF,
681 651 show=True)
682 652
683 653 nrow, ncol = self.getSubplots()
684 654
685 655 for y in range(nrow):
686 656 for x in range(ncol):
687 657
688 658 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
689 659
690 660 if showprofile:
691 661 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
692 662
693 663 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
694 664 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
695 665 timerange=None,
696 666 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
697 667 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
698 668 server=None, folder=None, username=None, password=None,
699 669 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
700 670
701 671 if pairsList == None:
702 672 pairsIndexList = dataOut.pairsIndexList
703 673 else:
704 674 pairsIndexList = []
705 675 for pair in pairsList:
706 676 if pair not in dataOut.pairsList:
707 677 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
708 678 pairsIndexList.append(dataOut.pairsList.index(pair))
709 679
710 680 if pairsIndexList == []:
711 681 return
712 682
713 683 if len(pairsIndexList) > 4:
714 684 pairsIndexList = pairsIndexList[0:4]
715 685
716 686 # tmin = None
717 687 # tmax = None
718 688 x = dataOut.getTimeRange()
719 689 y = dataOut.getHeiRange()
720 690
721 691 #thisDatetime = dataOut.datatime
722 692 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
723 693 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
724 694 xlabel = ""
725 695 ylabel = "Range (Km)"
726 696
727 697 if not self.isConfig:
728 698 nplots = len(pairsIndexList)
729 699 self.setup(id=id,
730 700 nplots=nplots,
731 701 wintitle=wintitle,
732 702 showprofile=showprofile,
733 703 show=show)
734 704
735 705 if timerange != None:
736 706 self.timerange = timerange
737 707
738 708 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
739 709
740 710 if ymin == None: ymin = numpy.nanmin(y)
741 711 if ymax == None: ymax = numpy.nanmax(y)
742 712 if zmin == None: zmin = 0.
743 713 if zmax == None: zmax = 1.
744 714
745 715 self.FTP_WEI = ftp_wei
746 716 self.EXP_CODE = exp_code
747 717 self.SUB_EXP_CODE = sub_exp_code
748 718 self.PLOT_POS = plot_pos
749 719
750 720 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
751 721
752 722 self.isConfig = True
753 723
754 724 self.setWinTitle(title)
755 725
756 726 if ((self.xmax - x[1]) < (x[1]-x[0])):
757 727 x[1] = self.xmax
758 728
759 729 for i in range(self.nplots):
760 730
761 731 pair = dataOut.pairsList[pairsIndexList[i]]
762 732
763 733 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
764 734 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
765 735 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
766 736
767 737
768 738 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
769 739 coherence = numpy.abs(avgcoherenceComplex)
770 740
771 741 z = coherence.reshape((1,-1))
772 742
773 743 counter = 0
774 744
775 745 title = "Coherence %d%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
776 746 axes = self.axesList[i*self.__nsubplots*2]
777 747 axes.pcolorbuffer(x, y, z,
778 748 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
779 749 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
780 750 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
781 751
782 752 if self.__showprofile:
783 753 counter += 1
784 754 axes = self.axesList[i*self.__nsubplots*2 + counter]
785 755 axes.pline(coherence, y,
786 756 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
787 757 xlabel='', ylabel='', title='', ticksize=7,
788 758 ytick_visible=False, nxticks=5,
789 759 grid='x')
790 760
791 761 counter += 1
792 762
793 763 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
794 764
795 765 z = phase.reshape((1,-1))
796 766
797 767 title = "Phase %d%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
798 768 axes = self.axesList[i*self.__nsubplots*2 + counter]
799 769 axes.pcolorbuffer(x, y, z,
800 770 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=-180, zmax=180,
801 771 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
802 772 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
803 773
804 774 if self.__showprofile:
805 775 counter += 1
806 776 axes = self.axesList[i*self.__nsubplots*2 + counter]
807 777 axes.pline(phase, y,
808 778 xmin=-180, xmax=180, ymin=ymin, ymax=ymax,
809 779 xlabel='', ylabel='', title='', ticksize=7,
810 780 ytick_visible=False, nxticks=4,
811 781 grid='x')
812 782
813 783 self.draw()
814 784
815 785 if x[1] >= self.axesList[0].xmax:
816 786 self.counter_imagwr = wr_period
817 787 self.__isConfig = False
818
819 if save:
820
821 if figfile == None:
822 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
823 figfile = self.getFilename(name = str_datetime)
824
825 self.counter_imagwr += 1
788 self.figfile = None
826 789
827 if (self.counter_imagwr>=wr_period):
828 # store png plot to local folder
829 self.saveFigure(figpath, figfile)
830 self.counter_imagwr = 0
831
832 if ftp:
833 # store png plot to FTP server according to RT-Web format
834 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
835 ftp_filename = os.path.join(figpath, name)
836 self.saveFigure(figpath, ftp_filename)
790 self.save(figpath=figpath,
791 figfile=figfile,
792 save=save,
793 ftp=ftp,
794 wr_period=wr_period,
795 thisDatetime=thisDatetime,
796 update_figfile=False)
837 797
838 798 class PowerProfile(Figure):
799
839 800 isConfig = None
840 801 __nsubplots = None
841 802
842 803 WIDTHPROF = None
843 804 HEIGHTPROF = None
844 805 PREFIX = 'spcprofile'
845 806
846 807 def __init__(self):
847 808 self.isConfig = False
848 809 self.__nsubplots = 1
849 810
811 self.PLOT_CODE = POWER_CODE
812
850 813 self.WIDTH = 300
851 814 self.HEIGHT = 500
852 815 self.counter_imagwr = 0
853 816
854 817 def getSubplots(self):
855 818 ncol = 1
856 819 nrow = 1
857 820
858 821 return nrow, ncol
859 822
860 823 def setup(self, id, nplots, wintitle, show):
861 824
862 825 self.nplots = nplots
863 826
864 827 ncolspan = 1
865 828 colspan = 1
866 829
867 830 self.createFigure(id = id,
868 831 wintitle = wintitle,
869 832 widthplot = self.WIDTH,
870 833 heightplot = self.HEIGHT,
871 834 show=show)
872 835
873 836 nrow, ncol = self.getSubplots()
874 837
875 838 counter = 0
876 839 for y in range(nrow):
877 840 for x in range(ncol):
878 841 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
879 842
880 843 def run(self, dataOut, id, wintitle="", channelList=None,
881 844 xmin=None, xmax=None, ymin=None, ymax=None,
882 845 save=False, figpath='./', figfile=None, show=True,
883 846 ftp=False, wr_period=1, server=None,
884 847 folder=None, username=None, password=None):
885 848
886 849
887 850 if channelList == None:
888 851 channelIndexList = dataOut.channelIndexList
889 852 channelList = dataOut.channelList
890 853 else:
891 854 channelIndexList = []
892 855 for channel in channelList:
893 856 if channel not in dataOut.channelList:
894 857 raise ValueError, "Channel %d is not in dataOut.channelList"
895 858 channelIndexList.append(dataOut.channelList.index(channel))
896 859
897 860 factor = dataOut.normFactor
898 861
899 862 y = dataOut.getHeiRange()
900 863
901 864 #for voltage
902 865 if dataOut.type == 'Voltage':
903 866 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
904 867 x = x.real
905 868 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
906 869
907 870 #for spectra
908 871 if dataOut.type == 'Spectra':
909 872 x = dataOut.data_spc[channelIndexList,:,:]/factor
910 873 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
911 874 x = numpy.average(x, axis=1)
912 875
913 876
914 877 xdB = 10*numpy.log10(x)
915 878
916 879 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
917 880 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
918 881 xlabel = "dB"
919 882 ylabel = "Range (Km)"
920 883
921 884 if not self.isConfig:
922 885
923 886 nplots = 1
924 887
925 888 self.setup(id=id,
926 889 nplots=nplots,
927 890 wintitle=wintitle,
928 891 show=show)
929 892
930 893 if ymin == None: ymin = numpy.nanmin(y)
931 894 if ymax == None: ymax = numpy.nanmax(y)
932 895 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
933 896 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
934 897
935 898 self.__isConfig = True
936 899
937 900 self.setWinTitle(title)
938 901
939 902 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
940 903 axes = self.axesList[0]
941 904
942 905 legendlabels = ["channel %d"%x for x in channelList]
943 906 axes.pmultiline(xdB, y,
944 907 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
945 908 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
946 909 ytick_visible=True, nxticks=5,
947 910 grid='x')
948 911
949 912 self.draw()
950 913
951 if save:
952
953 if figfile == None:
954 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
955 figfile = self.getFilename(name = str_datetime)
956
957 self.counter_imagwr += 1
958
959 if (self.counter_imagwr>=wr_period):
960 # store png plot to local folder
961 self.saveFigure(figpath, figfile)
962 self.counter_imagwr = 0
914 self.save(figpath=figpath,
915 figfile=figfile,
916 save=save,
917 ftp=ftp,
918 wr_period=wr_period,
919 thisDatetime=thisDatetime)
963 920
964 921 class Noise(Figure):
965 922
966 923 isConfig = None
967 924 __nsubplots = None
968 925
969 926 PREFIX = 'noise'
970 927
971 928 def __init__(self):
972 929
973 930 self.timerange = 24*60*60
974 931 self.isConfig = False
975 932 self.__nsubplots = 1
976 933 self.counter_imagwr = 0
977 934 self.WIDTH = 600
978 935 self.HEIGHT = 300
979 936 self.WIDTHPROF = 120
980 937 self.HEIGHTPROF = 0
981 938 self.xdata = None
982 939 self.ydata = None
983 940
984 self.PLOT_CODE = 17
941 self.PLOT_CODE = NOISE_CODE
942
985 943 self.FTP_WEI = None
986 944 self.EXP_CODE = None
987 945 self.SUB_EXP_CODE = None
988 946 self.PLOT_POS = None
989 947 self.figfile = None
990 948
991 949 self.xmin = None
992 950 self.xmax = None
993 951
994 952 def getSubplots(self):
995 953
996 954 ncol = 1
997 955 nrow = 1
998 956
999 957 return nrow, ncol
1000 958
1001 959 def openfile(self, filename):
1002 960 f = open(filename,'w+')
1003 961 f.write('\n\n')
1004 962 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1005 963 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1006 964 f.close()
1007 965
1008 966 def save_data(self, filename_phase, data, data_datetime):
1009 967 f=open(filename_phase,'a')
1010 968 timetuple_data = data_datetime.timetuple()
1011 969 day = str(timetuple_data.tm_mday)
1012 970 month = str(timetuple_data.tm_mon)
1013 971 year = str(timetuple_data.tm_year)
1014 972 hour = str(timetuple_data.tm_hour)
1015 973 minute = str(timetuple_data.tm_min)
1016 974 second = str(timetuple_data.tm_sec)
1017 975 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1018 976 f.close()
1019 977
1020 978
1021 979 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1022 980
1023 981 self.__showprofile = showprofile
1024 982 self.nplots = nplots
1025 983
1026 984 ncolspan = 7
1027 985 colspan = 6
1028 986 self.__nsubplots = 2
1029 987
1030 988 self.createFigure(id = id,
1031 989 wintitle = wintitle,
1032 990 widthplot = self.WIDTH+self.WIDTHPROF,
1033 991 heightplot = self.HEIGHT+self.HEIGHTPROF,
1034 992 show=show)
1035 993
1036 994 nrow, ncol = self.getSubplots()
1037 995
1038 996 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1039 997
1040 998
1041 999 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1042 1000 xmin=None, xmax=None, ymin=None, ymax=None,
1043 1001 timerange=None,
1044 1002 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1045 1003 server=None, folder=None, username=None, password=None,
1046 1004 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1047 1005
1048 1006 if channelList == None:
1049 1007 channelIndexList = dataOut.channelIndexList
1050 1008 channelList = dataOut.channelList
1051 1009 else:
1052 1010 channelIndexList = []
1053 1011 for channel in channelList:
1054 1012 if channel not in dataOut.channelList:
1055 1013 raise ValueError, "Channel %d is not in dataOut.channelList"
1056 1014 channelIndexList.append(dataOut.channelList.index(channel))
1057 1015
1058 1016 x = dataOut.getTimeRange()
1059 1017 #y = dataOut.getHeiRange()
1060 1018 factor = dataOut.normFactor
1061 1019 noise = dataOut.noise/factor
1062 1020 noisedB = 10*numpy.log10(noise)
1063 1021
1064 1022 #thisDatetime = dataOut.datatime
1065 1023 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1066 1024 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1067 1025 xlabel = ""
1068 1026 ylabel = "Intensity (dB)"
1069 1027
1070 1028 if not self.isConfig:
1071 1029
1072 1030 nplots = 1
1073 1031
1074 1032 self.setup(id=id,
1075 1033 nplots=nplots,
1076 1034 wintitle=wintitle,
1077 1035 showprofile=showprofile,
1078 1036 show=show)
1079 1037
1080 1038 if timerange != None:
1081 1039 self.timerange = timerange
1082 1040
1083 1041 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1084 1042
1085 1043 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1086 1044 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1087 1045
1088 1046 self.FTP_WEI = ftp_wei
1089 1047 self.EXP_CODE = exp_code
1090 1048 self.SUB_EXP_CODE = sub_exp_code
1091 1049 self.PLOT_POS = plot_pos
1092 1050
1093 1051
1094 1052 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1095 1053 self.isConfig = True
1096 1054 self.figfile = figfile
1097 1055 self.xdata = numpy.array([])
1098 1056 self.ydata = numpy.array([])
1099 1057
1100 1058 #open file beacon phase
1101 1059 path = '%s%03d' %(self.PREFIX, self.id)
1102 1060 noise_file = os.path.join(path,'%s.txt'%self.name)
1103 1061 self.filename_noise = os.path.join(figpath,noise_file)
1104 1062 self.openfile(self.filename_noise)
1105 1063
1106 1064
1107 1065 #store data beacon phase
1108 1066 self.save_data(self.filename_noise, noisedB, thisDatetime)
1109 1067
1110 1068
1111 1069 self.setWinTitle(title)
1112 1070
1113 1071
1114 1072 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1115 1073
1116 1074 legendlabels = ["channel %d"%(idchannel+1) for idchannel in channelList]
1117 1075 axes = self.axesList[0]
1118 1076
1119 1077 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1120 1078
1121 1079 if len(self.ydata)==0:
1122 1080 self.ydata = noisedB[channelIndexList].reshape(-1,1)
1123 1081 else:
1124 1082 self.ydata = numpy.hstack((self.ydata, noisedB[channelIndexList].reshape(-1,1)))
1125 1083
1126 1084
1127 1085 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1128 1086 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1129 1087 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1130 1088 XAxisAsTime=True, grid='both'
1131 1089 )
1132 1090
1133 1091 self.draw()
1134 1092
1135 1093 if x[1] >= self.axesList[0].xmax:
1136 1094 self.counter_imagwr = wr_period
1137 1095 del self.xdata
1138 1096 del self.ydata
1139 1097 self.__isConfig = False
1140
1141 if save != '':
1142
1143 if self.figfile == None:
1144 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1145 self.figfile = self.getFilename(name = str_datetime)
1146
1147 self.counter_imagwr += 1
1098 self.figfile = None
1148 1099
1149 if (self.counter_imagwr>=wr_period):
1150 # store png plot to local folder
1151 self.saveFigure(figpath, self.figfile)
1152 self.counter_imagwr = 0
1153
1154 if ftp:
1155 # store png plot to FTP server according to RT-Web format
1156 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
1157 ftp_filename = os.path.join(figpath, name)
1158 self.saveFigure(figpath, ftp_filename)
1100 self.save(figpath=figpath,
1101 figfile=figfile,
1102 save=save,
1103 ftp=ftp,
1104 wr_period=wr_period,
1105 thisDatetime=thisDatetime,
1106 update_figfile=False)
1159 1107
1160 1108
1161 1109 class BeaconPhase(Figure):
1162 1110
1163 1111 __isConfig = None
1164 1112 __nsubplots = None
1165 1113
1166 1114 PREFIX = 'beacon_phase'
1167 1115
1168 1116 def __init__(self):
1169 1117
1170 1118 self.timerange = 24*60*60
1171 1119 self.__isConfig = False
1172 1120 self.__nsubplots = 1
1173 1121 self.counter_imagwr = 0
1174 1122 self.WIDTH = 600
1175 1123 self.HEIGHT = 300
1176 1124 self.WIDTHPROF = 120
1177 1125 self.HEIGHTPROF = 0
1178 1126 self.xdata = None
1179 1127 self.ydata = None
1180 1128
1181 self.PLOT_CODE = 18
1129 self.PLOT_CODE = BEACON_CODE
1130
1182 1131 self.FTP_WEI = None
1183 1132 self.EXP_CODE = None
1184 1133 self.SUB_EXP_CODE = None
1185 1134 self.PLOT_POS = None
1186 1135
1187 1136 self.filename_phase = None
1188 1137
1189 1138 self.figfile = None
1190 1139
1191 1140 self.xmin = None
1192 1141 self.xmax = None
1193 1142
1194 1143 def getSubplots(self):
1195 1144
1196 1145 ncol = 1
1197 1146 nrow = 1
1198 1147
1199 1148 return nrow, ncol
1200 1149
1201 1150 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1202 1151
1203 1152 self.__showprofile = showprofile
1204 1153 self.nplots = nplots
1205 1154
1206 1155 ncolspan = 7
1207 1156 colspan = 6
1208 1157 self.__nsubplots = 2
1209 1158
1210 1159 self.createFigure(id = id,
1211 1160 wintitle = wintitle,
1212 1161 widthplot = self.WIDTH+self.WIDTHPROF,
1213 1162 heightplot = self.HEIGHT+self.HEIGHTPROF,
1214 1163 show=show)
1215 1164
1216 1165 nrow, ncol = self.getSubplots()
1217 1166
1218 1167 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1219 1168
1220 1169 def save_phase(self, filename_phase):
1221 1170 f = open(filename_phase,'w+')
1222 1171 f.write('\n\n')
1223 1172 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1224 1173 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1225 1174 f.close()
1226 1175
1227 1176 def save_data(self, filename_phase, data, data_datetime):
1228 1177 f=open(filename_phase,'a')
1229 1178 timetuple_data = data_datetime.timetuple()
1230 1179 day = str(timetuple_data.tm_mday)
1231 1180 month = str(timetuple_data.tm_mon)
1232 1181 year = str(timetuple_data.tm_year)
1233 1182 hour = str(timetuple_data.tm_hour)
1234 1183 minute = str(timetuple_data.tm_min)
1235 1184 second = str(timetuple_data.tm_sec)
1236 1185 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1237 1186 f.close()
1238 1187
1239 1188
1240 1189 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1241 1190 xmin=None, xmax=None, ymin=None, ymax=None,
1242 1191 timerange=None,
1243 1192 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1244 1193 server=None, folder=None, username=None, password=None,
1245 1194 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1246 1195
1247 1196 if pairsList == None:
1248 1197 pairsIndexList = dataOut.pairsIndexList
1249 1198 else:
1250 1199 pairsIndexList = []
1251 1200 for pair in pairsList:
1252 1201 if pair not in dataOut.pairsList:
1253 1202 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
1254 1203 pairsIndexList.append(dataOut.pairsList.index(pair))
1255 1204
1256 1205 if pairsIndexList == []:
1257 1206 return
1258 1207
1259 1208 # if len(pairsIndexList) > 4:
1260 1209 # pairsIndexList = pairsIndexList[0:4]
1261 1210
1262 1211 x = dataOut.getTimeRange()
1263 1212 #y = dataOut.getHeiRange()
1264 1213
1265 1214
1266 1215 #thisDatetime = dataOut.datatime
1267 1216 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1268 1217 title = wintitle + " Phase of Beacon Signal" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1269 1218 xlabel = "Local Time"
1270 1219 ylabel = "Phase"
1271 1220
1272 1221 nplots = len(pairsIndexList)
1273 1222 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1274 1223 phase_beacon = numpy.zeros(len(pairsIndexList))
1275 1224 for i in range(nplots):
1276 1225 pair = dataOut.pairsList[pairsIndexList[i]]
1277 1226 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
1278 1227 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
1279 1228 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
1280 1229 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1281 1230 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1282 1231
1283 1232 #print "Phase %d%d" %(pair[0], pair[1])
1284 1233 #print phase[dataOut.beacon_heiIndexList]
1285 1234
1286 1235 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1287 1236
1288 1237 if not self.__isConfig:
1289 1238
1290 1239 nplots = len(pairsIndexList)
1291 1240
1292 1241 self.setup(id=id,
1293 1242 nplots=nplots,
1294 1243 wintitle=wintitle,
1295 1244 showprofile=showprofile,
1296 1245 show=show)
1297 1246
1298 1247 if timerange != None:
1299 1248 self.timerange = timerange
1300 1249
1301 1250 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1302 1251
1303 1252 if ymin == None: ymin = numpy.nanmin(phase_beacon) - 10.0
1304 1253 if ymax == None: ymax = numpy.nanmax(phase_beacon) + 10.0
1305 1254
1306 1255 self.FTP_WEI = ftp_wei
1307 1256 self.EXP_CODE = exp_code
1308 1257 self.SUB_EXP_CODE = sub_exp_code
1309 1258 self.PLOT_POS = plot_pos
1310 1259
1311 1260 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1312 1261 self.__isConfig = True
1313 1262 self.figfile = figfile
1314 1263 self.xdata = numpy.array([])
1315 1264 self.ydata = numpy.array([])
1316 1265
1317 1266 #open file beacon phase
1318 1267 path = '%s%03d' %(self.PREFIX, self.id)
1319 1268 beacon_file = os.path.join(path,'%s.txt'%self.name)
1320 1269 self.filename_phase = os.path.join(figpath,beacon_file)
1321 1270 #self.save_phase(self.filename_phase)
1322 1271
1323 1272
1324 1273 #store data beacon phase
1325 1274 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1326 1275
1327 1276 self.setWinTitle(title)
1328 1277
1329 1278
1330 1279 title = "Beacon Signal %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1331 1280
1332 1281 legendlabels = ["pairs %d%d"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1333 1282
1334 1283 axes = self.axesList[0]
1335 1284
1336 1285 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1337 1286
1338 1287 if len(self.ydata)==0:
1339 1288 self.ydata = phase_beacon.reshape(-1,1)
1340 1289 else:
1341 1290 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1342 1291
1343 1292
1344 1293 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1345 1294 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1346 1295 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1347 1296 XAxisAsTime=True, grid='both'
1348 1297 )
1349 1298
1350 1299 self.draw()
1351 1300
1352 1301 if x[1] >= self.axesList[0].xmax:
1353 1302 self.counter_imagwr = wr_period
1354 1303 del self.xdata
1355 1304 del self.ydata
1356 1305 self.__isConfig = False
1357
1358 if save:
1359
1360 if self.figfile == None:
1361 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
1362 self.figfile = self.getFilename(name = str_datetime)
1363
1364 self.counter_imagwr += 1
1306 self.figfile = None
1365 1307
1366 if (self.counter_imagwr>=wr_period):
1367 # store png plot to local folder
1368 self.saveFigure(figpath, self.figfile)
1369 self.counter_imagwr = 0
1370
1371 if ftp:
1372 # store png plot to FTP server according to RT-Web format
1373 name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
1374 ftp_filename = os.path.join(figpath, name)
1375 self.saveFigure(figpath, ftp_filename)
1308 self.save(figpath=figpath,
1309 figfile=figfile,
1310 save=save,
1311 ftp=ftp,
1312 wr_period=wr_period,
1313 thisDatetime=thisDatetime,
1314 update_figfile=False)
Show file before | Show file after | Hide comments
@@ -1,186 +1,180
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from figure import Figure
11 11
12 12 class Scope(Figure):
13 13
14 14 isConfig = None
15 15
16 16 def __init__(self):
17 17
18 18 self.isConfig = False
19 19 self.WIDTH = 300
20 20 self.HEIGHT = 200
21 21 self.counter_imagwr = 0
22 22
23 23 def getSubplots(self):
24 24
25 25 nrow = self.nplots
26 26 ncol = 3
27 27 return nrow, ncol
28 28
29 29 def setup(self, id, nplots, wintitle, show):
30 30
31 31 self.nplots = nplots
32 32
33 33 self.createFigure(id=id,
34 34 wintitle=wintitle,
35 35 show=show)
36 36
37 37 nrow,ncol = self.getSubplots()
38 38 colspan = 3
39 39 rowspan = 1
40 40
41 41 for i in range(nplots):
42 42 self.addAxes(nrow, ncol, i, 0, colspan, rowspan)
43 43
44 44 def plot_iq(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
45 45 yreal = y[channelIndexList,:].real
46 46 yimag = y[channelIndexList,:].imag
47 47
48 48 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
49 49 xlabel = "Range (Km)"
50 50 ylabel = "Intensity - IQ"
51 51
52 52 if not self.isConfig:
53 53 nplots = len(channelIndexList)
54 54
55 55 self.setup(id=id,
56 56 nplots=nplots,
57 57 wintitle='',
58 58 show=show)
59 59
60 60 if xmin == None: xmin = numpy.nanmin(x)
61 61 if xmax == None: xmax = numpy.nanmax(x)
62 62 if ymin == None: ymin = min(numpy.nanmin(yreal),numpy.nanmin(yimag))
63 63 if ymax == None: ymax = max(numpy.nanmax(yreal),numpy.nanmax(yimag))
64 64
65 65 self.isConfig = True
66 66
67 67 self.setWinTitle(title)
68 68
69 69 for i in range(len(self.axesList)):
70 70 title = "Channel %d" %(i)
71 71 axes = self.axesList[i]
72 72
73 73 axes.pline(x, yreal[i,:],
74 74 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
75 75 xlabel=xlabel, ylabel=ylabel, title=title)
76 76
77 77 axes.addpline(x, yimag[i,:], idline=1, color="red", linestyle="solid", lw=2)
78 78
79 79 def plot_power(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
80 80 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
81 81 yreal = y.real
82 82
83 83 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
84 84 xlabel = "Range (Km)"
85 85 ylabel = "Intensity"
86 86
87 87 if not self.isConfig:
88 88 nplots = len(channelIndexList)
89 89
90 90 self.setup(id=id,
91 91 nplots=nplots,
92 92 wintitle='',
93 93 show=show)
94 94
95 95 if xmin == None: xmin = numpy.nanmin(x)
96 96 if xmax == None: xmax = numpy.nanmax(x)
97 97 if ymin == None: ymin = numpy.nanmin(yreal)
98 98 if ymax == None: ymax = numpy.nanmax(yreal)
99 99
100 100 self.isConfig = True
101 101
102 102 self.setWinTitle(title)
103 103
104 104 for i in range(len(self.axesList)):
105 105 title = "Channel %d" %(i)
106 106 axes = self.axesList[i]
107 107 ychannel = yreal[i,:]
108 108 axes.pline(x, ychannel,
109 109 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
110 110 xlabel=xlabel, ylabel=ylabel, title=title)
111 111
112 112
113 113 def run(self, dataOut, id, wintitle="", channelList=None,
114 114 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
115 115 figpath='./', figfile=None, show=True, wr_period=1,
116 116 ftp=False, server=None, folder=None, username=None, password=None, type='power'):
117 117
118 118 """
119 119
120 120 Input:
121 121 dataOut :
122 122 id :
123 123 wintitle :
124 124 channelList :
125 125 xmin : None,
126 126 xmax : None,
127 127 ymin : None,
128 128 ymax : None,
129 129 """
130 130
131 131 if channelList == None:
132 132 channelIndexList = dataOut.channelIndexList
133 133 else:
134 134 channelIndexList = []
135 135 for channel in channelList:
136 136 if channel not in dataOut.channelList:
137 137 raise ValueError, "Channel %d is not in dataOut.channelList"
138 138 channelIndexList.append(dataOut.channelList.index(channel))
139 139
140 140 x = dataOut.heightList
141 141 y = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
142 142 y = y.real
143 143
144 144 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
145 145
146 146 if type == "power":
147 147 self.plot_power(dataOut.heightList,
148 148 dataOut.data,
149 149 id,
150 150 channelIndexList,
151 151 thisDatetime,
152 152 wintitle,
153 153 show,
154 154 xmin,
155 155 xmax,
156 156 ymin,
157 157 ymax)
158 158
159 159 if type == "iq":
160 160 self.plot_iq(dataOut.heightList,
161 161 dataOut.data,
162 162 id,
163 163 channelIndexList,
164 164 thisDatetime,
165 165 wintitle,
166 166 show,
167 167 xmin,
168 168 xmax,
169 169 ymin,
170 170 ymax)
171 171
172 172
173 173 self.draw()
174 174
175 if save:
176 date = thisDatetime.strftime("%Y%m%d_%H%M%S")
177 if figfile == None:
178 figfile = self.getFilename(name = date)
179
180 self.saveFigure(figpath, figfile)
181
182 self.counter_imagwr += 1
183 if (ftp and (self.counter_imagwr==wr_period)):
184 ftp_filename = os.path.join(figpath,figfile)
185 self.sendByFTP_Thread(ftp_filename, server, folder, username, password)
186 self.counter_imagwr = 0
175 self.save(figpath=figpath,
176 figfile=figfile,
177 save=save,
178 ftp=ftp,
179 wr_period=wr_period,
180 thisDatetime=thisDatetime)
Show file before | Show file after | Hide comments
@@ -1,1345 +1,1345
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22
23 23 LOCALTIME = True
24 24
25 25 def isNumber(cad):
26 26 """
27 27 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 28
29 29 Excepciones:
30 30 Si un determinado string no puede ser convertido a numero
31 31 Input:
32 32 str, string al cual se le analiza para determinar si convertible a un numero o no
33 33
34 34 Return:
35 35 True : si el string es uno numerico
36 36 False : no es un string numerico
37 37 """
38 38 try:
39 39 float( cad )
40 40 return True
41 41 except:
42 42 return False
43 43
44 44 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 45 """
46 46 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 47
48 48 Inputs:
49 49 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 50
51 51 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 52 segundos contados desde 01/01/1970.
53 53 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55
56 56 Return:
57 57 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 58 fecha especificado, de lo contrario retorna False.
59 59
60 60 Excepciones:
61 61 Si el archivo no existe o no puede ser abierto
62 62 Si la cabecera no puede ser leida.
63 63
64 64 """
65 65 basicHeaderObj = BasicHeader(LOCALTIME)
66 66
67 67 try:
68 68 fp = open(filename,'rb')
69 69 except IOError:
70 70 traceback.print_exc()
71 71 raise IOError, "The file %s can't be opened" %(filename)
72 72
73 73 sts = basicHeaderObj.read(fp)
74 74 fp.close()
75 75
76 76 if not(sts):
77 77 print "Skipping the file %s because it has not a valid header" %(filename)
78 78 return 0
79 79
80 80 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 81 return 0
82 82
83 83 return 1
84 84
85 85 def isFileinThisTime(filename, startTime, endTime):
86 86 """
87 87 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 88
89 89 Inputs:
90 90 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 91
92 92 startTime : tiempo inicial del rango seleccionado en formato datetime.time
93 93
94 94 endTime : tiempo final del rango seleccionado en formato datetime.time
95 95
96 96 Return:
97 97 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
98 98 fecha especificado, de lo contrario retorna False.
99 99
100 100 Excepciones:
101 101 Si el archivo no existe o no puede ser abierto
102 102 Si la cabecera no puede ser leida.
103 103
104 104 """
105 105
106 106
107 107 try:
108 108 fp = open(filename,'rb')
109 109 except IOError:
110 110 traceback.print_exc()
111 111 raise IOError, "The file %s can't be opened" %(filename)
112 112
113 113 basicHeaderObj = BasicHeader(LOCALTIME)
114 114 sts = basicHeaderObj.read(fp)
115 115 fp.close()
116 116
117 117 thisDatetime = basicHeaderObj.datatime
118 118 thisTime = thisDatetime.time()
119 119
120 120 if not(sts):
121 121 print "Skipping the file %s because it has not a valid header" %(filename)
122 122 return None
123 123
124 124 if not ((startTime <= thisTime) and (endTime > thisTime)):
125 125 return None
126 126
127 127 return thisDatetime
128 128
129 129 def getFileFromSet(path, ext, set):
130 130 validFilelist = []
131 131 fileList = os.listdir(path)
132 132
133 133 # 0 1234 567 89A BCDE
134 134 # H YYYY DDD SSS .ext
135 135
136 136 for thisFile in fileList:
137 137 try:
138 138 year = int(thisFile[1:5])
139 139 doy = int(thisFile[5:8])
140 140 except:
141 141 continue
142 142
143 143 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
144 144 continue
145 145
146 146 validFilelist.append(thisFile)
147 147
148 148 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
149 149
150 150 if len(myfile)!= 0:
151 151 return myfile[0]
152 152 else:
153 153 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
154 154 print 'the filename %s does not exist'%filename
155 155 print '...going to the last file: '
156 156
157 157 if validFilelist:
158 158 validFilelist = sorted( validFilelist, key=str.lower )
159 159 return validFilelist[-1]
160 160
161 161 return None
162 162
163 163 def getlastFileFromPath(path, ext):
164 164 """
165 165 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
166 166 al final de la depuracion devuelve el ultimo file de la lista que quedo.
167 167
168 168 Input:
169 169 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
170 170 ext : extension de los files contenidos en una carpeta
171 171
172 172 Return:
173 173 El ultimo file de una determinada carpeta, no se considera el path.
174 174 """
175 175 validFilelist = []
176 176 fileList = os.listdir(path)
177 177
178 178 # 0 1234 567 89A BCDE
179 179 # H YYYY DDD SSS .ext
180 180
181 181 for thisFile in fileList:
182 182
183 183 year = thisFile[1:5]
184 184 if not isNumber(year):
185 185 continue
186 186
187 187 doy = thisFile[5:8]
188 188 if not isNumber(doy):
189 189 continue
190 190
191 191 year = int(year)
192 192 doy = int(doy)
193 193
194 194 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
195 195 continue
196 196
197 197 validFilelist.append(thisFile)
198 198
199 199 if validFilelist:
200 200 validFilelist = sorted( validFilelist, key=str.lower )
201 201 return validFilelist[-1]
202 202
203 203 return None
204 204
205 205 def checkForRealPath(path, foldercounter, year, doy, set, ext):
206 206 """
207 207 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
208 208 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
209 209 el path exacto de un determinado file.
210 210
211 211 Example :
212 212 nombre correcto del file es .../.../D2009307/P2009307367.ext
213 213
214 214 Entonces la funcion prueba con las siguientes combinaciones
215 215 .../.../y2009307367.ext
216 216 .../.../Y2009307367.ext
217 217 .../.../x2009307/y2009307367.ext
218 218 .../.../x2009307/Y2009307367.ext
219 219 .../.../X2009307/y2009307367.ext
220 220 .../.../X2009307/Y2009307367.ext
221 221 siendo para este caso, la ultima combinacion de letras, identica al file buscado
222 222
223 223 Return:
224 224 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
225 225 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
226 226 para el filename
227 227 """
228 228 fullfilename = None
229 229 find_flag = False
230 230 filename = None
231 231
232 232 prefixDirList = [None,'d','D']
233 233 if ext.lower() == ".r": #voltage
234 234 prefixFileList = ['d','D']
235 235 elif ext.lower() == ".pdata": #spectra
236 236 prefixFileList = ['p','P']
237 237 else:
238 238 return None, filename
239 239
240 240 #barrido por las combinaciones posibles
241 241 for prefixDir in prefixDirList:
242 242 thispath = path
243 243 if prefixDir != None:
244 244 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
245 245 if foldercounter == 0:
246 246 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
247 247 else:
248 248 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
249 249 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
250 250 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
251 251 fullfilename = os.path.join( thispath, filename ) #formo el path completo
252 252
253 253 if os.path.exists( fullfilename ): #verifico que exista
254 254 find_flag = True
255 255 break
256 256 if find_flag:
257 257 break
258 258
259 259 if not(find_flag):
260 260 return None, filename
261 261
262 262 return fullfilename, filename
263 263
264 264 def isDoyFolder(folder):
265 265 try:
266 266 year = int(folder[1:5])
267 267 except:
268 268 return 0
269 269
270 270 try:
271 271 doy = int(folder[5:8])
272 272 except:
273 273 return 0
274 274
275 275 return 1
276 276
277 277 class JRODataIO:
278 278
279 279 c = 3E8
280 280
281 281 isConfig = False
282 282
283 283 basicHeaderObj = None
284 284
285 285 systemHeaderObj = None
286 286
287 287 radarControllerHeaderObj = None
288 288
289 289 processingHeaderObj = None
290 290
291 291 online = 0
292 292
293 293 dtype = None
294 294
295 295 pathList = []
296 296
297 297 filenameList = []
298 298
299 299 filename = None
300 300
301 301 ext = None
302 302
303 303 flagIsNewFile = 1
304 304
305 305 flagDiscontinuousBlock = 0
306 306
307 307 flagIsNewBlock = 0
308 308
309 309 fp = None
310 310
311 311 firstHeaderSize = 0
312 312
313 313 basicHeaderSize = 24
314 314
315 315 versionFile = 1103
316 316
317 317 fileSize = None
318 318
319 319 # ippSeconds = None
320 320
321 321 fileSizeByHeader = None
322 322
323 323 fileIndex = None
324 324
325 325 profileIndex = None
326 326
327 327 blockIndex = None
328 328
329 329 nTotalBlocks = None
330 330
331 331 maxTimeStep = 30
332 332
333 333 lastUTTime = None
334 334
335 335 datablock = None
336 336
337 337 dataOut = None
338 338
339 339 blocksize = None
340 340
341 341 getByBlock = False
342 342
343 343 def __init__(self):
344 344
345 345 raise ValueError, "Not implemented"
346 346
347 347 def run(self):
348 348
349 349 raise ValueError, "Not implemented"
350 350
351 351 class JRODataReader(JRODataIO):
352 352
353 353 nReadBlocks = 0
354 354
355 355 delay = 10 #number of seconds waiting a new file
356 356
357 357 nTries = 3 #quantity tries
358 358
359 359 nFiles = 3 #number of files for searching
360 360
361 361 path = None
362 362
363 363 foldercounter = 0
364 364
365 365 flagNoMoreFiles = 0
366 366
367 367 datetimeList = []
368 368
369 369 __isFirstTimeOnline = 1
370 370
371 371 __printInfo = True
372 372
373 373 profileIndex = None
374 374
375 375 nTxs = 1
376 376
377 377 txIndex = None
378 378
379 379 def __init__(self):
380 380
381 381 """
382 382
383 383 """
384 384
385 385 raise ValueError, "This method has not been implemented"
386 386
387 387
388 388 def createObjByDefault(self):
389 389 """
390 390
391 391 """
392 392 raise ValueError, "This method has not been implemented"
393 393
394 394 def getBlockDimension(self):
395 395
396 396 raise ValueError, "No implemented"
397 397
398 398 def __searchFilesOffLine(self,
399 399 path,
400 400 startDate,
401 401 endDate,
402 402 startTime=datetime.time(0,0,0),
403 403 endTime=datetime.time(23,59,59),
404 404 set=None,
405 405 expLabel='',
406 406 ext='.r',
407 407 walk=True):
408 408
409 409 pathList = []
410 410
411 411 if not walk:
412 412 #pathList.append(path)
413 413 multi_path = path.split(',')
414 414 for single_path in multi_path:
415 415 pathList.append(single_path)
416 416
417 417 else:
418 418 #dirList = []
419 419 multi_path = path.split(',')
420 420 for single_path in multi_path:
421 421 dirList = []
422 422 for thisPath in os.listdir(single_path):
423 423 if not os.path.isdir(os.path.join(single_path,thisPath)):
424 424 continue
425 425 if not isDoyFolder(thisPath):
426 426 continue
427 427
428 428 dirList.append(thisPath)
429 429
430 430 if not(dirList):
431 431 return None, None
432 432
433 433 thisDate = startDate
434 434
435 435 while(thisDate <= endDate):
436 436 year = thisDate.timetuple().tm_year
437 437 doy = thisDate.timetuple().tm_yday
438 438
439 439 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
440 440 if len(matchlist) == 0:
441 441 thisDate += datetime.timedelta(1)
442 442 continue
443 443 for match in matchlist:
444 444 pathList.append(os.path.join(single_path,match,expLabel))
445 445
446 446 thisDate += datetime.timedelta(1)
447 447
448 448 if pathList == []:
449 449 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
450 450 return None, None
451 451
452 452 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
453 453
454 454 filenameList = []
455 455 datetimeList = []
456 456 pathDict = {}
457 457 filenameList_to_sort = []
458 458
459 459 for i in range(len(pathList)):
460 460
461 461 thisPath = pathList[i]
462 462
463 463 fileList = glob.glob1(thisPath, "*%s" %ext)
464 464 if len(fileList) < 1:
465 465 continue
466 466 fileList.sort()
467 467 pathDict.setdefault(fileList[0])
468 468 pathDict[fileList[0]] = i
469 469 filenameList_to_sort.append(fileList[0])
470 470
471 471 filenameList_to_sort.sort()
472 472
473 473 for file in filenameList_to_sort:
474 474 thisPath = pathList[pathDict[file]]
475 475
476 476 fileList = glob.glob1(thisPath, "*%s" %ext)
477 477 fileList.sort()
478 478
479 479 for file in fileList:
480 480
481 481 filename = os.path.join(thisPath,file)
482 482 thisDatetime = isFileinThisTime(filename, startTime, endTime)
483 483
484 484 if not(thisDatetime):
485 485 continue
486 486
487 487 filenameList.append(filename)
488 488 datetimeList.append(thisDatetime)
489 489
490 490 if not(filenameList):
491 491 print "Any file was found for the time range %s - %s" %(startTime, endTime)
492 492 return None, None
493 493
494 494 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
495 495 print
496 496
497 497 for i in range(len(filenameList)):
498 498 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
499 499
500 500 self.filenameList = filenameList
501 501 self.datetimeList = datetimeList
502 502
503 503 return pathList, filenameList
504 504
505 505 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
506 506
507 507 """
508 508 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
509 509 devuelve el archivo encontrado ademas de otros datos.
510 510
511 511 Input:
512 512 path : carpeta donde estan contenidos los files que contiene data
513 513
514 514 expLabel : Nombre del subexperimento (subfolder)
515 515
516 516 ext : extension de los files
517 517
518 518 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
519 519
520 520 Return:
521 521 directory : eL directorio donde esta el file encontrado
522 522 filename : el ultimo file de una determinada carpeta
523 523 year : el anho
524 524 doy : el numero de dia del anho
525 525 set : el set del archivo
526 526
527 527
528 528 """
529 529 dirList = []
530 530
531 531 if not walk:
532 532 fullpath = path
533 533 foldercounter = 0
534 534 else:
535 535 #Filtra solo los directorios
536 536 for thisPath in os.listdir(path):
537 537 if not os.path.isdir(os.path.join(path,thisPath)):
538 538 continue
539 539 if not isDoyFolder(thisPath):
540 540 continue
541 541
542 542 dirList.append(thisPath)
543 543
544 544 if not(dirList):
545 545 return None, None, None, None, None, None
546 546
547 547 dirList = sorted( dirList, key=str.lower )
548 548
549 549 doypath = dirList[-1]
550 550 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
551 551 fullpath = os.path.join(path, doypath, expLabel)
552 552
553 553
554 554 print "%s folder was found: " %(fullpath )
555 555
556 556 if set == None:
557 557 filename = getlastFileFromPath(fullpath, ext)
558 558 else:
559 559 filename = getFileFromSet(fullpath, ext, set)
560 560
561 561 if not(filename):
562 562 return None, None, None, None, None, None
563 563
564 564 print "%s file was found" %(filename)
565 565
566 566 if not(self.__verifyFile(os.path.join(fullpath, filename))):
567 567 return None, None, None, None, None, None
568 568
569 569 year = int( filename[1:5] )
570 570 doy = int( filename[5:8] )
571 571 set = int( filename[8:11] )
572 572
573 573 return fullpath, foldercounter, filename, year, doy, set
574 574
575 575 def __setNextFileOffline(self):
576 576
577 577 idFile = self.fileIndex
578 578
579 579 while (True):
580 580 idFile += 1
581 581 if not(idFile < len(self.filenameList)):
582 582 self.flagNoMoreFiles = 1
583 583 print "No more Files"
584 584 return 0
585 585
586 586 filename = self.filenameList[idFile]
587 587
588 588 if not(self.__verifyFile(filename)):
589 589 continue
590 590
591 591 fileSize = os.path.getsize(filename)
592 592 fp = open(filename,'rb')
593 593 break
594 594
595 595 self.flagIsNewFile = 1
596 596 self.fileIndex = idFile
597 597 self.filename = filename
598 598 self.fileSize = fileSize
599 599 self.fp = fp
600 600
601 print "Setting the file: %s"%self.filename
601 print "[Reading] Setting the file: %s"%self.filename
602 602
603 603 return 1
604 604
605 605 def __setNextFileOnline(self):
606 606 """
607 607 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
608 608 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
609 609 siguientes.
610 610
611 611 Affected:
612 612 self.flagIsNewFile
613 613 self.filename
614 614 self.fileSize
615 615 self.fp
616 616 self.set
617 617 self.flagNoMoreFiles
618 618
619 619 Return:
620 620 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
621 621 1 : si el file fue abierto con exito y esta listo a ser leido
622 622
623 623 Excepciones:
624 624 Si un determinado file no puede ser abierto
625 625 """
626 626 nFiles = 0
627 627 fileOk_flag = False
628 628 firstTime_flag = True
629 629
630 630 self.set += 1
631 631
632 632 if self.set > 999:
633 633 self.set = 0
634 634 self.foldercounter += 1
635 635
636 636 #busca el 1er file disponible
637 637 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
638 638 if fullfilename:
639 639 if self.__verifyFile(fullfilename, False):
640 640 fileOk_flag = True
641 641
642 642 #si no encuentra un file entonces espera y vuelve a buscar
643 643 if not(fileOk_flag):
644 644 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
645 645
646 646 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
647 647 tries = self.nTries
648 648 else:
649 649 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
650 650
651 651 for nTries in range( tries ):
652 652 if firstTime_flag:
653 653 print "\tWaiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
654 654 sleep( self.delay )
655 655 else:
656 656 print "\tSearching next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
657 657
658 658 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
659 659 if fullfilename:
660 660 if self.__verifyFile(fullfilename):
661 661 fileOk_flag = True
662 662 break
663 663
664 664 if fileOk_flag:
665 665 break
666 666
667 667 firstTime_flag = False
668 668
669 669 print "\tSkipping the file \"%s\" due to this file doesn't exist" % filename
670 670 self.set += 1
671 671
672 672 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
673 673 self.set = 0
674 674 self.doy += 1
675 675 self.foldercounter = 0
676 676
677 677 if fileOk_flag:
678 678 self.fileSize = os.path.getsize( fullfilename )
679 679 self.filename = fullfilename
680 680 self.flagIsNewFile = 1
681 681 if self.fp != None: self.fp.close()
682 682 self.fp = open(fullfilename, 'rb')
683 683 self.flagNoMoreFiles = 0
684 print 'Setting the file: %s' % fullfilename
684 print '[Reading] Setting the file: %s' % fullfilename
685 685 else:
686 686 self.fileSize = 0
687 687 self.filename = None
688 688 self.flagIsNewFile = 0
689 689 self.fp = None
690 690 self.flagNoMoreFiles = 1
691 print 'No more Files'
691 print '[Reading] No more files to read'
692 692
693 693 return fileOk_flag
694 694
695 695 def setNextFile(self):
696 696 if self.fp != None:
697 697 self.fp.close()
698 698
699 699 if self.online:
700 700 newFile = self.__setNextFileOnline()
701 701 else:
702 702 newFile = self.__setNextFileOffline()
703 703
704 704 if not(newFile):
705 705 return 0
706 706
707 707 self.__readFirstHeader()
708 708 self.nReadBlocks = 0
709 709 return 1
710 710
711 711 def __waitNewBlock(self):
712 712 """
713 713 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
714 714
715 715 Si el modo de lectura es OffLine siempre retorn 0
716 716 """
717 717 if not self.online:
718 718 return 0
719 719
720 720 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
721 721 return 0
722 722
723 723 currentPointer = self.fp.tell()
724 724
725 725 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
726 726
727 727 for nTries in range( self.nTries ):
728 728
729 729 self.fp.close()
730 730 self.fp = open( self.filename, 'rb' )
731 731 self.fp.seek( currentPointer )
732 732
733 733 self.fileSize = os.path.getsize( self.filename )
734 734 currentSize = self.fileSize - currentPointer
735 735
736 736 if ( currentSize >= neededSize ):
737 737 self.basicHeaderObj.read(self.fp)
738 738 return 1
739 739
740 740 if self.fileSize == self.fileSizeByHeader:
741 741 # self.flagEoF = True
742 742 return 0
743 743
744 744 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
745 745 sleep( self.delay )
746 746
747 747
748 748 return 0
749 749
750 750 def waitDataBlock(self,pointer_location):
751 751
752 752 currentPointer = pointer_location
753 753
754 754 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
755 755
756 756 for nTries in range( self.nTries ):
757 757 self.fp.close()
758 758 self.fp = open( self.filename, 'rb' )
759 759 self.fp.seek( currentPointer )
760 760
761 761 self.fileSize = os.path.getsize( self.filename )
762 762 currentSize = self.fileSize - currentPointer
763 763
764 764 if ( currentSize >= neededSize ):
765 765 return 1
766 766
767 767 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
768 768 sleep( self.delay )
769 769
770 770 return 0
771 771
772 772 def __jumpToLastBlock(self):
773 773
774 774 if not(self.__isFirstTimeOnline):
775 775 return
776 776
777 777 csize = self.fileSize - self.fp.tell()
778 778 blocksize = self.processingHeaderObj.blockSize
779 779
780 780 #salta el primer bloque de datos
781 781 if csize > self.processingHeaderObj.blockSize:
782 782 self.fp.seek(self.fp.tell() + blocksize)
783 783 else:
784 784 return
785 785
786 786 csize = self.fileSize - self.fp.tell()
787 787 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
788 788 while True:
789 789
790 790 if self.fp.tell()<self.fileSize:
791 791 self.fp.seek(self.fp.tell() + neededsize)
792 792 else:
793 793 self.fp.seek(self.fp.tell() - neededsize)
794 794 break
795 795
796 796 # csize = self.fileSize - self.fp.tell()
797 797 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
798 798 # factor = int(csize/neededsize)
799 799 # if factor > 0:
800 800 # self.fp.seek(self.fp.tell() + factor*neededsize)
801 801
802 802 self.flagIsNewFile = 0
803 803 self.__isFirstTimeOnline = 0
804 804
805 805 def __setNewBlock(self):
806 806
807 807 if self.fp == None:
808 808 return 0
809 809
810 810 if self.online:
811 811 self.__jumpToLastBlock()
812 812
813 813 if self.flagIsNewFile:
814 814 return 1
815 815
816 816 self.lastUTTime = self.basicHeaderObj.utc
817 817 currentSize = self.fileSize - self.fp.tell()
818 818 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
819 819
820 820 if (currentSize >= neededSize):
821 821 self.basicHeaderObj.read(self.fp)
822 822 return 1
823 823
824 824 if self.__waitNewBlock():
825 825 return 1
826 826
827 827 if not(self.setNextFile()):
828 828 return 0
829 829
830 830 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
831 831
832 832 self.flagDiscontinuousBlock = 0
833 833
834 834 if deltaTime > self.maxTimeStep:
835 835 self.flagDiscontinuousBlock = 1
836 836
837 837 return 1
838 838
839 839 def readNextBlock(self):
840 840 if not(self.__setNewBlock()):
841 841 return 0
842 842
843 843 if not(self.readBlock()):
844 844 return 0
845 845
846 846 return 1
847 847
848 848 def __readFirstHeader(self):
849 849
850 850 self.basicHeaderObj.read(self.fp)
851 851 self.systemHeaderObj.read(self.fp)
852 852 self.radarControllerHeaderObj.read(self.fp)
853 853 self.processingHeaderObj.read(self.fp)
854 854
855 855 self.firstHeaderSize = self.basicHeaderObj.size
856 856
857 857 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
858 858 if datatype == 0:
859 859 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
860 860 elif datatype == 1:
861 861 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
862 862 elif datatype == 2:
863 863 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
864 864 elif datatype == 3:
865 865 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
866 866 elif datatype == 4:
867 867 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
868 868 elif datatype == 5:
869 869 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
870 870 else:
871 871 raise ValueError, 'Data type was not defined'
872 872
873 873 self.dtype = datatype_str
874 874 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
875 875 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
876 876 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
877 877 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
878 878 self.getBlockDimension()
879 879
880 880 def __verifyFile(self, filename, msgFlag=True):
881 881 msg = None
882 882 try:
883 883 fp = open(filename, 'rb')
884 884 currentPosition = fp.tell()
885 885 except IOError:
886 886 traceback.print_exc()
887 887 if msgFlag:
888 888 print "[Reading] The file %s can't be opened" % (filename)
889 889 return False
890 890
891 891 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
892 892
893 893 if neededSize == 0:
894 894 basicHeaderObj = BasicHeader(LOCALTIME)
895 895 systemHeaderObj = SystemHeader()
896 896 radarControllerHeaderObj = RadarControllerHeader()
897 897 processingHeaderObj = ProcessingHeader()
898 898
899 899 try:
900 900 if not( basicHeaderObj.read(fp) ): raise IOError
901 901 if not( systemHeaderObj.read(fp) ): raise IOError
902 902 if not( radarControllerHeaderObj.read(fp) ): raise IOError
903 903 if not( processingHeaderObj.read(fp) ): raise IOError
904 904 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
905 905
906 906 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
907 907
908 908 except IOError:
909 909 traceback.print_exc()
910 910 if msgFlag:
911 911 print "[Reading] The file %s is empty or it hasn't enough data" % filename
912 912
913 913 fp.close()
914 914 return False
915 915 else:
916 916 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
917 917
918 918 fp.close()
919 919 fileSize = os.path.getsize(filename)
920 920 currentSize = fileSize - currentPosition
921 921 if currentSize < neededSize:
922 922 if msgFlag and (msg != None):
923 923 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
924 924 return False
925 925
926 926 return True
927 927
928 928 def setup(self,
929 929 path=None,
930 930 startDate=None,
931 931 endDate=None,
932 932 startTime=datetime.time(0,0,0),
933 933 endTime=datetime.time(23,59,59),
934 934 set=None,
935 935 expLabel = "",
936 936 ext = None,
937 937 online = False,
938 938 delay = 60,
939 939 walk = True,
940 940 getblock = False,
941 941 nTxs = 1):
942 942
943 943 if path == None:
944 944 raise ValueError, "[Reading] The path is not valid"
945 945
946 946 if ext == None:
947 947 ext = self.ext
948 948
949 949 if online:
950 950 print "[Reading] Searching files in online mode..."
951 951
952 952 for nTries in range( self.nTries ):
953 953 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
954 954
955 955 if fullpath:
956 956 break
957 957
958 958 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
959 959 sleep( self.delay )
960 960
961 961 if not(fullpath):
962 print "There 'isn't any valid file in %s" % path
962 print "[Reading] There 'isn't any valid file in %s" % path
963 963 return None
964 964
965 965 self.year = year
966 966 self.doy = doy
967 967 self.set = set - 1
968 968 self.path = path
969 969 self.foldercounter = foldercounter
970 970 last_set = None
971 971
972 972 else:
973 973 print "[Reading] Searching files in offline mode ..."
974 974 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
975 975 startTime=startTime, endTime=endTime,
976 976 set=set, expLabel=expLabel, ext=ext,
977 977 walk=walk)
978 978
979 979 if not(pathList):
980 980 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
981 981 datetime.datetime.combine(startDate,startTime).ctime(),
982 982 datetime.datetime.combine(endDate,endTime).ctime())
983 983
984 984 sys.exit(-1)
985 985
986 986
987 987 self.fileIndex = -1
988 988 self.pathList = pathList
989 989 self.filenameList = filenameList
990 990 file_name = os.path.basename(filenameList[-1])
991 991 basename, ext = os.path.splitext(file_name)
992 992 last_set = int(basename[-3:])
993 993
994 994 self.online = online
995 995 self.delay = delay
996 996 ext = ext.lower()
997 997 self.ext = ext
998 998 self.getByBlock = getblock
999 999 self.nTxs = int(nTxs)
1000 1000
1001 1001 if not(self.setNextFile()):
1002 1002 if (startDate!=None) and (endDate!=None):
1003 1003 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1004 1004 elif startDate != None:
1005 1005 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1006 1006 else:
1007 1007 print "[Reading] No files"
1008 1008
1009 1009 sys.exit(-1)
1010 1010
1011 1011 # self.updateDataHeader()
1012 1012 if last_set != None:
1013 1013 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1014 1014 return
1015 1015
1016 1016 def getBasicHeader(self):
1017 1017
1018 1018 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1019 1019
1020 1020 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1021 1021
1022 1022 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1023 1023
1024 1024 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1025 1025
1026 1026 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1027 1027
1028 1028 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1029 1029
1030 1030 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1031 1031
1032 1032 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1033 1033
1034 1034
1035 1035 def getFirstHeader(self):
1036 1036
1037 1037 raise ValueError, "This method has not been implemented"
1038 1038
1039 1039 def getData(self):
1040 1040
1041 1041 raise ValueError, "This method has not been implemented"
1042 1042
1043 1043 def hasNotDataInBuffer(self):
1044 1044
1045 1045 raise ValueError, "This method has not been implemented"
1046 1046
1047 1047 def readBlock(self):
1048 1048
1049 1049 raise ValueError, "This method has not been implemented"
1050 1050
1051 1051 def isEndProcess(self):
1052 1052
1053 1053 return self.flagNoMoreFiles
1054 1054
1055 1055 def printReadBlocks(self):
1056 1056
1057 1057 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1058 1058
1059 1059 def printTotalBlocks(self):
1060 1060
1061 1061 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1062 1062
1063 1063 def printNumberOfBlock(self):
1064 1064
1065 1065 if self.flagIsNewBlock:
1066 1066 print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1067 1067 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1068 1068
1069 1069 def printInfo(self):
1070 1070
1071 1071 if self.__printInfo == False:
1072 1072 return
1073 1073
1074 1074 self.basicHeaderObj.printInfo()
1075 1075 self.systemHeaderObj.printInfo()
1076 1076 self.radarControllerHeaderObj.printInfo()
1077 1077 self.processingHeaderObj.printInfo()
1078 1078
1079 1079 self.__printInfo = False
1080 1080
1081 1081
1082 1082 def run(self, **kwargs):
1083 1083
1084 1084 if not(self.isConfig):
1085 1085
1086 1086 # self.dataOut = dataOut
1087 1087 self.setup(**kwargs)
1088 1088 self.isConfig = True
1089 1089
1090 1090 self.getData()
1091 1091
1092 1092 class JRODataWriter(JRODataIO):
1093 1093
1094 1094 """
1095 1095 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1096 1096 de los datos siempre se realiza por bloques.
1097 1097 """
1098 1098
1099 1099 blockIndex = 0
1100 1100
1101 1101 path = None
1102 1102
1103 1103 setFile = None
1104 1104
1105 1105 profilesPerBlock = None
1106 1106
1107 1107 blocksPerFile = None
1108 1108
1109 1109 nWriteBlocks = 0
1110 1110
1111 1111 def __init__(self, dataOut=None):
1112 1112 raise ValueError, "Not implemented"
1113 1113
1114 1114
1115 1115 def hasAllDataInBuffer(self):
1116 1116 raise ValueError, "Not implemented"
1117 1117
1118 1118
1119 1119 def setBlockDimension(self):
1120 1120 raise ValueError, "Not implemented"
1121 1121
1122 1122
1123 1123 def writeBlock(self):
1124 1124 raise ValueError, "No implemented"
1125 1125
1126 1126
1127 1127 def putData(self):
1128 1128 raise ValueError, "No implemented"
1129 1129
1130 1130
1131 1131 def setBasicHeader(self):
1132 1132
1133 1133 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1134 1134 self.basicHeaderObj.version = self.versionFile
1135 1135 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1136 1136
1137 1137 utc = numpy.floor(self.dataOut.utctime)
1138 1138 milisecond = (self.dataOut.utctime - utc)* 1000.0
1139 1139
1140 1140 self.basicHeaderObj.utc = utc
1141 1141 self.basicHeaderObj.miliSecond = milisecond
1142 1142 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1143 1143 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1144 1144 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1145 1145
1146 1146 def setFirstHeader(self):
1147 1147 """
1148 1148 Obtiene una copia del First Header
1149 1149
1150 1150 Affected:
1151 1151
1152 1152 self.basicHeaderObj
1153 1153 self.systemHeaderObj
1154 1154 self.radarControllerHeaderObj
1155 1155 self.processingHeaderObj self.
1156 1156
1157 1157 Return:
1158 1158 None
1159 1159 """
1160 1160
1161 1161 raise ValueError, "No implemented"
1162 1162
1163 1163 def __writeFirstHeader(self):
1164 1164 """
1165 1165 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1166 1166
1167 1167 Affected:
1168 1168 __dataType
1169 1169
1170 1170 Return:
1171 1171 None
1172 1172 """
1173 1173
1174 1174 # CALCULAR PARAMETROS
1175 1175
1176 1176 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1177 1177 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1178 1178
1179 1179 self.basicHeaderObj.write(self.fp)
1180 1180 self.systemHeaderObj.write(self.fp)
1181 1181 self.radarControllerHeaderObj.write(self.fp)
1182 1182 self.processingHeaderObj.write(self.fp)
1183 1183
1184 1184 self.dtype = self.dataOut.dtype
1185 1185
1186 1186 def __setNewBlock(self):
1187 1187 """
1188 1188 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1189 1189
1190 1190 Return:
1191 1191 0 : si no pudo escribir nada
1192 1192 1 : Si escribio el Basic el First Header
1193 1193 """
1194 1194 if self.fp == None:
1195 1195 self.setNextFile()
1196 1196
1197 1197 if self.flagIsNewFile:
1198 1198 return 1
1199 1199
1200 1200 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1201 1201 self.basicHeaderObj.write(self.fp)
1202 1202 return 1
1203 1203
1204 1204 if not( self.setNextFile() ):
1205 1205 return 0
1206 1206
1207 1207 return 1
1208 1208
1209 1209
1210 1210 def writeNextBlock(self):
1211 1211 """
1212 1212 Selecciona el bloque siguiente de datos y los escribe en un file
1213 1213
1214 1214 Return:
1215 1215 0 : Si no hizo pudo escribir el bloque de datos
1216 1216 1 : Si no pudo escribir el bloque de datos
1217 1217 """
1218 1218 if not( self.__setNewBlock() ):
1219 1219 return 0
1220 1220
1221 1221 self.writeBlock()
1222 1222
1223 1223 return 1
1224 1224
1225 1225 def setNextFile(self):
1226 1226 """
1227 1227 Determina el siguiente file que sera escrito
1228 1228
1229 1229 Affected:
1230 1230 self.filename
1231 1231 self.subfolder
1232 1232 self.fp
1233 1233 self.setFile
1234 1234 self.flagIsNewFile
1235 1235
1236 1236 Return:
1237 1237 0 : Si el archivo no puede ser escrito
1238 1238 1 : Si el archivo esta listo para ser escrito
1239 1239 """
1240 1240 ext = self.ext
1241 1241 path = self.path
1242 1242
1243 1243 if self.fp != None:
1244 1244 self.fp.close()
1245 1245
1246 1246 timeTuple = time.localtime( self.dataOut.utctime)
1247 1247 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1248 1248
1249 1249 fullpath = os.path.join( path, subfolder )
1250 1250 if not( os.path.exists(fullpath) ):
1251 1251 os.mkdir(fullpath)
1252 1252 self.setFile = -1 #inicializo mi contador de seteo
1253 1253 else:
1254 1254 filesList = os.listdir( fullpath )
1255 1255 if len( filesList ) > 0:
1256 1256 filesList = sorted( filesList, key=str.lower )
1257 1257 filen = filesList[-1]
1258 1258 # el filename debera tener el siguiente formato
1259 1259 # 0 1234 567 89A BCDE (hex)
1260 1260 # x YYYY DDD SSS .ext
1261 1261 if isNumber( filen[8:11] ):
1262 1262 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1263 1263 else:
1264 1264 self.setFile = -1
1265 1265 else:
1266 1266 self.setFile = -1 #inicializo mi contador de seteo
1267 1267
1268 1268 setFile = self.setFile
1269 1269 setFile += 1
1270 1270
1271 1271 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1272 1272 timeTuple.tm_year,
1273 1273 timeTuple.tm_yday,
1274 1274 setFile,
1275 1275 ext )
1276 1276
1277 1277 filename = os.path.join( path, subfolder, filen )
1278 1278
1279 1279 fp = open( filename,'wb' )
1280 1280
1281 1281 self.blockIndex = 0
1282 1282
1283 1283 #guardando atributos
1284 1284 self.filename = filename
1285 1285 self.subfolder = subfolder
1286 1286 self.fp = fp
1287 1287 self.setFile = setFile
1288 1288 self.flagIsNewFile = 1
1289 1289
1290 1290 self.setFirstHeader()
1291 1291
1292 1292 print '[Writing] file: %s'%self.filename
1293 1293
1294 1294 self.__writeFirstHeader()
1295 1295
1296 1296 return 1
1297 1297
1298 1298 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1299 1299 """
1300 1300 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1301 1301
1302 1302 Inputs:
1303 1303 path : el path destino en el cual se escribiran los files a crear
1304 1304 format : formato en el cual sera salvado un file
1305 1305 set : el setebo del file
1306 1306
1307 1307 Return:
1308 1308 0 : Si no realizo un buen seteo
1309 1309 1 : Si realizo un buen seteo
1310 1310 """
1311 1311
1312 1312 if ext == None:
1313 1313 ext = self.ext
1314 1314
1315 1315 ext = ext.lower()
1316 1316
1317 1317 self.ext = ext
1318 1318
1319 1319 self.path = path
1320 1320
1321 1321 self.setFile = set - 1
1322 1322
1323 1323 self.blocksPerFile = blocksPerFile
1324 1324
1325 1325 self.profilesPerBlock = profilesPerBlock
1326 1326
1327 1327 self.dataOut = dataOut
1328 1328
1329 1329 if not(self.setNextFile()):
1330 1330 print "[Writing] There isn't a next file"
1331 1331 return 0
1332 1332
1333 1333 self.setBlockDimension()
1334 1334
1335 1335 return 1
1336 1336
1337 1337 def run(self, dataOut, **kwargs):
1338 1338
1339 1339 if not(self.isConfig):
1340 1340
1341 1341 self.setup(dataOut, **kwargs)
1342 1342 self.isConfig = True
1343 1343
1344 1344 self.putData()
1345 1345
Show file before | Show file after | Hide comments
@@ -1,135 +1,135
1 1 '''
2 2 Created on Jul 15, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import time
7 7 import threading
8 8 import cPickle
9 9
10 10 try:
11 11 from gevent import sleep
12 12 except:
13 13 from time import sleep
14 14
15 15 SERIALIZER = cPickle
16 16
17 17 # from schainpy.serializer import DynamicSerializer
18 18 from schainpy.model.io.jroIO_usrp import USRPReader
19 19 from schainpy.serializer.DataTranslate import obj2Serial
20 20
21 21 class USRPReaderAPI(USRPReader, threading.Thread):
22 22
23 23 # __isBufferEmpty = True
24 24
25 25 __DATAKEYLIST = ['data','utctime','flagNoData']
26 26
27 27 def __init__(self, serializer='msgpack'):
28 28
29 29 threading.Thread.__init__(self)
30 30 USRPReader.__init__(self)
31 31
32 32 # self.__serializerObj = DynamicSerializer.DynamicSerializer('msgpack')
33 33 self.__mySerial = None
34 34 self.__isBufferEmpty = True
35 35
36 36 self.setSerializer(serializer)
37 37
38 38 def setSerializer(self, serializer):
39 39
40 40 self.__serializer = serializer
41 41
42 42 def getSerializer(self):
43 43
44 44 return self.__serializer
45 45
46 46 def getProfileIndex(self):
47 47
48 48 return self.profileIndex
49 49
50 50 def getSerialMetaData(self):
51 51
52 52 if self.__isBufferEmpty:
53 53 ini = time.time()
54 54
55 55 while True:
56 56
57 57 if not self.__isBufferEmpty:
58 58 break
59 59
60 60 if time.time() - ini > 20:
61 61 break
62 62
63 63 sleep(1e-12)
64 64
65 65
66 66 # if not self.getData():
67 67 # self.__isBufferEmpty = False
68 68 # return None
69 69
70 70 if self.dataOut.flagNoData:
71 71 return None
72 72
73 73 myMetadataSerial = obj2Serial(self.dataOut,
74 74 serializer = self.__serializer)
75 75
76 76 return myMetadataSerial
77 77
78 78 def getSerialData(self):
79 79
80 80 if self.__isBufferEmpty:
81 81 ini = time.time()
82 82
83 83 while True:
84 84
85 85 if not self.__isBufferEmpty:
86 86 break
87 87
88 88 if time.time() - ini > 20:
89 89 break
90 90
91 91 sleep(1e-12)
92 92
93 93
94 94 # if not self.getData():
95 95 # self.__isBufferEmpty = False
96 96 # return None
97 97
98 98 if self.dataOut.flagNoData:
99 99 return None
100 100
101 101 self.__isBufferEmpty = True
102 102
103 103 return self.__mySerial
104 104
105 105 def run(self):
106 106
107 107 '''
108 This method will be called many times so here you should put all your code
108 This method will be called once when start() is called
109 109 '''
110 110
111 111 if not self.isConfig:
112 112 raise IOError, 'setup() method has to be called before start()'
113 113
114 114 while True:
115 115
116 116 if not self.__isBufferEmpty:
117 117 sleep(1e-12)
118 118 continue
119 119
120 120 if not self.getData():
121 121 break
122 122
123 123 print ".",
124 124
125 125 self.__mySerial = obj2Serial(self.dataOut,
126 126 keyList = self.__DATAKEYLIST,
127 127 serializer = self.__serializer)
128 128 self.__isBufferEmpty = False
129 129
130 130 # print self.profileIndex
131 131 # print 'wait 1 second'
132 132
133 133 # sleep(0.1)
134 134
135 135 return No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,268 +1,275
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
5 5 '''
6 6
7 7 class ProcessingUnit(object):
8 8
9 9 """
10 10 Esta es la clase base para el procesamiento de datos.
11 11
12 12 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
13 13 - Metodos internos (callMethod)
14 14 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
15 15 tienen que ser agreagados con el metodo "add".
16 16
17 17 """
18 18 # objeto de datos de entrada (Voltage, Spectra o Correlation)
19 19 dataIn = None
20 20 dataInList = []
21 21
22 22 # objeto de datos de entrada (Voltage, Spectra o Correlation)
23 23 dataOut = None
24 24
25 25 operations2RunDict = None
26 26
27 27 isConfig = False
28 28
29 29
30 30 def __init__(self):
31 31
32 32 self.dataIn = None
33 33 self.dataInList = []
34 34
35 self.dataOut = {}
35 self.dataOut = None
36 36
37 37 self.operations2RunDict = {}
38 38
39 39 self.isConfig = False
40 40
41 41 def addOperation(self, opObj, objId):
42 42
43 43 """
44 44 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
45 45 identificador asociado a este objeto.
46 46
47 47 Input:
48 48
49 49 object : objeto de la clase "Operation"
50 50
51 51 Return:
52 52
53 53 objId : identificador del objeto, necesario para ejecutar la operacion
54 54 """
55 55
56 56 self.operations2RunDict[objId] = opObj
57 57
58 58 return objId
59 59
60 60 def operation(self, **kwargs):
61 61
62 62 """
63 63 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
64 64 atributos del objeto dataOut
65 65
66 66 Input:
67 67
68 68 **kwargs : Diccionario de argumentos de la funcion a ejecutar
69 69 """
70 70
71 71 raise ValueError, "ImplementedError"
72 72
73 73 def callMethod(self, name, **kwargs):
74 74
75 75 """
76 76 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
77 77
78 78 Input:
79 79 name : nombre del metodo a ejecutar
80 80
81 81 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
82 82
83 83 """
84 84
85 85 #Checking the inputs
86 86 if name == 'run':
87 87
88 88 if not self.checkInputs():
89 89 self.dataOut.flagNoData = True
90 90 return False
91 91 else:
92 92 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
93 93 if self.dataOut.isEmpty():
94 94 return False
95 95
96 96 #Getting the pointer to method
97 97 methodToCall = getattr(self, name)
98 98
99 99 #Executing the self method
100 100 methodToCall(**kwargs)
101 101
102 102 #Checkin the outputs
103 103
104 104 # if name == 'run':
105 105 # pass
106 106 # else:
107 107 # pass
108 108 #
109 109 # if name != 'run':
110 110 # return True
111
111
112 if self.dataOut == None:
113 return False
114
112 115 if self.dataOut.isEmpty():
113 116 return False
114 117
115 118 return True
116 119
117 120 def callObject(self, objId, **kwargs):
118 121
119 122 """
120 123 Ejecuta la operacion asociada al identificador del objeto "objId"
121 124
122 125 Input:
123 126
124 127 objId : identificador del objeto a ejecutar
125 128
126 129 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
127 130
128 131 Return:
129 132
130 133 None
131 134 """
132 135
133 136 if self.dataOut.isEmpty():
134 137 return False
135 138
136 139 externalProcObj = self.operations2RunDict[objId]
137 140
138 141 externalProcObj.run(self.dataOut, **kwargs)
139 142
140 143 return True
141 144
142 145 def call(self, opType, opName=None, opId=None, **kwargs):
143 146
144 147 """
145 148 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
146 149 identificada con el id "opId"; con los argumentos "**kwargs".
147 150
148 151 False si la operacion no se ha ejecutado.
149 152
150 153 Input:
151 154
152 155 opType : Puede ser "self" o "external"
153 156
154 157 La operacion puede ser de dos tipos (callMethod or callObject):
155 158
156 159 1. Un metodo propio de esta clase:
157 160
158 161 opType = "self"
159 162
160 163 2. El metodo "run" de un objeto del tipo Operation o de un derivado de ella:
161 164
162 165 opType = "other" or "external".
163 166
164 167 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
165 168 usada para llamar a un metodo interno de la clase Processing
166 169
167 170 opId : Si la operacion es externa (opType = 'other'), entonces el "opId" sera
168 171 usada para llamar al metodo "run" de la clase Operation registrada con ese Id
169 172
170 173 Exception:
171 174 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
172 175 "addOperation" e identificado con el valor "opId" = el id de la operacion.
173 176 De lo contrario retornara un error del tipo IOError
174 177
175 178 """
176 179
177 180 if opType == 'self':
178 181
179 182 if not opName:
180 183 raise IOError, "opName parameter should be defined"
181 184
182 185 sts = self.callMethod(opName, **kwargs)
183 186
184 187 if opType == 'other' or opType == 'external':
185 188
186 189 if not opId:
187 190 raise IOError, "opId parameter should be defined"
188 191
189 192 if opId not in self.operations2RunDict.keys():
190 193 raise IOError, "This id operation have not been registered"
191 194
192 195 sts = self.callObject(opId, **kwargs)
193 196
194 197 return sts
195 198
196 199 def setInput(self, dataIn):
197 200
198 201 self.dataIn = dataIn
199 202 self.dataInList.append(dataIn)
200 203
201 204 def getOutputObj(self):
202 205
203 206 return self.dataOut
204 207
205 208 def checkInputs(self):
206 209
207 210 for thisDataIn in self.dataInList:
208 211
209 212 if thisDataIn.isEmpty():
210 213 return False
211 214
212 215 return True
213 216
214 217 def setup(self):
215 218
216 219 raise ValueError, "Not implemented"
217 220
218 221 def run(self):
219 222
220 223 raise ValueError, "Not implemented"
221
224
225 def close(self):
226 #Close every thread, queue or any other object here is it is neccesary.
227 return
228
222 229 class Operation(object):
223 230
224 231 """
225 232 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
226 233 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
227 234 acumulacion dentro de esta clase
228 235
229 236 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
230 237
231 238 """
232 239
233 240 __buffer = None
234 241 isConfig = False
235 242
236 243 def __init__(self):
237 244
238 245 self.__buffer = None
239 246 self.isConfig = False
240 247
241 248 def setup(self):
242 249
243 250 self.isConfig = True
244 251
245 252 raise ValueError, "Not implemented"
246 253
247 254 def run(self, dataIn, **kwargs):
248 255
249 256 """
250 257 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
251 258 atributos del objeto dataIn.
252 259
253 260 Input:
254 261
255 262 dataIn : objeto del tipo JROData
256 263
257 264 Return:
258 265
259 266 None
260 267
261 268 Affected:
262 269 __buffer : buffer de recepcion de datos.
263 270
264 271 """
265 272 if not self.isConfig:
266 273 self.setup(**kwargs)
267 274
268 275 raise ValueError, "ImplementedError" No newline at end of file
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (565 lines changed) Show them Hide them
@@ -1,353 +1,912
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7
8 try:
9 import paramiko
10 import scp
11 except:
12 print "You should install paramiko if you will use SSH protocol to upload files to a server"
13
7 14 import multiprocessing
15
16 import time
17 import threading
18
19
20 try:
21 from gevent import sleep
22 except:
23 from time import sleep
24
8 25 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 26
27 class Remote(threading.Thread):
28 """
29 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
30 used to upload or download files remotely.
31
32 Non-standard Python modules used:
33 None
34
35 Written by:
36
37 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
38
39 """
40
41 server = None
42 username = None
43 password = None
44 remotefolder = None
45
46 period = 60
47 fileList = []
48 bussy = False
49
50 def __init__(self, server, username, password, remotefolder, period=60):
51
52 threading.Thread.__init__(self)
53 self._stop = threading.Event()
54
55 self.status = 0
56
57 self.period = period
58 self.fileList = []
59 self.bussy = False
60
61 self.stopFlag = False
62
63 print "[Remote Server] Opening server: %s" %server
64 if self.open(server, username, password, remotefolder):
65 print "[Remote Server] %s server was opened successfully" %server
66
67 def stop(self):
68
69 self.stopFlag = True
70
71 def open(self):
72 """
73 Connect to server and create a connection class (FTP or SSH) to remote server.
74 """
75 raise NotImplementedError, "Implement this method in child class"
76
77 def close(self):
78 """
79 Close connection to server
80 """
81 raise NotImplementedError, "Implement this method in child class"
82
83 def mkdir(self, remotefolder):
84 """
85 Create a folder remotely
86 """
87 raise NotImplementedError, "Implement this method in child class"
88
89 def cd(self, remotefolder):
90 """
91 Change working directory in remote server
92 """
93 raise NotImplementedError, "Implement this method in child class"
94
95 def download(self, filename, localfolder=None):
96 """
97 Download a file from server to local host
98 """
99 raise NotImplementedError, "Implement this method in child class"
100
101 def sendFile(self, fullfilename):
102 """
103 sendFile method is used to upload a local file to the current directory in remote server
104
105 Inputs:
106 fullfilename - full path name of local file to store in remote directory
107
108 Returns:
109 0 in error case else 1
110 """
111 raise NotImplementedError, "Implement this method in child class"
112
113 def upload(self, fullfilename, remotefolder=None):
114 """
115 upload method is used to upload a local file to remote directory. This method changes
116 working directory before sending a file.
117
118 Inputs:
119 fullfilename - full path name of local file to store in remote directory
120
121 remotefolder - remote directory
122
123 Returns:
124 0 in error case else 1
125 """
126 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
127
128 if not self.status:
129 return 0
130
131 if remotefolder == None:
132 remotefolder = self.remotefolder
133
134 if not self.cd(remotefolder):
135 return 0
136
137 if not self.sendFile(fullfilename):
138 print "[Remote Server] Error uploading file %s" %fullfilename
139 return 0
140
141 print "[Remote Server] upload finished successfully"
142
143 return 1
144
145 def delete(self, filename):
146 """
147 Remove a file from remote server
148 """
149 pass
150
151 def updateFileList(self, fileList):
152 """
153 Remove a file from remote server
154 """
155
156 if fileList == self.fileList:
157 return 1
158
159 init = time.time()
160
161 while(self.bussy):
162 sleep(0.1)
163 if time.time() - init > 2*self.period:
164 return 0
165
166 self.fileList = fileList
167
168 return 1
169
170 def run(self):
171
172 if not self.cd(self.remotefolder):
173 raise ValueError, "It could not change to the new remote directory: %s" %remotefolder
174
175 while True:
176
177 sleep(self.period)
178
179 self.bussy = True
180
181 for thisFile in self.fileList:
182 self.upload(thisFile, self.remotefolder)
183
184 self.bussy = False
185
186 if self.stopFlag:
187 break
188
189 print "[Remote Server] Thread stopped successfully"
190
191 class FTPClient(Remote):
192
193 __ftpClientObj = None
194
195 def __init__(self, server, username, password, remotefolder, period=60):
196 """
197 """
198 Remote.__init__(self, server, username, password, remotefolder, period)
199
200 def open(self, server, username, password, remotefolder):
201
202 """
203 This method is used to set FTP parameters and establish a connection to remote server
204
205 Inputs:
206 server - remote server IP Address
207
208 username - remote server Username
209
210 password - remote server password
211
212 remotefolder - remote server current working directory
213
214 Return: void
215
216 Affects:
217 self.status - in case of error or fail connection this parameter is set to 0 else 1
218
219 """
220
221 if server == None:
222 raise ValueError, "FTP server should be defined"
223
224 if username == None:
225 raise ValueError, "FTP username should be defined"
226
227 if password == None:
228 raise ValueError, "FTP password should be defined"
229
230 if remotefolder == None:
231 raise ValueError, "FTP remote folder should be defined"
232
233 try:
234 ftpClientObj = ftplib.FTP(server)
235 except ftplib.all_errors:
236 print "FTP server connection fail: %s" %server
237 self.status = 0
238 return 0
239
240 try:
241 ftpClientObj.login(username, password)
242 except ftplib.all_errors:
243 print "FTP username or password are incorrect"
244 self.status = 0
245 return 0
246
247 if remotefolder == None:
248 remotefolder = ftpClientObj.pwd()
249 else:
250 try:
251 ftpClientObj.cwd(remotefolder)
252 except ftplib.all_errors:
253 print "FTP remote folder is invalid: %s" %remotefolder
254 remotefolder = ftpClientObj.pwd()
255
256 self.server = server
257 self.username = username
258 self.password = password
259 self.remotefolder = remotefolder
260 self.__ftpClientObj = ftpClientObj
261 self.status = 1
262
263 return 1
264
265 def close(self):
266 """
267 Close connection to remote server
268 """
269 if not self.status:
270 return 0
271
272 self.__ftpClientObj.close()
273
274 def mkdir(self, remotefolder):
275 """
276 mkdir is used to make a new directory in remote server
277
278 Input:
279 remotefolder - directory name
280
281 Return:
282 0 in error case else 1
283 """
284 if not self.status:
285 return 0
286
287 try:
288 self.__ftpClientObj.mkd(dirname)
289 except ftplib.all_errors:
290 print "Error creating remote folder: %s" %remotefolder
291 return 0
292
293 return 1
294
295 def cd(self, remotefolder):
296 """
297 cd is used to change remote working directory on server
298
299 Input:
300 remotefolder - current working directory
301
302 Affects:
303 self.remotefolder
304
305 Return:
306 0 in case of error else 1
307 """
308 if not self.status:
309 return 0
310
311 if remotefolder == self.remotefolder:
312 return 1
313
314 try:
315 self.__ftpClientObj.cwd(remotefolder)
316 except ftplib.all_errors:
317 print 'Error changing to %s' %remotefolder
318 print 'Trying to create remote folder'
319
320 if not self.mkdir(remotefolder):
321 print 'Remote folder could not be created'
322 return 0
323
324 try:
325 self.__ftpClientObj.cwd(remotefolder)
326 except ftplib.all_errors:
327 return 0
328
329 self.remotefolder = remotefolder
330
331 return 1
332
333 def sendFile(self, fullfilename):
334
335 if not self.status:
336 return 0
337
338 file = open(fullfilename, 'rb')
339
340 filename = os.path.split(fullfilename)[-1]
341
342 command = "STOR %s" %filename
343
344 try:
345 self.__ftpClientObj.storbinary(command, file)
346 except ftplib.all_errors:
347 return 0
348
349 try:
350 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
351 except ftplib.all_errors, e:
352 print e
353
354 file.close()
355
356 return 1
357
358 class SSHClient(Remote):
359
360 __sshClientObj = None
361 __scpClientObj = None
362
363 def __init__(self, server, username, password, remotefolder, period=60):
364 """
365 """
366 Remote.__init__(self, server, username, password, remotefolder, period)
367
368 def open(self, server, username, password, remotefolder, port=22):
369
370 """
371 This method is used to set SSH parameters and establish a connection to a remote server
372
373 Inputs:
374 server - remote server IP Address
375
376 username - remote server Username
377
378 password - remote server password
379
380 remotefolder - remote server current working directory
381
382 Return: void
383
384 Affects:
385 self.status - in case of error or fail connection this parameter is set to 0 else 1
386
387 """
388
389 if server == None:
390 raise ValueError, "SSH server should be defined"
391
392 if username == None:
393 raise ValueError, "SSH username should be defined"
394
395 if password == None:
396 raise ValueError, "SSH password should be defined"
397
398 if remotefolder == None:
399 raise ValueError, "SSH remote folder should be defined"
400
401 try:
402 sshClientObj = paramiko.SSHClient()
403 except:
404 print "SSH server connection fail: %s" %server
405 self.status = 0
406 return 0
407
408 sshClientObj.load_system_host_keys()
409 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
410
411 try:
412 sshClientObj.connect(server, username=username, password=password, port=port)
413 except :
414 print "SSH username or password are incorrect: %s"
415 self.status = 0
416 return 0
417
418 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
419
420 if remotefolder == None:
421 remotefolder = self.pwd()
422
423 self.server = server
424 self.username = username
425 self.password = password
426 self.remotefolder = remotefolder
427 self.__sshClientObj = sshClientObj
428 self.__scpClientObj = scpClientObj
429 self.status = 1
430
431 return 1
432
433 def close(self):
434 """
435 Close connection to remote server
436 """
437 if not self.status:
438 return 0
439
440 self.__sshObj.close()
441
442 def mkdir(self, remotefolder):
443 """
444 mkdir is used to make a new directory in remote server
445
446 Input:
447 remotefolder - directory name
448
449 Return:
450 0 in error case else 1
451 """
452 if not self.status:
453 return 0
454
455 stdin, stdout, stderr = self.__sshClientObj.exec_command('mkdir %s' %remotefolder)
456 result = stderr.readlines()[0]
457
458 if len(result) > 1:
459 return 0
460
461 return 1
462
463 def pwd(self):
464
465 if not self.status:
466 return None
467
468 stdin, stdout, stderr = self.__sshClientObj.exec_command('pwd')
469 result = stdout.readlines()[0]
470
471 if len(result) < 1:
472 return None
473
474 return result[:-1]
475
476 def cd(self, remotefolder):
477 """
478 cd is used to change remote working directory on server
479
480 Input:
481 remotefolder - current working directory
482
483 Affects:
484 self.remotefolder
485
486 Return:
487 0 in case of error else 1
488 """
489 if not self.status:
490 return 0
491
492 if remotefolder == self.remotefolder:
493 return 1
494
495 self.remotefolder = remotefolder
496
497 return 1
498
499 def sendFile(self, fullfilename):
500
501 if not self.status:
502 return 0
503
504 try:
505 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
506 except:
507 return 0
508
509 return 1
510
511 class SendToServer(ProcessingUnit):
512
513 def __init__(self):
514
515 ProcessingUnit.__init__(self)
516
517 self.isConfig = False
518 self.clientObj = None
519
520 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp'):
521
522 self.clientObj = None
523 self.localfolder = localfolder
524 self.ext = ext
525 self.period = period
526
527 if str.lower(protocol) == 'ftp':
528 self.clientObj = FTPClient(server, username, password, remotefolder, period)
529
530 if str.lower(protocol) == 'ssh':
531 self.clientObj = SSHClient(server, username, password, remotefolder, period)
532
533 if not self.clientObj:
534 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
535
536 self.clientObj.start()
537
538 def findFiles(self):
539
540 filenameList = glob.glob1(self.localfolder, '*%s' %self.ext)
541
542 if len(filenameList) < 1:
543 return []
544
545 fullfilenameList = [os.path.join(self.localfolder, thisFile) for thisFile in filenameList]
546
547 return fullfilenameList
548
549 def run(self, **kwargs):
550
551 if not self.isConfig:
552 self.init = time.time()
553 self.setup(**kwargs)
554 self.isConfig = True
555
556 if time.time() - self.init >= self.period:
557 fullfilenameList = self.findFiles()
558 self.clientObj.updateFileList(fullfilenameList)
559 self.init = time.time()
560
561 def close(self):
562 print "[Remote Server] Stopping thread"
563 self.clientObj.stop()
564
565
10 566 class FTP(object):
11 567 """
12 568 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
13 569
14 570 Non-standard Python modules used: None
15 571
16 572 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
17 573 """
18 574
19 575 def __init__(self,server = None, username=None, password=None, remotefolder=None):
20 576 """
21 577 This method is used to setting parameters for FTP and establishing connection to remote server
22 578
23 579 Inputs:
24 580 server - remote server IP Address
25 581
26 582 username - remote server Username
27 583
28 584 password - remote server password
29 585
30 586 remotefolder - remote server current working directory
31 587
32 588 Return: void
33 589
34 590 Affects:
35 591 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
36 592
37 593 self.folderList - sub-folder list of remote folder
38 594
39 595 self.fileList - file list of remote folder
40 596
41 597
42 598 """
43 599
44 600 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
45 601 server, username, password, remotefolder = self.parmsByDefault()
46 602
47 603 self.server = server
48 604 self.username = username
49 605 self.password = password
50 606 self.remotefolder = remotefolder
51 607 self.file = None
52 608 self.ftp = None
53 609 self.status = 0
54 610
55 611 try:
56 612 self.ftp = ftplib.FTP(self.server)
57 613 self.ftp.login(self.username,self.password)
58 614 self.ftp.cwd(self.remotefolder)
59 615 # print 'Connect to FTP Server: Successfully'
60 616
61 617 except ftplib.all_errors:
62 618 print 'Error FTP Service'
63 619 self.status = 1
64 620 return
65 621
66 622
67 623
68 624 self.dirList = []
69 625
70 626 try:
71 627 self.dirList = self.ftp.nlst()
72 628
73 629 except ftplib.error_perm, resp:
74 630 if str(resp) == "550 No files found":
75 631 print "no files in this directory"
76 632 self.status = 1
77 633 return
78 634
79 635 except ftplib.all_errors:
80 636 print 'Error Displaying Dir-Files'
81 637 self.status = 1
82 638 return
83 639
84 640 self.fileList = []
85 641 self.folderList = []
86 642 #only for test
87 643 for f in self.dirList:
88 644 name, ext = os.path.splitext(f)
89 645 if ext != '':
90 646 self.fileList.append(f)
91 647 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
92 648
93 649 def parmsByDefault(self):
94 650 server = 'jro-app.igp.gob.pe'
95 651 username = 'wmaster'
96 652 password = 'mst2010vhf'
97 653 remotefolder = '/home/wmaster/graficos'
98 654
99 655 return server, username, password, remotefolder
100 656
101 657
102 658 def mkd(self,dirname):
103 659 """
104 660 mkd is used to make directory in remote server
105 661
106 662 Input:
107 663 dirname - directory name
108 664
109 665 Return:
110 666 1 in error case else 0
111 667 """
112 668 try:
113 669 self.ftp.mkd(dirname)
114 670 except:
115 671 print 'Error creating remote folder:%s'%dirname
116 672 return 1
117 673
118 674 return 0
119 675
120 676
121 677 def delete(self,filename):
122 678 """
123 679 delete is used to delete file in current working directory of remote server
124 680
125 681 Input:
126 682 filename - filename to delete in remote folder
127 683
128 684 Return:
129 685 1 in error case else 0
130 686 """
131 687
132 688 try:
133 689 self.ftp.delete(filename)
134 690 except:
135 691 print 'Error deleting remote file:%s'%filename
136 692 return 1
137 693
138 694 return 0
139 695
140 696 def download(self,filename,localfolder):
141 697 """
142 698 download is used to downloading file from remote folder into local folder
143 699
144 700 Inputs:
145 701 filename - filename to donwload
146 702
147 703 localfolder - directory local to store filename
148 704
149 705 Returns:
150 706 self.status - 1 in error case else 0
151 707 """
152 708
153 709 self.status = 0
154 710
155 711
156 712 if not(filename in self.fileList):
157 713 print 'filename:%s not exists'%filename
158 714 self.status = 1
159 715 return self.status
160 716
161 717 newfilename = os.path.join(localfolder,filename)
162 718
163 719 self.file = open(newfilename, 'wb')
164 720
165 721 try:
166 722 print 'Download: ' + filename
167 723 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
168 724 print 'Download Complete'
169 725 except ftplib.all_errors:
170 726 print 'Error Downloading ' + filename
171 727 self.status = 1
172 728 return self.status
173 729
174 730 self.file.close()
175 731
176 732 return self.status
177 733
178 734
179 735 def __handleDownload(self,block):
180 736 """
181 737 __handleDownload is used to handle writing file
182 738 """
183 739 self.file.write(block)
184 740
185 741
186 742 def upload(self,filename,remotefolder=None):
187 743 """
188 744 upload is used to uploading local file to remote directory
189 745
190 746 Inputs:
191 747 filename - full path name of local file to store in remote directory
192 748
193 749 remotefolder - remote directory
194 750
195 751 Returns:
196 752 self.status - 1 in error case else 0
197 753 """
198 754
199 755 if remotefolder == None:
200 756 remotefolder = self.remotefolder
201 757
202 758 self.status = 0
203 759
204 760 try:
205 761 self.ftp.cwd(remotefolder)
206 762
207 763 self.file = open(filename, 'rb')
208 764
209 765 (head, tail) = os.path.split(filename)
210 766
211 767 command = "STOR " + tail
212 768
213 769 print 'Uploading: ' + tail
214 770 self.ftp.storbinary(command, self.file)
215 771 print 'Upload Completed'
216 772
217 773 except ftplib.all_errors:
218 774 print 'Error Uploading ' + tail
219 775 self.status = 1
220 776 return self.status
221 777
222 778 self.file.close()
223 779
224 780 #back to initial directory in __init__()
225 781 self.ftp.cwd(self.remotefolder)
226 782
227 783 return self.status
228 784
229 785
230 786 def dir(self,remotefolder):
231 787 """
232 788 dir is used to change working directory of remote server and get folder and file list
233 789
234 790 Input:
235 791 remotefolder - current working directory
236 792
237 793 Affects:
238 794 self.fileList - file list of working directory
239 795
240 796 Return:
241 797 infoList - list with filenames and size of file in bytes
242 798
243 799 self.folderList - folder list
244 800 """
245 801
246 802 self.remotefolder = remotefolder
247 803 print 'Change to ' + self.remotefolder
248 804 try:
249 805 self.ftp.cwd(remotefolder)
250 806 except ftplib.all_errors:
251 807 print 'Error Change to ' + self.remotefolder
252 808 infoList = None
253 809 self.folderList = None
254 810 return infoList,self.folderList
255 811
256 812 self.dirList = []
257 813
258 814 try:
259 815 self.dirList = self.ftp.nlst()
260 816
261 817 except ftplib.error_perm, resp:
262 818 if str(resp) == "550 No files found":
263 819 print "no files in this directory"
264 820 infoList = None
265 821 self.folderList = None
266 822 return infoList,self.folderList
267 823 except ftplib.all_errors:
268 824 print 'Error Displaying Dir-Files'
269 825 infoList = None
270 826 self.folderList = None
271 827 return infoList,self.folderList
272 828
273 829 infoList = []
274 830 self.fileList = []
275 831 self.folderList = []
276 832 for f in self.dirList:
277 833 name,ext = os.path.splitext(f)
278 834 if ext != '':
279 835 self.fileList.append(f)
280 836 value = (f,self.ftp.size(f))
281 837 infoList.append(value)
282 838
283 839 if ext == '':
284 840 self.folderList.append(f)
285 841
286 842 return infoList,self.folderList
287 843
288 844
289 845 def close(self):
290 846 """
291 847 close is used to close and end FTP connection
292 848
293 849 Inputs: None
294 850
295 851 Return: void
296 852
297 853 """
298 854 self.ftp.close()
299 855
300 856 class SendByFTP(Operation):
857
301 858 def __init__(self):
859
302 860 self.status = 1
303 861 self.counter = 0
304 862
305 863 def error_print(self, ValueError):
864
306 865 print ValueError, 'Error FTP'
307 866 print "don't worry the program is running..."
308 867
309 868 def worker_ftp(self, server, username, password, remotefolder, filenameList):
310 869
311 self.ftpObj = FTP(server, username, password, remotefolder)
870 self.ftpClientObj = FTP(server, username, password, remotefolder)
312 871 for filename in filenameList:
313 self.ftpObj.upload(filename)
314 self.ftpObj.close()
872 self.ftpClientObj.upload(filename)
873 self.ftpClientObj.close()
315 874
316 875 def ftp_thread(self, server, username, password, remotefolder):
317 876 if not(self.status):
318 877 return
319 878
320 879 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
321 880 p.start()
322 881
323 882 p.join(3)
324 883
325 884 if p.is_alive():
326 885 p.terminate()
327 886 p.join()
328 887 print 'killing ftp process...'
329 888 self.status = 0
330 889 return
331 890
332 891 self.status = 1
333 892 return
334 893
335 894 def filterByExt(self, ext, localfolder):
336 895 fnameList = glob.glob1(localfolder,ext)
337 896 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
338 897
339 898 if len(self.filenameList) == 0:
340 899 self.status = 0
341 900
342 901 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
343 902
344 903 self.counter += 1
345 904 if self.counter >= period:
346 905 self.filterByExt(ext, localfolder)
347 906
348 907 self.ftp_thread(server, username, password, remotefolder)
349 908
350 909 self.counter = 0
351 910
352 911 self.status = 1
353 912
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now