##// END OF EJS Templates
schain-jc
RSS
merge from master
José Chávez -
r1014:c5cbd628ce0f merge
parent child
Show More
Side by Side Unified
Context file:
r1014:c5cbd628ce0f Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,69
1 {
2 "configurations": [
3 {
4 "name": "Mac",
5 "includePath": [
6 "/usr/include",
7 "/usr/local/include",
8 "${workspaceRoot}"
9 ],
10 "defines": [],
11 "intelliSenseMode": "clang-x64",
12 "browse": {
13 "path": [
14 "/usr/include",
15 "/usr/local/include",
16 "${workspaceRoot}"
17 ],
18 "limitSymbolsToIncludedHeaders": true,
19 "databaseFilename": ""
20 }
21 },
22 {
23 "name": "Linux",
24 "includePath": [
25 "/usr/include/c++/4.8.4",
26 "/usr/include/x86_64-linux-gnu/c++/4.8",
27 "/usr/local/include",
28 "/usr/include",
29 "/usr/include/x86_64-linux-gnu",
30 "${workspaceRoot}"
31 ],
32 "defines": [],
33 "intelliSenseMode": "clang-x64",
34 "browse": {
35 "path": [
36 "/usr/include/c++/4.8.4",
37 "/usr/include/x86_64-linux-gnu/c++/4.8",
38 "/usr/local/include",
39 "/usr/include",
40 "/usr/include/x86_64-linux-gnu",
41 "${workspaceRoot}"
42 ],
43 "limitSymbolsToIncludedHeaders": true,
44 "databaseFilename": ""
45 }
46 },
47 {
48 "name": "Win32",
49 "includePath": [
50 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include",
51 "${workspaceRoot}"
52 ],
53 "defines": [
54 "_DEBUG",
55 "UNICODE"
56 ],
57 "intelliSenseMode": "msvc-x64",
58 "browse": {
59 "path": [
60 "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include/*",
61 "${workspaceRoot}"
62 ],
63 "limitSymbolsToIncludedHeaders": true,
64 "databaseFilename": ""
65 }
66 }
67 ],
68 "version": 2
69 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,7
1 {
2 "files.associations": {
3 "arrayobject.h": "c",
4 "ndarrayobject.h": "c",
5 "complex": "c"
6 }
7 } No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,109 +1,118
1 1 # Byte-compiled / optimized / DLL files
2 2 __pycache__/
3 3 *.py[cod]
4 4 *$py.class
5 5
6 6 # C extensions
7 7 *.so
8 8
9 9 # Distribution / packaging
10 10 .Python
11 11 env/
12 12 build/
13 13 develop-eggs/
14 14 dist/
15 15 downloads/
16 16 eggs/
17 17 .eggs/
18 18 lib/
19 19 lib64/
20 20 parts/
21 21 sdist/
22 22 var/
23 23 wheels/
24 24 *.egg-info/
25 25 .installed.cfg
26 26 *.egg
27 27
28 28 # PyInstaller
29 29 # Usually these files are written by a python script from a template
30 30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 31 *.manifest
32 32 *.spec
33 33
34 34 # Installer logs
35 35 pip-log.txt
36 36 pip-delete-this-directory.txt
37 37
38 38 # Unit test / coverage reports
39 39 htmlcov/
40 40 .tox/
41 41 .coverage
42 42 .coverage.*
43 43 .cache
44 44 nosetests.xml
45 45 coverage.xml
46 46 *,cover
47 47 .hypothesis/
48 48
49 49 # Translations
50 50 *.mo
51 51 *.pot
52 52
53 53 # Django stuff:
54 54 *.log
55 55 local_settings.py
56 56
57 57 # Flask stuff:
58 58 instance/
59 59 .webassets-cache
60 60
61 61 # Scrapy stuff:
62 62 .scrapy
63 63
64 64 # Sphinx documentation
65 65 docs/_build/
66 66
67 67 # PyBuilder
68 68 target/
69 69
70 70 # Jupyter Notebook
71 71 .ipynb_checkpoints
72 72
73 73 # pyenv
74 74 .python-version
75 75
76 76 # celery beat schedule file
77 77 celerybeat-schedule
78 78
79 79 # SageMath parsed files
80 80 *.sage.py
81 81
82 82 # dotenv
83 83 .env
84 84
85 85 # virtualenv
86 86 .venv
87 87 venv/
88 88 ENV/
89 89
90 90 # Spyder project settings
91 91 .spyderproject
92 92 .spyproject
93 93
94 94 # Rope project settings
95 95 .ropeproject
96 96
97 97 # mkdocs documentation
98 98 /site
99 99
100 100 # eclipse
101 101 .project
102 102 .pydevproject
103 <<<<<<< HEAD
103 104
104 105 # vscode
105 106
106 107 .vscode
107 108
108 109 schainpy/scripts/
109 schaingui/node_modules/ No newline at end of file
110 schaingui/node_modules/
111 =======
112 .svn/
113 *.png
114 *.pyc
115 schainpy/scripts
116
117 schaingui/node_modules
118 >>>>>>> master
Show file before | Show file after | Hide comments
@@ -1,132 +1,132
1 1 # Signal Chain
2 2
3 3 ## Introduction
4 4
5 5 Signal Chain (SCh) is a radar data processing library developed using [Python](www.python.org) at JRO. SCh provides modules to read, write, process and plot data.
6 6
7 7 ## Installation
8 8
9 9 Install system dependencies, clone the latest version from [git](http://jro-dev.igp.gob.pe/rhodecode/schain/) and install it as a normal python package.
10 10
11 11 ```
12 12 $ sudo apt-get install python-pip python-dev gfortran libpng-dev freetype* libblas-dev liblapack-dev libatlas-base-dev python-qt4 python-tk libssl-dev libhdf5-dev
13 13 $ sudo pip install numpy
14 14 $ git clone http://jro-dev.igp.gob.pe/rhodecode/schain/
15 15 $ cd schain
16 16 $ sudo pip install ./
17 17 ```
18 18
19 19 **Its recommended to install schain in a virtual environment**
20 20
21 21 ```
22 22 $ sudo pip install virtualenv
23 23 $ virtualenv /path/to/virtual --system-site-packages
24 24 $ source /path/to/virtual/bin/activate
25 25 (virtual) $ cd schain
26 26 (virtual) $ pip install ./
27 27 ```
28 28
29 29 ## First Script
30 30
31 31 Read Spectra data (.pdata) - remove dc - plot spectra & RTI
32 32
33 33 Import SCh and creating a project
34 34
35 35 ```python
36 36 #!/usr/bin/python
37 37
38 38 from schainpy.controller import Project
39 39
40 40 controller = Project()
41 41 controller.setup(id = '100',
42 42 name='test',
43 43 description='Basic experiment')
44 44
45 45
46 46 ```
47 47
48 48 Adding read unit and operations
49 49
50 50 ```python
51 51 read_unit = controller.addReadUnit(datatype='Spectra',
52 52 path='/path/to/pdata/',
53 53 startDate='2014/01/31',
54 54 endDate='2014/03/31',
55 55 startTime='00:00:00',
56 56 endTime='23:59:59',
57 57 online=0,
58 58 walk=0)
59 59
60 60 proc_unit = controller.addProcUnit(datatype='Spectra',
61 61 inputId=read_unit.getId())
62 62
63 63 op = proc_unit.addOperation(name='selectChannels')
64 64 op.addParameter(name='channelList', value='0,1', format='intlist')
65 65
66 66 op = proc_unit.addOperation(name='selectHeights')
67 67 op.addParameter(name='minHei', value='80', format='float')
68 68 op.addParameter(name='maxHei', value='200', format='float')
69 69
70 70 op = proc_unit.addOperation(name='removeDC')
71 71
72 72 ```
73 73
74 74 Plotting data & start project
75 75
76 76 ```python
77 77 op = proc_unit.addOperation(name='SpectraPlot', optype='other')
78 78 op.addParameter(name='id', value='1', format='int')
79 79 op.addParameter(name='wintitle', value='Spectra', format='str')
80 80
81 81 op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
82 82 op.addParameter(name='id', value='2', format='int')
83 83 op.addParameter(name='wintitle', value='RTI', format='str')
84 84
85 85 controller.start()
86 86
87 87 ```
88 88
89 89 Full script
90 90
91 91
92 92 ```python
93 93 #!/usr/bin/python
94 94
95 95 from schainpy.controller import Project
96 96
97 97 controller = Project()
98 98 controller.setup(id = '100',
99 99 name='test',
100 100 description='Basic experiment')
101 101 read_unit = controller.addReadUnit(datatype='Spectra',
102 102 path='/path/to/pdata/',
103 103 startDate='2014/01/31',
104 104 endDate='2014/03/31',
105 105 startTime='00:00:00',
106 106 endTime='23:59:59',
107 107 online=0,
108 108 walk=0)
109 109
110 110 proc_unit = controller.addProcUnit(datatype='Spectra',
111 111 inputId=read_unit.getId())
112 112
113 113 op = proc_unit.addOperation(name='selectChannels')
114 114 op.addParameter(name='channelList', value='0,1', format='intlist')
115 115
116 116 op = proc_unit.addOperation(name='selectHeights')
117 117 op.addParameter(name='minHei', value='80', format='float')
118 118 op.addParameter(name='maxHei', value='200', format='float')
119 119
120 120 op = proc_unit.addOperation(name='removeDC')
121 121
122 122 op = proc_unit.addOperation(name='SpectraPlot', optype='other')
123 123 op.addParameter(name='id', value='6', format='int')
124 124 op.addParameter(name='wintitle', value='Spectra', format='str')
125 125
126 126 op = procUnitConfObj1.addOperation(name='RTIPlot', optype='other')
127 127 op.addParameter(name='id', value='2', format='int')
128 128 op.addParameter(name='wintitle', value='RTI', format='str')
129 129
130 130 controller.start()
131 131
132 ```
132 ``` No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1333 +1,1333
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, Queue, cpu_count
13 13 from profilehooks import profile, coverage
14 14
15 15 import schainpy
16 16 import schainpy.admin
17 17
18 18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 19 from xml.dom import minidom
20 20
21 21 from schainpy.model import *
22 22 from time import sleep
23 23
24 24
25 25
26 26 def prettify(elem):
27 27 """Return a pretty-printed XML string for the Element.
28 28 """
29 29 rough_string = tostring(elem, 'utf-8')
30 30 reparsed = minidom.parseString(rough_string)
31 31 return reparsed.toprettyxml(indent=" ")
32 32
33 33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 34 skip = 0
35 35 cursor = 0
36 36 nFiles = None
37 37 processes = []
38 38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
40 40 days = (dt2 - dt1).days
41 41
42 42 for day in range(days+1):
43 43 skip = 0
44 44 cursor = 0
45 45 q = Queue()
46 46 processes = []
47 47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 49 firstProcess.start()
50 50 if by_day:
51 51 continue
52 52 nFiles = q.get()
53 53 if nFiles==0:
54 54 continue
55 55 firstProcess.terminate()
56 56 skip = int(math.ceil(nFiles/nProcess))
57 57 while True:
58 58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 59 processes[cursor].start()
60 60 if nFiles < cursor*skip:
61 61 break
62 62 cursor += 1
63 63
64 64 def beforeExit(exctype, value, trace):
65 65 for process in processes:
66 66 process.terminate()
67 67 process.join()
68 68 print traceback.print_tb(trace)
69 69
70 70 sys.excepthook = beforeExit
71 71
72 72 for process in processes:
73 73 process.join()
74 74 process.terminate()
75 75
76 76 time.sleep(3)
77 77
78 78
79 79 class ParameterConf():
80 80
81 81 id = None
82 82 name = None
83 83 value = None
84 84 format = None
85 85
86 86 __formated_value = None
87 87
88 88 ELEMENTNAME = 'Parameter'
89 89
90 90 def __init__(self):
91 91
92 92 self.format = 'str'
93 93
94 94 def getElementName(self):
95 95
96 96 return self.ELEMENTNAME
97 97
98 98 def getValue(self):
99 99
100 100 value = self.value
101 101 format = self.format
102 102
103 103 if self.__formated_value != None:
104 104
105 105 return self.__formated_value
106 106
107 107 if format == 'obj':
108 108 return value
109 109
110 110 if format == 'str':
111 111 self.__formated_value = str(value)
112 112 return self.__formated_value
113 113
114 114 if value == '':
115 115 raise ValueError, "%s: This parameter value is empty" %self.name
116 116
117 117 if format == 'list':
118 118 strList = value.split(',')
119 119
120 120 self.__formated_value = strList
121 121
122 122 return self.__formated_value
123 123
124 124 if format == 'intlist':
125 125 """
126 126 Example:
127 127 value = (0,1,2)
128 128 """
129 129
130 130 new_value = ast.literal_eval(value)
131 131
132 132 if type(new_value) not in (tuple, list):
133 133 new_value = [int(new_value)]
134 134
135 135 self.__formated_value = new_value
136 136
137 137 return self.__formated_value
138 138
139 139 if format == 'floatlist':
140 140 """
141 141 Example:
142 142 value = (0.5, 1.4, 2.7)
143 143 """
144 144
145 145 new_value = ast.literal_eval(value)
146 146
147 147 if type(new_value) not in (tuple, list):
148 148 new_value = [float(new_value)]
149 149
150 150 self.__formated_value = new_value
151 151
152 152 return self.__formated_value
153 153
154 154 if format == 'date':
155 155 strList = value.split('/')
156 156 intList = [int(x) for x in strList]
157 157 date = datetime.date(intList[0], intList[1], intList[2])
158 158
159 159 self.__formated_value = date
160 160
161 161 return self.__formated_value
162 162
163 163 if format == 'time':
164 164 strList = value.split(':')
165 165 intList = [int(x) for x in strList]
166 166 time = datetime.time(intList[0], intList[1], intList[2])
167 167
168 168 self.__formated_value = time
169 169
170 170 return self.__formated_value
171 171
172 172 if format == 'pairslist':
173 173 """
174 174 Example:
175 175 value = (0,1),(1,2)
176 176 """
177 177
178 178 new_value = ast.literal_eval(value)
179 179
180 180 if type(new_value) not in (tuple, list):
181 181 raise ValueError, "%s has to be a tuple or list of pairs" %value
182 182
183 183 if type(new_value[0]) not in (tuple, list):
184 184 if len(new_value) != 2:
185 185 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 186 new_value = [new_value]
187 187
188 188 for thisPair in new_value:
189 189 if len(thisPair) != 2:
190 190 raise ValueError, "%s has to be a tuple or list of pairs" %value
191 191
192 192 self.__formated_value = new_value
193 193
194 194 return self.__formated_value
195 195
196 196 if format == 'multilist':
197 197 """
198 198 Example:
199 199 value = (0,1,2),(3,4,5)
200 200 """
201 201 multiList = ast.literal_eval(value)
202 202
203 203 if type(multiList[0]) == int:
204 204 multiList = ast.literal_eval("(" + value + ")")
205 205
206 206 self.__formated_value = multiList
207 207
208 208 return self.__formated_value
209 209
210 210 if format == 'bool':
211 211 value = int(value)
212 212
213 213 if format == 'int':
214 214 value = float(value)
215 215
216 216 format_func = eval(format)
217 217
218 218 self.__formated_value = format_func(value)
219 219
220 220 return self.__formated_value
221 221
222 222 def updateId(self, new_id):
223 223
224 224 self.id = str(new_id)
225 225
226 226 def setup(self, id, name, value, format='str'):
227 227 self.id = str(id)
228 228 self.name = name
229 229 if format == 'obj':
230 230 self.value = value
231 231 else:
232 232 self.value = str(value)
233 233 self.format = str.lower(format)
234 234
235 235 self.getValue()
236 236
237 237 return 1
238 238
239 239 def update(self, name, value, format='str'):
240 240
241 241 self.name = name
242 242 self.value = str(value)
243 243 self.format = format
244 244
245 245 def makeXml(self, opElement):
246 246 if self.name not in ('queue',):
247 247 parmElement = SubElement(opElement, self.ELEMENTNAME)
248 248 parmElement.set('id', str(self.id))
249 249 parmElement.set('name', self.name)
250 250 parmElement.set('value', self.value)
251 251 parmElement.set('format', self.format)
252 252
253 253 def readXml(self, parmElement):
254 254
255 255 self.id = parmElement.get('id')
256 256 self.name = parmElement.get('name')
257 257 self.value = parmElement.get('value')
258 258 self.format = str.lower(parmElement.get('format'))
259 259
260 260 #Compatible with old signal chain version
261 261 if self.format == 'int' and self.name == 'idfigure':
262 262 self.name = 'id'
263 263
264 264 def printattr(self):
265 265
266 266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
267 267
268 268 class OperationConf():
269 269
270 270 id = None
271 271 name = None
272 272 priority = None
273 273 type = None
274 274
275 275 parmConfObjList = []
276 276
277 277 ELEMENTNAME = 'Operation'
278 278
279 279 def __init__(self):
280 280
281 281 self.id = '0'
282 282 self.name = None
283 283 self.priority = None
284 284 self.type = 'self'
285 285
286 286
287 287 def __getNewId(self):
288 288
289 289 return int(self.id)*10 + len(self.parmConfObjList) + 1
290 290
291 291 def updateId(self, new_id):
292 292
293 293 self.id = str(new_id)
294 294
295 295 n = 1
296 296 for parmObj in self.parmConfObjList:
297 297
298 298 idParm = str(int(new_id)*10 + n)
299 299 parmObj.updateId(idParm)
300 300
301 301 n += 1
302 302
303 303 def getElementName(self):
304 304
305 305 return self.ELEMENTNAME
306 306
307 307 def getParameterObjList(self):
308 308
309 309 return self.parmConfObjList
310 310
311 311 def getParameterObj(self, parameterName):
312 312
313 313 for parmConfObj in self.parmConfObjList:
314 314
315 315 if parmConfObj.name != parameterName:
316 316 continue
317 317
318 318 return parmConfObj
319 319
320 320 return None
321 321
322 322 def getParameterObjfromValue(self, parameterValue):
323 323
324 324 for parmConfObj in self.parmConfObjList:
325 325
326 326 if parmConfObj.getValue() != parameterValue:
327 327 continue
328 328
329 329 return parmConfObj.getValue()
330 330
331 331 return None
332 332
333 333 def getParameterValue(self, parameterName):
334 334
335 335 parameterObj = self.getParameterObj(parameterName)
336
336
337 337 # if not parameterObj:
338 # return None
339
338 # return None
339
340 340 value = parameterObj.getValue()
341 341
342 342 return value
343 343
344 344
345 345 def getKwargs(self):
346 346
347 347 kwargs = {}
348 348
349 349 for parmConfObj in self.parmConfObjList:
350 350 if self.name == 'run' and parmConfObj.name == 'datatype':
351 351 continue
352 352
353 353 kwargs[parmConfObj.name] = parmConfObj.getValue()
354 354
355 355 return kwargs
356 356
357 357 def setup(self, id, name, priority, type):
358 358
359 359 self.id = str(id)
360 360 self.name = name
361 361 self.type = type
362 362 self.priority = priority
363 363
364 364 self.parmConfObjList = []
365 365
366 366 def removeParameters(self):
367 367
368 368 for obj in self.parmConfObjList:
369 369 del obj
370 370
371 371 self.parmConfObjList = []
372 372
373 373 def addParameter(self, name, value, format='str'):
374 374
375 375 id = self.__getNewId()
376 376
377 377 parmConfObj = ParameterConf()
378 378 if not parmConfObj.setup(id, name, value, format):
379 379 return None
380 380
381 381 self.parmConfObjList.append(parmConfObj)
382 382
383 383 return parmConfObj
384 384
385 385 def changeParameter(self, name, value, format='str'):
386 386
387 387 parmConfObj = self.getParameterObj(name)
388 388 parmConfObj.update(name, value, format)
389 389
390 390 return parmConfObj
391 391
392 392 def makeXml(self, procUnitElement):
393 393
394 394 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
395 395 opElement.set('id', str(self.id))
396 396 opElement.set('name', self.name)
397 397 opElement.set('type', self.type)
398 398 opElement.set('priority', str(self.priority))
399 399
400 400 for parmConfObj in self.parmConfObjList:
401 401 parmConfObj.makeXml(opElement)
402 402
403 403 def readXml(self, opElement):
404 404
405 405 self.id = opElement.get('id')
406 406 self.name = opElement.get('name')
407 407 self.type = opElement.get('type')
408 408 self.priority = opElement.get('priority')
409 409
410 410 #Compatible with old signal chain version
411 411 #Use of 'run' method instead 'init'
412 412 if self.type == 'self' and self.name == 'init':
413 413 self.name = 'run'
414 414
415 415 self.parmConfObjList = []
416 416
417 417 parmElementList = opElement.iter(ParameterConf().getElementName())
418 418
419 419 for parmElement in parmElementList:
420 420 parmConfObj = ParameterConf()
421 421 parmConfObj.readXml(parmElement)
422 422
423 423 #Compatible with old signal chain version
424 424 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
425 425 if self.type != 'self' and self.name == 'Plot':
426 426 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
427 427 self.name = parmConfObj.value
428 428 continue
429 429
430 430 self.parmConfObjList.append(parmConfObj)
431 431
432 432 def printattr(self):
433 433
434 434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
435 435 self.id,
436 436 self.name,
437 437 self.type,
438 438 self.priority)
439 439
440 440 for parmConfObj in self.parmConfObjList:
441 441 parmConfObj.printattr()
442 442
443 443 def createObject(self, plotter_queue=None):
444 444
445 445
446 446 if self.type == 'self':
447 447 raise ValueError, "This operation type cannot be created"
448 448
449 449 if self.type == 'plotter':
450 450 #Plotter(plotter_name)
451 451 if not plotter_queue:
452 452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
453 453
454 454 opObj = Plotter(self.name, plotter_queue)
455 455
456 456 if self.type == 'external' or self.type == 'other':
457 457
458 458 className = eval(self.name)
459 459 kwargs = self.getKwargs()
460 460
461 461 opObj = className(**kwargs)
462 462
463 463 return opObj
464 464
465 465
466 466 class ProcUnitConf():
467 467
468 468 id = None
469 469 name = None
470 470 datatype = None
471 471 inputId = None
472 472 parentId = None
473 473
474 474 opConfObjList = []
475 475
476 476 procUnitObj = None
477 477 opObjList = []
478 478
479 479 ELEMENTNAME = 'ProcUnit'
480 480
481 481 def __init__(self):
482 482
483 483 self.id = None
484 484 self.datatype = None
485 485 self.name = None
486 486 self.inputId = None
487 487
488 488 self.opConfObjList = []
489 489
490 490 self.procUnitObj = None
491 491 self.opObjDict = {}
492 492
493 493 def __getPriority(self):
494 494
495 495 return len(self.opConfObjList)+1
496 496
497 497 def __getNewId(self):
498 498
499 499 return int(self.id)*10 + len(self.opConfObjList) + 1
500 500
501 501 def getElementName(self):
502 502
503 503 return self.ELEMENTNAME
504 504
505 505 def getId(self):
506 506
507 507 return self.id
508 508
509 509 def updateId(self, new_id, parentId=parentId):
510 510
511 511
512 512 new_id = int(parentId)*10 + (int(self.id) % 10)
513 513 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
514 514
515 515 #If this proc unit has not inputs
516 516 if self.inputId == '0':
517 517 new_inputId = 0
518 518
519 519 n = 1
520 520 for opConfObj in self.opConfObjList:
521 521
522 522 idOp = str(int(new_id)*10 + n)
523 523 opConfObj.updateId(idOp)
524 524
525 525 n += 1
526 526
527 527 self.parentId = str(parentId)
528 528 self.id = str(new_id)
529 529 self.inputId = str(new_inputId)
530 530
531 531
532 532 def getInputId(self):
533 533
534 534 return self.inputId
535 535
536 536 def getOperationObjList(self):
537 537
538 538 return self.opConfObjList
539 539
540 540 def getOperationObj(self, name=None):
541 541
542 542 for opConfObj in self.opConfObjList:
543 543
544 544 if opConfObj.name != name:
545 545 continue
546 546
547 547 return opConfObj
548 548
549 549 return None
550 550
551 551 def getOpObjfromParamValue(self, value=None):
552 552
553 553 for opConfObj in self.opConfObjList:
554 554 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
555 555 continue
556 556 return opConfObj
557 557 return None
558 558
559 559 def getProcUnitObj(self):
560 560
561 561 return self.procUnitObj
562 562
563 563 def setup(self, id, name, datatype, inputId, parentId=None):
564 564
565 565 #Compatible with old signal chain version
566 566 if datatype==None and name==None:
567 567 raise ValueError, "datatype or name should be defined"
568 568
569 569 if name==None:
570 570 if 'Proc' in datatype:
571 571 name = datatype
572 572 else:
573 573 name = '%sProc' %(datatype)
574 574
575 575 if datatype==None:
576 576 datatype = name.replace('Proc','')
577 577
578 578 self.id = str(id)
579 579 self.name = name
580 580 self.datatype = datatype
581 581 self.inputId = inputId
582 582 self.parentId = parentId
583 583
584 584 self.opConfObjList = []
585 585
586 586 self.addOperation(name='run', optype='self')
587 587
588 588 def removeOperations(self):
589 589
590 590 for obj in self.opConfObjList:
591 591 del obj
592 592
593 593 self.opConfObjList = []
594 594 self.addOperation(name='run')
595 595
596 596 def addParameter(self, **kwargs):
597 597 '''
598 598 Add parameters to "run" operation
599 599 '''
600 600 opObj = self.opConfObjList[0]
601 601
602 602 opObj.addParameter(**kwargs)
603 603
604 604 return opObj
605 605
606 606 def addOperation(self, name, optype='self'):
607 607
608 608 id = self.__getNewId()
609 609 priority = self.__getPriority()
610 610
611 611 opConfObj = OperationConf()
612 612 opConfObj.setup(id, name=name, priority=priority, type=optype)
613 613
614 614 self.opConfObjList.append(opConfObj)
615 615
616 616 return opConfObj
617 617
618 618 def makeXml(self, projectElement):
619 619
620 620 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
621 621 procUnitElement.set('id', str(self.id))
622 622 procUnitElement.set('name', self.name)
623 623 procUnitElement.set('datatype', self.datatype)
624 624 procUnitElement.set('inputId', str(self.inputId))
625 625
626 626 for opConfObj in self.opConfObjList:
627 627 opConfObj.makeXml(procUnitElement)
628 628
629 629 def readXml(self, upElement):
630 630
631 631 self.id = upElement.get('id')
632 632 self.name = upElement.get('name')
633 633 self.datatype = upElement.get('datatype')
634 634 self.inputId = upElement.get('inputId')
635 635
636 636 if self.ELEMENTNAME == "ReadUnit":
637 637 self.datatype = self.datatype.replace("Reader", "")
638 638
639 639 if self.ELEMENTNAME == "ProcUnit":
640 640 self.datatype = self.datatype.replace("Proc", "")
641 641
642 642 if self.inputId == 'None':
643 643 self.inputId = '0'
644 644
645 645 self.opConfObjList = []
646 646
647 647 opElementList = upElement.iter(OperationConf().getElementName())
648 648
649 649 for opElement in opElementList:
650 650 opConfObj = OperationConf()
651 651 opConfObj.readXml(opElement)
652 652 self.opConfObjList.append(opConfObj)
653 653
654 654 def printattr(self):
655 655
656 656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
657 self.id,
658 self.name,
659 self.datatype,
660 self.inputId)
661
657 self.id,
658 self.name,
659 self.datatype,
660 self.inputId)
661
662 662 for opConfObj in self.opConfObjList:
663 663 opConfObj.printattr()
664 664
665 665
666 666 def getKwargs(self):
667 667
668 668 opObj = self.opConfObjList[0]
669 669 kwargs = opObj.getKwargs()
670 670
671 671 return kwargs
672 672
673 673 def createObjects(self, plotter_queue=None):
674 674
675 675 className = eval(self.name)
676 676 kwargs = self.getKwargs()
677 677 procUnitObj = className(**kwargs)
678 678
679 679 for opConfObj in self.opConfObjList:
680 680
681 681 if opConfObj.type=='self' and self.name=='run':
682 682 continue
683 683 elif opConfObj.type=='self':
684 684 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
685 685 continue
686 686
687 687 opObj = opConfObj.createObject(plotter_queue)
688 688
689 689 self.opObjDict[opConfObj.id] = opObj
690 690
691 691 procUnitObj.addOperation(opObj, opConfObj.id)
692 692
693 693 self.procUnitObj = procUnitObj
694 694
695 695 return procUnitObj
696 696
697 697 ## @profile
698 698 def run(self):
699 699
700 700 is_ok = False
701 701
702 702 for opConfObj in self.opConfObjList:
703 703
704 704 kwargs = {}
705 705 for parmConfObj in opConfObj.getParameterObjList():
706 706 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
707 707 continue
708 708
709 709 kwargs[parmConfObj.name] = parmConfObj.getValue()
710 710
711 711 #ini = time.time()
712 712
713 713 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
714 714 sts = self.procUnitObj.call(opType = opConfObj.type,
715 715 opName = opConfObj.name,
716 716 opId = opConfObj.id,
717 )
718
719 # total_time = time.time() - ini
720 #
721 # if total_time > 0.002:
722 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
723
717 **kwargs)
718
719 # total_time = time.time() - ini
720 #
721 # if total_time > 0.002:
722 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
723
724 724 is_ok = is_ok or sts
725 725
726 726 return is_ok
727 727
728 728 def close(self):
729 729
730 730 for opConfObj in self.opConfObjList:
731 731 if opConfObj.type == 'self':
732 732 continue
733 733
734 734 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
735 735 opObj.close()
736 736
737 737 self.procUnitObj.close()
738 738
739 739 return
740 740
741 741 class ReadUnitConf(ProcUnitConf):
742 742
743 743 path = None
744 744 startDate = None
745 745 endDate = None
746 746 startTime = None
747 747 endTime = None
748 748
749 749 ELEMENTNAME = 'ReadUnit'
750 750
751 751 def __init__(self):
752 752
753 753 self.id = None
754 754 self.datatype = None
755 755 self.name = None
756 756 self.inputId = None
757 757
758 758 self.parentId = None
759 759
760 760 self.opConfObjList = []
761 761 self.opObjList = []
762 762
763 763 def getElementName(self):
764 764
765 765 return self.ELEMENTNAME
766 766
767 767 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
768 768 endTime="", parentId=None, queue=None, server=None, **kwargs):
769 769
770 770 #Compatible with old signal chain version
771 771 if datatype==None and name==None:
772 772 raise ValueError, "datatype or name should be defined"
773 773
774 774 if name==None:
775 775 if 'Reader' in datatype:
776 776 name = datatype
777 777 else:
778 778 name = '%sReader' %(datatype)
779 779 if datatype==None:
780 780 datatype = name.replace('Reader','')
781 781
782 782 self.id = id
783 783 self.name = name
784 784 self.datatype = datatype
785 785 if path != '':
786 786 self.path = os.path.abspath(path)
787 787 self.startDate = startDate
788 788 self.endDate = endDate
789 789 self.startTime = startTime
790 790 self.endTime = endTime
791 791
792 792 self.inputId = '0'
793 793 self.parentId = parentId
794 794 self.queue = queue
795 795 self.server = server
796 796 self.addRunOperation(**kwargs)
797 797
798 798 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
799 799
800 800 #Compatible with old signal chain version
801 801 if datatype==None and name==None:
802 802 raise ValueError, "datatype or name should be defined"
803 803
804 804 if name==None:
805 805 if 'Reader' in datatype:
806 806 name = datatype
807 807 else:
808 808 name = '%sReader' %(datatype)
809 809
810 810 if datatype==None:
811 811 datatype = name.replace('Reader','')
812 812
813 813 self.datatype = datatype
814 814 self.name = name
815 815 self.path = path
816 816 self.startDate = startDate
817 817 self.endDate = endDate
818 818 self.startTime = startTime
819 819 self.endTime = endTime
820 820
821 821 self.inputId = '0'
822 822 self.parentId = parentId
823 823
824 824 self.updateRunOperation(**kwargs)
825 825
826 826 def removeOperations(self):
827 827
828 828 for obj in self.opConfObjList:
829 829 del obj
830 830
831 831 self.opConfObjList = []
832 832
833 833 def addRunOperation(self, **kwargs):
834 834
835 835 opObj = self.addOperation(name = 'run', optype = 'self')
836 836
837 837 if self.server is None:
838 838 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
839 839 opObj.addParameter(name='path' , value=self.path, format='str')
840 840 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
841 841 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
842 842 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
843 843 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
844 844 opObj.addParameter(name='queue' , value=self.queue, format='obj')
845 845 for key, value in kwargs.items():
846 846 opObj.addParameter(name=key, value=value, format=type(value).__name__)
847 847 else:
848 848 opObj.addParameter(name='server' , value=self.server, format='str')
849 849
850 850
851 851 return opObj
852 852
853 853 def updateRunOperation(self, **kwargs):
854 854
855 855 opObj = self.getOperationObj(name = 'run')
856 856 opObj.removeParameters()
857 857
858 858 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
859 859 opObj.addParameter(name='path' , value=self.path, format='str')
860 860 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
861 861 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
862 862 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
863 863 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
864 864
865 865 for key, value in kwargs.items():
866 866 opObj.addParameter(name=key, value=value, format=type(value).__name__)
867 867
868 868 return opObj
869
869
870 870 # def makeXml(self, projectElement):
871 #
871 #
872 872 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
873 873 # procUnitElement.set('id', str(self.id))
874 874 # procUnitElement.set('name', self.name)
875 875 # procUnitElement.set('datatype', self.datatype)
876 876 # procUnitElement.set('inputId', str(self.inputId))
877 #
877 #
878 878 # for opConfObj in self.opConfObjList:
879 879 # opConfObj.makeXml(procUnitElement)
880
880
881 881 def readXml(self, upElement):
882 882
883 883 self.id = upElement.get('id')
884 884 self.name = upElement.get('name')
885 885 self.datatype = upElement.get('datatype')
886 886 self.inputId = upElement.get('inputId')
887 887
888 888 if self.ELEMENTNAME == "ReadUnit":
889 889 self.datatype = self.datatype.replace("Reader", "")
890 890
891 891 if self.inputId == 'None':
892 892 self.inputId = '0'
893 893
894 894 self.opConfObjList = []
895 895
896 896 opElementList = upElement.iter(OperationConf().getElementName())
897 897
898 898 for opElement in opElementList:
899 899 opConfObj = OperationConf()
900 900 opConfObj.readXml(opElement)
901 901 self.opConfObjList.append(opConfObj)
902 902
903 903 if opConfObj.name == 'run':
904 904 self.path = opConfObj.getParameterValue('path')
905 905 self.startDate = opConfObj.getParameterValue('startDate')
906 906 self.endDate = opConfObj.getParameterValue('endDate')
907 907 self.startTime = opConfObj.getParameterValue('startTime')
908 908 self.endTime = opConfObj.getParameterValue('endTime')
909 909
910 910 class Project():
911 911
912 912 id = None
913 913 name = None
914 914 description = None
915 915 filename = None
916 916
917 917 procUnitConfObjDict = None
918 918
919 919 ELEMENTNAME = 'Project'
920 920
921 921 plotterQueue = None
922 922
923 923 def __init__(self, plotter_queue=None):
924 924
925 925 self.id = None
926 926 self.name = None
927 927 self.description = None
928 928
929 929 self.plotterQueue = plotter_queue
930 930
931 931 self.procUnitConfObjDict = {}
932 932
933 933 def __getNewId(self):
934 934
935 935 idList = self.procUnitConfObjDict.keys()
936 936
937 937 id = int(self.id)*10
938 938
939 939 while True:
940 940 id += 1
941 941
942 942 if str(id) in idList:
943 943 continue
944 944
945 945 break
946 946
947 947 return str(id)
948 948
949 949 def getElementName(self):
950 950
951 951 return self.ELEMENTNAME
952 952
953 953 def getId(self):
954 954
955 955 return self.id
956 956
957 957 def updateId(self, new_id):
958 958
959 959 self.id = str(new_id)
960 960
961 961 keyList = self.procUnitConfObjDict.keys()
962 962 keyList.sort()
963 963
964 964 n = 1
965 965 newProcUnitConfObjDict = {}
966 966
967 967 for procKey in keyList:
968 968
969 969 procUnitConfObj = self.procUnitConfObjDict[procKey]
970 970 idProcUnit = str(int(self.id)*10 + n)
971 971 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
972 972
973 973 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
974 974 n += 1
975 975
976 976 self.procUnitConfObjDict = newProcUnitConfObjDict
977 977
978 978 def setup(self, id, name, description):
979 979
980 980 self.id = str(id)
981 981 self.name = name
982 982 self.description = description
983 983
984 984 def update(self, name, description):
985 985
986 986 self.name = name
987 987 self.description = description
988 988
989 989 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
990 990
991 991 if id is None:
992 992 idReadUnit = self.__getNewId()
993 993 else:
994 994 idReadUnit = str(id)
995 995
996 996 readUnitConfObj = ReadUnitConf()
997 997 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
998 998
999 999 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1000 1000
1001 1001 return readUnitConfObj
1002 1002
1003 1003 def addProcUnit(self, inputId='0', datatype=None, name=None):
1004 1004
1005 1005 idProcUnit = self.__getNewId()
1006 1006
1007 1007 procUnitConfObj = ProcUnitConf()
1008 1008 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1009 1009
1010 1010 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1011 1011
1012 1012 return procUnitConfObj
1013 1013
1014 1014 def removeProcUnit(self, id):
1015 1015
1016 1016 if id in self.procUnitConfObjDict.keys():
1017 1017 self.procUnitConfObjDict.pop(id)
1018 1018
1019 1019 def getReadUnitId(self):
1020 1020
1021 1021 readUnitConfObj = self.getReadUnitObj()
1022 1022
1023 1023 return readUnitConfObj.id
1024 1024
1025 1025 def getReadUnitObj(self):
1026 1026
1027 1027 for obj in self.procUnitConfObjDict.values():
1028 1028 if obj.getElementName() == "ReadUnit":
1029 1029 return obj
1030 1030
1031 1031 return None
1032 1032
1033 1033 def getProcUnitObj(self, id=None, name=None):
1034 1034
1035 1035 if id != None:
1036 1036 return self.procUnitConfObjDict[id]
1037 1037
1038 1038 if name != None:
1039 1039 return self.getProcUnitObjByName(name)
1040 1040
1041 1041 return None
1042 1042
1043 1043 def getProcUnitObjByName(self, name):
1044 1044
1045 1045 for obj in self.procUnitConfObjDict.values():
1046 1046 if obj.name == name:
1047 1047 return obj
1048 1048
1049 1049 return None
1050 1050
1051 1051 def procUnitItems(self):
1052 1052
1053 1053 return self.procUnitConfObjDict.items()
1054 1054
1055 1055 def makeXml(self):
1056 1056
1057 1057 projectElement = Element('Project')
1058 1058 projectElement.set('id', str(self.id))
1059 1059 projectElement.set('name', self.name)
1060 1060 projectElement.set('description', self.description)
1061 1061
1062 1062 for procUnitConfObj in self.procUnitConfObjDict.values():
1063 1063 procUnitConfObj.makeXml(projectElement)
1064 1064
1065 1065 self.projectElement = projectElement
1066 1066
1067 1067 def writeXml(self, filename=None):
1068 1068
1069 1069 if filename == None:
1070 1070 if self.filename:
1071 1071 filename = self.filename
1072 1072 else:
1073 1073 filename = "schain.xml"
1074 1074
1075 1075 if not filename:
1076 1076 print "filename has not been defined. Use setFilename(filename) for do it."
1077 1077 return 0
1078 1078
1079 1079 abs_file = os.path.abspath(filename)
1080 1080
1081 1081 if not os.access(os.path.dirname(abs_file), os.W_OK):
1082 1082 print "No write permission on %s" %os.path.dirname(abs_file)
1083 1083 return 0
1084 1084
1085 1085 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1086 1086 print "File %s already exists and it could not be overwriten" %abs_file
1087 1087 return 0
1088 1088
1089 1089 self.makeXml()
1090 1090
1091 1091 ElementTree(self.projectElement).write(abs_file, method='xml')
1092 1092
1093 1093 self.filename = abs_file
1094 1094
1095 1095 return 1
1096 1096
1097 1097 def readXml(self, filename = None):
1098 1098
1099 1099 if not filename:
1100 1100 print "filename is not defined"
1101 1101 return 0
1102 1102
1103 1103 abs_file = os.path.abspath(filename)
1104 1104
1105 1105 if not os.path.isfile(abs_file):
1106 1106 print "%s file does not exist" %abs_file
1107 1107 return 0
1108 1108
1109 1109 self.projectElement = None
1110 1110 self.procUnitConfObjDict = {}
1111 1111
1112 1112 try:
1113 1113 self.projectElement = ElementTree().parse(abs_file)
1114 1114 except:
1115 1115 print "Error reading %s, verify file format" %filename
1116 1116 return 0
1117 1117
1118 1118 self.project = self.projectElement.tag
1119 1119
1120 1120 self.id = self.projectElement.get('id')
1121 1121 self.name = self.projectElement.get('name')
1122 self.description = self.projectElement.get('description')
1123
1122 self.description = self.projectElement.get('description')
1123
1124 1124 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1125 1125
1126 1126 for readUnitElement in readUnitElementList:
1127 1127 readUnitConfObj = ReadUnitConf()
1128 1128 readUnitConfObj.readXml(readUnitElement)
1129 1129
1130 1130 if readUnitConfObj.parentId == None:
1131 1131 readUnitConfObj.parentId = self.id
1132 1132
1133 1133 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1134 1134
1135 1135 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1136 1136
1137 1137 for procUnitElement in procUnitElementList:
1138 1138 procUnitConfObj = ProcUnitConf()
1139 1139 procUnitConfObj.readXml(procUnitElement)
1140 1140
1141 1141 if procUnitConfObj.parentId == None:
1142 1142 procUnitConfObj.parentId = self.id
1143 1143
1144 1144 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1145 1145
1146 1146 self.filename = abs_file
1147 1147
1148 1148 return 1
1149 1149
1150 1150 def printattr(self):
1151 1151
1152 1152 print "Project[%s]: name = %s, description = %s" %(self.id,
1153 self.name,
1154 self.description)
1153 self.name,
1154 self.description)
1155 1155
1156 1156 for procUnitConfObj in self.procUnitConfObjDict.values():
1157 1157 procUnitConfObj.printattr()
1158 1158
1159 1159 def createObjects(self):
1160 1160
1161 1161 for procUnitConfObj in self.procUnitConfObjDict.values():
1162 1162 procUnitConfObj.createObjects(self.plotterQueue)
1163 1163
1164 1164 def __connect(self, objIN, thisObj):
1165 1165
1166 1166 thisObj.setInput(objIN.getOutputObj())
1167 1167
1168 1168 def connectObjects(self):
1169 1169
1170 1170 for thisPUConfObj in self.procUnitConfObjDict.values():
1171 1171
1172 1172 inputId = thisPUConfObj.getInputId()
1173 1173
1174 1174 if int(inputId) == 0:
1175 1175 continue
1176 1176
1177 1177 #Get input object
1178 1178 puConfINObj = self.procUnitConfObjDict[inputId]
1179 1179 puObjIN = puConfINObj.getProcUnitObj()
1180 1180
1181 1181 #Get current object
1182 1182 thisPUObj = thisPUConfObj.getProcUnitObj()
1183 1183
1184 1184 self.__connect(puObjIN, thisPUObj)
1185 1185
1186 1186 def __handleError(self, procUnitConfObj, send_email=True):
1187 1187
1188 1188 import socket
1189 1189
1190 1190 err = traceback.format_exception(sys.exc_info()[0],
1191 sys.exc_info()[1],
1192 sys.exc_info()[2])
1193
1191 sys.exc_info()[1],
1192 sys.exc_info()[2])
1193
1194 1194 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1195 1195 print "***** %s" %err[-1]
1196 1196
1197 1197 message = "".join(err)
1198 1198
1199 1199 sys.stderr.write(message)
1200 1200
1201 1201 if not send_email:
1202 1202 return
1203 1203
1204 1204 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1205 1205
1206 1206 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1207 1207 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1208 1208 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1209 1209 subtitle += "Configuration file: %s\n" %self.filename
1210 1210 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1211 1211
1212 1212 readUnitConfObj = self.getReadUnitObj()
1213 1213 if readUnitConfObj:
1214 1214 subtitle += "\nInput parameters:\n"
1215 1215 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1216 1216 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1217 1217 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1218 1218 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1219 1219 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1220 1220 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1221 1221
1222 1222 adminObj = schainpy.admin.SchainNotify()
1223 1223 adminObj.sendAlert(message=message,
1224 subject=subject,
1225 subtitle=subtitle,
1226 filename=self.filename)
1227
1224 subject=subject,
1225 subtitle=subtitle,
1226 filename=self.filename)
1227
1228 1228 def isPaused(self):
1229 1229 return 0
1230 1230
1231 1231 def isStopped(self):
1232 1232 return 0
1233 1233
1234 1234 def runController(self):
1235 1235 """
1236 1236 returns 0 when this process has been stopped, 1 otherwise
1237 1237 """
1238 1238
1239 1239 if self.isPaused():
1240 1240 print "Process suspended"
1241 1241
1242 1242 while True:
1243 1243 sleep(0.1)
1244 1244
1245 1245 if not self.isPaused():
1246 1246 break
1247 1247
1248 1248 if self.isStopped():
1249 1249 break
1250 1250
1251 1251 print "Process reinitialized"
1252 1252
1253 1253 if self.isStopped():
1254 1254 print "Process stopped"
1255 1255 return 0
1256 1256
1257 1257 return 1
1258 1258
1259 1259 def setFilename(self, filename):
1260 1260
1261 1261 self.filename = filename
1262 1262
1263 1263 def setPlotterQueue(self, plotter_queue):
1264 1264
1265 1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1266 1266
1267 1267 def getPlotterQueue(self):
1268 1268
1269 1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1270 1270
1271 1271 def useExternalPlotter(self):
1272 1272
1273 1273 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1274 1274
1275 1275
1276 1276 def run(self):
1277 1277
1278 1278 print
1279 1279 print "*"*60
1280 1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 1281 print "*"*60
1282 1282 print
1283 1283
1284 1284 keyList = self.procUnitConfObjDict.keys()
1285 1285 keyList.sort()
1286 1286
1287 1287 while(True):
1288 1288
1289 1289 is_ok = False
1290 1290
1291 1291 for procKey in keyList:
1292 1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293 1293
1294 1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 1295
1296 1296 try:
1297 1297 sts = procUnitConfObj.run()
1298 1298 is_ok = is_ok or sts
1299 1299 except KeyboardInterrupt:
1300 1300 is_ok = False
1301 1301 break
1302 1302 except ValueError, e:
1303 1303 sleep(0.5)
1304 1304 self.__handleError(procUnitConfObj, send_email=True)
1305 1305 is_ok = False
1306 1306 break
1307 1307 except:
1308 1308 sleep(0.5)
1309 1309 self.__handleError(procUnitConfObj)
1310 1310 is_ok = False
1311 1311 break
1312 1312
1313 1313 #If every process unit finished so end process
1314 1314 if not(is_ok):
1315 1315 # print "Every process unit have finished"
1316 1316 break
1317 1317
1318 1318 if not self.runController():
1319 1319 break
1320 1320
1321 1321 #Closing every process
1322 1322 for procKey in keyList:
1323 1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 1324 procUnitConfObj.close()
1325 1325
1326 1326 print "Process finished"
1327 1327
1328 1328 def start(self, filename=None):
1329 1329
1330 1330 self.writeXml(filename)
1331 1331 self.createObjects()
1332 1332 self.connectObjects()
1333 1333 self.run()
Show file before | Show file after | Hide comments
@@ -1,179 +1,179
1 1 import threading
2 2 from Queue import Queue
3 3
4 4 from schainpy.controller import Project
5 5 from schainpy.model.graphics.jroplotter import PlotManager
6 6
7 7 class ControllerThread(threading.Thread, Project):
8 8
9 9 def __init__(self, plotter_queue=None):
10 10
11 11 threading.Thread.__init__(self)
12 12 Project.__init__(self, plotter_queue)
13 13
14 14 self.setDaemon(True)
15 15
16 16 self.lock = threading.Lock()
17 self.control = {'stop':False, 'pause':False}
18
17 self.control = { 'stop':False, 'pause':False }
18
19 19 def __del__(self):
20 20
21 21 self.control['stop'] = True
22 22
23 23 def stop(self):
24 24
25 25 self.lock.acquire()
26 26
27 27 self.control['stop'] = True
28 28
29 29 self.lock.release()
30 30
31 31 def pause(self):
32 32
33 33 self.lock.acquire()
34 34
35 35 self.control['pause'] = not(self.control['pause'])
36 36 paused = self.control['pause']
37 37
38 38 self.lock.release()
39 39
40 40 return paused
41 41
42 42 def isPaused(self):
43 43
44 44 self.lock.acquire()
45 45 paused = self.control['pause']
46 46 self.lock.release()
47 47
48 48 return paused
49 49
50 50 def isStopped(self):
51 51
52 52 self.lock.acquire()
53 53 stopped = self.control['stop']
54 54 self.lock.release()
55 55
56 56 return stopped
57 57
58 58 def run(self):
59 59 self.control['stop'] = False
60 60 self.control['pause'] = False
61 61
62 62 self.writeXml()
63 63
64 64 self.createObjects()
65 65 self.connectObjects()
66 66 Project.run(self)
67 67
68 68 def isRunning(self):
69 69
70 70 return self.is_alive()
71 71
72 72 def isFinished(self):
73 73
74 74 return not self.is_alive()
75 75
76 76 def setPlotters(self):
77 77
78 78 plotterList = PlotManager.plotterList
79 79
80 80 for thisPUConfObj in self.procUnitConfObjDict.values():
81 81
82 82 inputId = thisPUConfObj.getInputId()
83 83
84 84 if int(inputId) == 0:
85 85 continue
86 86
87 87 for thisOpObj in thisPUConfObj.getOperationObjList():
88 88
89 89 if thisOpObj.type == "self":
90 90 continue
91 91
92 92 if thisOpObj.name in plotterList:
93 thisOpObj.type = "plotter"
93 thisOpObj.type = "other"
94 94
95 95 def setPlotterQueue(self, plotter_queue):
96 96
97 97 self.plotterQueue = plotter_queue
98 98
99 99 def getPlotterQueue(self):
100 100
101 101 return self.plotterQueue
102 102
103 103 def useExternalPlotter(self):
104 104
105 105 self.plotterQueue = Queue(10)
106 106 self.setPlotters()
107 107
108 108 plotManagerObj = PlotManager(self.plotterQueue)
109 109 plotManagerObj.setController(self)
110 110
111 111 return plotManagerObj
112 112
113 113 # from PyQt4 import QtCore
114 114 # from PyQt4.QtCore import SIGNAL
115 115 #
116 116 # class ControllerQThread(QtCore.QThread, Project):
117 117 #
118 118 # def __init__(self, filename):
119 119 #
120 120 # QtCore.QThread.__init__(self)
121 121 # Project.__init__(self)
122 122 #
123 123 # self.filename = filename
124 124 #
125 125 # self.lock = threading.Lock()
126 126 # self.control = {'stop':False, 'pause':False}
127 127 #
128 128 # def __del__(self):
129 129 #
130 130 # self.control['stop'] = True
131 131 # self.wait()
132 132 #
133 133 # def stop(self):
134 134 #
135 135 # self.lock.acquire()
136 136 #
137 137 # self.control['stop'] = True
138 138 #
139 139 # self.lock.release()
140 140 #
141 141 # def pause(self):
142 142 #
143 143 # self.lock.acquire()
144 144 #
145 145 # self.control['pause'] = not(self.control['pause'])
146 146 # paused = self.control['pause']
147 147 #
148 148 # self.lock.release()
149 149 #
150 150 # return paused
151 151 #
152 152 # def isPaused(self):
153 153 #
154 154 # self.lock.acquire()
155 155 # paused = self.control['pause']
156 156 # self.lock.release()
157 157 #
158 158 # return paused
159 159 #
160 160 # def isStopped(self):
161 161 #
162 162 # self.lock.acquire()
163 163 # stopped = self.control['stop']
164 164 # self.lock.release()
165 165 #
166 166 # return stopped
167 167 #
168 168 # def run(self):
169 169 #
170 170 # self.control['stop'] = False
171 171 # self.control['pause'] = False
172 172 #
173 173 # self.readXml(self.filename)
174 174 # self.createObjects()
175 175 # self.connectObjects()
176 176 # self.emit( SIGNAL( "jobStarted( PyQt_PyObject )" ), 1)
177 177 # Project.run(self)
178 178 # self.emit( SIGNAL( "jobFinished( PyQt_PyObject )" ), 1)
179 179 #
Show file before | Show file after | Hide comments
@@ -1,240 +1,240
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os, sys
7 7 import datetime
8 8 import numpy
9 9 import traceback
10 10
11 11 from time import sleep
12 12 from threading import Lock
13 13 # from threading import Thread
14 14
15 15 import schainpy
16 16 import schainpy.admin
17 17
18 18 from schainpy.model.proc.jroproc_base import Operation
19 19 from schainpy.model.serializer.data import obj2Dict, dict2Obj
20 20 from jroplot_correlation import *
21 21 from jroplot_heispectra import *
22 22 from jroplot_parameters import *
23 23 from jroplot_spectra import *
24 24 from jroplot_voltage import *
25 25
26 26
27 27 class Plotter(Operation):
28 28
29 29 isConfig = None
30 30 name = None
31 31 __queue = None
32 32
33 33 def __init__(self, plotter_name, plotter_queue=None, **kwargs):
34 34
35 35 Operation.__init__(self, **kwargs)
36 36
37 37 self.isConfig = False
38 38 self.name = plotter_name
39 39 self.__queue = plotter_queue
40 40
41 41 def getSubplots(self):
42 42
43 43 nrow = self.nplots
44 44 ncol = 1
45 45 return nrow, ncol
46 46
47 47 def setup(self, **kwargs):
48 48
49 49 print "Initializing ..."
50 50
51 51
52 52 def run(self, dataOut, id=None, **kwargs):
53 53
54 54 """
55 55
56 56 Input:
57 57 dataOut :
58 58 id :
59 59 """
60 60
61 61 packDict = {}
62 62
63 63 packDict['id'] = id
64 64 packDict['name'] = self.name
65 65 packDict['kwargs'] = kwargs
66 66
67 67 # packDict['data'] = obj2Dict(dataOut)
68 68 packDict['data'] = dataOut
69 69
70 70 self.__queue.put(packDict)
71 71
72 72 # class PlotManager(Thread):
73 73 class PlotManager():
74 74
75 75 __err = False
76 76 __stop = False
77 77 __realtime = False
78 78
79 79 controllerThreadObj = None
80 80
81 81 plotterList = ['Scope',
82 82 'SpectraPlot', 'RTIPlot',
83 83 'SpectraCutPlot',
84 84 'CrossSpectraPlot', 'CoherenceMap',
85 85 'PowerProfilePlot', 'Noise', 'BeaconPhase',
86 86 'CorrelationPlot',
87 87 'SpectraHeisScope', 'RTIfromSpectraHeis']
88 88
89 89 def __init__(self, plotter_queue):
90 90
91 91 # Thread.__init__(self)
92 92 # self.setDaemon(True)
93 93
94 94 self.__queue = plotter_queue
95 95 self.__lock = Lock()
96 96
97 97 self.plotInstanceDict = {}
98 98
99 99 self.__err = False
100 100 self.__stop = False
101 101 self.__realtime = False
102 102
103 103 def __handleError(self, name="", send_email=False):
104 104
105 105 err = traceback.format_exception(sys.exc_info()[0],
106 106 sys.exc_info()[1],
107 107 sys.exc_info()[2])
108 108
109 109 print "***** Error occurred in PlotManager *****"
110 110 print "***** [%s]: %s" %(name, err[-1])
111 111
112 112 message = "\nError ocurred in %s:\n" %name
113 113 message += "".join(err)
114 114
115 115 sys.stderr.write(message)
116 116
117 117 if not send_email:
118 118 return
119 119
120 120 import socket
121 121
122 122 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, name)
123 123
124 124 subtitle = "%s:\n" %(name)
125 125 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
126 126 subtitle += "Working directory: %s\n" %os.path.abspath("./")
127 # subtitle += "Configuration file: %s\n" %self.filename
127 # subtitle += "Configuration file: %s\n" %self.filename
128 128 subtitle += "Time: %s\n" %str(datetime.datetime.now())
129 129
130 130 adminObj = schainpy.admin.SchainNotify()
131 131 adminObj.sendAlert(message=message,
132 132 subject=subject,
133 133 subtitle=subtitle)
134 134
135 135 def run(self):
136 136
137 137 if self.__queue.empty():
138 138 return
139 139
140 140 if self.__err:
141 141 serial_data = self.__queue.get()
142 142 self.__queue.task_done()
143 143 return
144 144
145 145 self.__lock.acquire()
146 146
147 147 # if self.__queue.full():
148 148 # for i in range(int(self.__queue.qsize()/2)):
149 149 # serial_data = self.__queue.get()
150 150 # self.__queue.task_done()
151 151
152 152 n = int(self.__queue.qsize()/3 + 1)
153 153
154 154 for i in range(n):
155 155
156 156 if self.__queue.empty():
157 157 break
158 158
159 159 serial_data = self.__queue.get()
160 160 self.__queue.task_done()
161 161
162 162 plot_id = serial_data['id']
163 163 plot_name = serial_data['name']
164 164 kwargs = serial_data['kwargs']
165 165 # dataDict = serial_data['data']
166 166 #
167 167 # dataPlot = dict2Obj(dataDict)
168 168
169 169 dataPlot = serial_data['data']
170 170
171 171 if plot_id not in self.plotInstanceDict.keys():
172 172 className = eval(plot_name)
173 173 self.plotInstanceDict[plot_id] = className(**kwargs)
174 174
175 175 plotter = self.plotInstanceDict[plot_id]
176 176 try:
177 177 plotter.run(dataPlot, plot_id, **kwargs)
178 178 except:
179 179 self.__err = True
180 180 self.__handleError(plot_name, send_email=True)
181 181 break
182 182
183 183 self.__lock.release()
184 184
185 185 def isEmpty(self):
186 186
187 187 return self.__queue.empty()
188 188
189 189 def stop(self):
190 190
191 191 self.__lock.acquire()
192 192
193 193 self.__stop = True
194 194
195 195 self.__lock.release()
196 196
197 197 def close(self):
198 198
199 199 self.__lock.acquire()
200 200
201 201 for plot_id in self.plotInstanceDict.keys():
202 202 plotter = self.plotInstanceDict[plot_id]
203 203 plotter.close()
204 204
205 205 self.__lock.release()
206 206
207 207 def setController(self, controllerThreadObj):
208 208
209 209 self.controllerThreadObj = controllerThreadObj
210 210
211 211 def start(self):
212 212
213 213 if not self.controllerThreadObj.isRunning():
214 214 raise RuntimeError, "controllerThreadObj has not been initialized. Use controllerThreadObj.start() before call this method"
215 215
216 216 self.join()
217 217
218 218 def join(self):
219 219
220 220 #Execute plotter while controller is running
221 221 while self.controllerThreadObj.isRunning():
222 222 self.run()
223 223
224 224 self.controllerThreadObj.stop()
225 225
226 226 #Wait until plotter queue is empty
227 227 while not self.isEmpty():
228 228 self.run()
229 229
230 230 self.close()
231 231
232 232 def isErrorDetected(self):
233 233
234 234 self.__lock.acquire()
235 235
236 236 err = self.__err
237 237
238 238 self.__lock.release()
239 239
240 240 return err
Show file before | Show file after | Hide comments
@@ -1,1813 +1,1813
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 14 import traceback
15 15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 24
25 25 LOCALTIME = True
26 26
27 27 def isNumber(cad):
28 28 """
29 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 30
31 31 Excepciones:
32 32 Si un determinado string no puede ser convertido a numero
33 33 Input:
34 34 str, string al cual se le analiza para determinar si convertible a un numero o no
35 35
36 36 Return:
37 37 True : si el string es uno numerico
38 38 False : no es un string numerico
39 39 """
40 40 try:
41 41 float( cad )
42 42 return True
43 43 except:
44 44 return False
45 45
46 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 47 """
48 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 49
50 50 Inputs:
51 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 52
53 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 56 segundos contados desde 01/01/1970.
57 57
58 58 Return:
59 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 60 fecha especificado, de lo contrario retorna False.
61 61
62 62 Excepciones:
63 63 Si el archivo no existe o no puede ser abierto
64 64 Si la cabecera no puede ser leida.
65 65
66 66 """
67 67 basicHeaderObj = BasicHeader(LOCALTIME)
68 68
69 69 try:
70 70 fp = open(filename,'rb')
71 71 except IOError:
72 72 print "The file %s can't be opened" %(filename)
73 73 return 0
74 74
75 75 sts = basicHeaderObj.read(fp)
76 76 fp.close()
77 77
78 78 if not(sts):
79 79 print "Skipping the file %s because it has not a valid header" %(filename)
80 80 return 0
81 81
82 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 83 return 0
84 84
85 85 return 1
86 86
87 87 def isTimeInRange(thisTime, startTime, endTime):
88 88
89 89 if endTime >= startTime:
90 90 if (thisTime < startTime) or (thisTime > endTime):
91 91 return 0
92 92
93 93 return 1
94 94 else:
95 95 if (thisTime < startTime) and (thisTime > endTime):
96 96 return 0
97 97
98 98 return 1
99 99
100 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 101 """
102 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 103
104 104 Inputs:
105 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 106
107 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 108
109 109 endDate : fecha final del rango seleccionado en formato datetime.date
110 110
111 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 112
113 113 endTime : tiempo final del rango seleccionado en formato datetime.time
114 114
115 115 Return:
116 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 117 fecha especificado, de lo contrario retorna False.
118 118
119 119 Excepciones:
120 120 Si el archivo no existe o no puede ser abierto
121 121 Si la cabecera no puede ser leida.
122 122
123 123 """
124 124
125 125
126 126 try:
127 127 fp = open(filename,'rb')
128 128 except IOError:
129 129 print "The file %s can't be opened" %(filename)
130 130 return None
131 131
132 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 133 systemHeaderObj = SystemHeader()
134 134 radarControllerHeaderObj = RadarControllerHeader()
135 135 processingHeaderObj = ProcessingHeader()
136 136
137 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 138
139 139 sts = firstBasicHeaderObj.read(fp)
140 140
141 141 if not(sts):
142 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 143 return None
144 144
145 145 if not systemHeaderObj.read(fp):
146 146 return None
147 147
148 148 if not radarControllerHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not processingHeaderObj.read(fp):
152 152 return None
153 153
154 154 filesize = os.path.getsize(filename)
155 155
156 156 offset = processingHeaderObj.blockSize + 24 #header size
157 157
158 158 if filesize <= offset:
159 159 print "[Reading] %s: This file has not enough data" %filename
160 160 return None
161 161
162 162 fp.seek(-offset, 2)
163 163
164 164 sts = lastBasicHeaderObj.read(fp)
165 165
166 166 fp.close()
167 167
168 168 thisDatetime = lastBasicHeaderObj.datatime
169 169 thisTime_last_block = thisDatetime.time()
170 170
171 171 thisDatetime = firstBasicHeaderObj.datatime
172 172 thisDate = thisDatetime.date()
173 173 thisTime_first_block = thisDatetime.time()
174 174
175 175 #General case
176 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 177 #-----------o----------------------------o-----------
178 178 # startTime endTime
179 179
180 180 if endTime >= startTime:
181 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 182 return None
183 183
184 184 return thisDatetime
185 185
186 186 #If endTime < startTime then endTime belongs to the next day
187 187
188 188
189 189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 190 #-----------o----------------------------o-----------
191 191 # endTime startTime
192 192
193 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 194 return None
195 195
196 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 197 return None
198 198
199 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 200 return None
201 201
202 202 return thisDatetime
203 203
204 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 205 """
206 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 207
208 208 Inputs:
209 209 folder : nombre completo del directorio.
210 210 Su formato deberia ser "/path_root/?YYYYDDD"
211 211
212 212 siendo:
213 213 YYYY : Anio (ejemplo 2015)
214 214 DDD : Dia del anio (ejemplo 305)
215 215
216 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 217
218 218 endDate : fecha final del rango seleccionado en formato datetime.date
219 219
220 220 Return:
221 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 222 fecha especificado, de lo contrario retorna False.
223 223 Excepciones:
224 224 Si el directorio no tiene el formato adecuado
225 225 """
226 226
227 227 basename = os.path.basename(folder)
228 228
229 229 if not isRadarFolder(basename):
230 230 print "The folder %s has not the rigth format" %folder
231 231 return 0
232 232
233 233 if startDate and endDate:
234 234 thisDate = getDateFromRadarFolder(basename)
235 235
236 236 if thisDate < startDate:
237 237 return 0
238 238
239 239 if thisDate > endDate:
240 240 return 0
241 241
242 242 return 1
243 243
244 244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 245 """
246 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 247
248 248 Inputs:
249 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 250
251 251 Su formato deberia ser "?YYYYDDDsss"
252 252
253 253 siendo:
254 254 YYYY : Anio (ejemplo 2015)
255 255 DDD : Dia del anio (ejemplo 305)
256 256 sss : set
257 257
258 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 259
260 260 endDate : fecha final del rango seleccionado en formato datetime.date
261 261
262 262 Return:
263 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 264 fecha especificado, de lo contrario retorna False.
265 265 Excepciones:
266 266 Si el archivo no tiene el formato adecuado
267 267 """
268 268
269 269 basename = os.path.basename(filename)
270 270
271 271 if not isRadarFile(basename):
272 272 print "The filename %s has not the rigth format" %filename
273 273 return 0
274 274
275 275 if startDate and endDate:
276 276 thisDate = getDateFromRadarFile(basename)
277 277
278 278 if thisDate < startDate:
279 279 return 0
280 280
281 281 if thisDate > endDate:
282 282 return 0
283 283
284 284 return 1
285 285
286 286 def getFileFromSet(path, ext, set):
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294 try:
295 295 year = int(thisFile[1:5])
296 296 doy = int(thisFile[5:8])
297 297 except:
298 298 continue
299 299
300 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 301 continue
302 302
303 303 validFilelist.append(thisFile)
304 304
305 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 306
307 307 if len(myfile)!= 0:
308 308 return myfile[0]
309 309 else:
310 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 311 print 'the filename %s does not exist'%filename
312 312 print '...going to the last file: '
313 313
314 314 if validFilelist:
315 315 validFilelist = sorted( validFilelist, key=str.lower )
316 316 return validFilelist[-1]
317 317
318 318 return None
319 319
320 320 def getlastFileFromPath(path, ext):
321 321 """
322 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 324
325 325 Input:
326 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 327 ext : extension de los files contenidos en una carpeta
328 328
329 329 Return:
330 330 El ultimo file de una determinada carpeta, no se considera el path.
331 331 """
332 332 validFilelist = []
333 333 fileList = os.listdir(path)
334 334
335 335 # 0 1234 567 89A BCDE
336 336 # H YYYY DDD SSS .ext
337 337
338 338 for thisFile in fileList:
339 339
340 340 year = thisFile[1:5]
341 341 if not isNumber(year):
342 342 continue
343 343
344 344 doy = thisFile[5:8]
345 345 if not isNumber(doy):
346 346 continue
347 347
348 348 year = int(year)
349 349 doy = int(doy)
350 350
351 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 352 continue
353 353
354 354 validFilelist.append(thisFile)
355 355
356 356 if validFilelist:
357 357 validFilelist = sorted( validFilelist, key=str.lower )
358 358 return validFilelist[-1]
359 359
360 360 return None
361 361
362 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 """
364 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 366 el path exacto de un determinado file.
367 367
368 368 Example :
369 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 370
371 371 Entonces la funcion prueba con las siguientes combinaciones
372 372 .../.../y2009307367.ext
373 373 .../.../Y2009307367.ext
374 374 .../.../x2009307/y2009307367.ext
375 375 .../.../x2009307/Y2009307367.ext
376 376 .../.../X2009307/y2009307367.ext
377 377 .../.../X2009307/Y2009307367.ext
378 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 379
380 380 Return:
381 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 383 para el filename
384 384 """
385 385 fullfilename = None
386 386 find_flag = False
387 387 filename = None
388 388
389 389 prefixDirList = [None,'d','D']
390 390 if ext.lower() == ".r": #voltage
391 391 prefixFileList = ['d','D']
392 392 elif ext.lower() == ".pdata": #spectra
393 393 prefixFileList = ['p','P']
394 394 else:
395 395 return None, filename
396 396
397 397 #barrido por las combinaciones posibles
398 398 for prefixDir in prefixDirList:
399 399 thispath = path
400 400 if prefixDir != None:
401 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 402 if foldercounter == 0:
403 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 404 else:
405 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 409
410 410 if os.path.exists( fullfilename ): #verifico que exista
411 411 find_flag = True
412 412 break
413 413 if find_flag:
414 414 break
415 415
416 416 if not(find_flag):
417 417 return None, filename
418 418
419 419 return fullfilename, filename
420 420
421 421 def isRadarFolder(folder):
422 422 try:
423 423 year = int(folder[1:5])
424 424 doy = int(folder[5:8])
425 425 except:
426 426 return 0
427 427
428 428 return 1
429 429
430 430 def isRadarFile(file):
431 431 try:
432 432 year = int(file[1:5])
433 433 doy = int(file[5:8])
434 434 set = int(file[8:11])
435 435 except:
436 436 return 0
437 437
438 438 return 1
439 439
440 440 def getDateFromRadarFile(file):
441 441 try:
442 442 year = int(file[1:5])
443 443 doy = int(file[5:8])
444 444 set = int(file[8:11])
445 445 except:
446 446 return None
447 447
448 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 449 return thisDate
450 450
451 451 def getDateFromRadarFolder(folder):
452 452 try:
453 453 year = int(folder[1:5])
454 454 doy = int(folder[5:8])
455 455 except:
456 456 return None
457 457
458 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 459 return thisDate
460 460
461 461 class JRODataIO:
462 462
463 463 c = 3E8
464 464
465 465 isConfig = False
466 466
467 467 basicHeaderObj = None
468 468
469 469 systemHeaderObj = None
470 470
471 471 radarControllerHeaderObj = None
472 472
473 473 processingHeaderObj = None
474 474
475 475 dtype = None
476 476
477 477 pathList = []
478 478
479 479 filenameList = []
480 480
481 481 filename = None
482 482
483 483 ext = None
484 484
485 485 flagIsNewFile = 1
486 486
487 487 flagDiscontinuousBlock = 0
488 488
489 489 flagIsNewBlock = 0
490 490
491 491 fp = None
492 492
493 493 firstHeaderSize = 0
494 494
495 495 basicHeaderSize = 24
496 496
497 497 versionFile = 1103
498 498
499 499 fileSize = None
500 500
501 501 # ippSeconds = None
502 502
503 503 fileSizeByHeader = None
504 504
505 505 fileIndex = None
506 506
507 507 profileIndex = None
508 508
509 509 blockIndex = None
510 510
511 511 nTotalBlocks = None
512 512
513 513 maxTimeStep = 30
514 514
515 515 lastUTTime = None
516 516
517 517 datablock = None
518 518
519 519 dataOut = None
520 520
521 521 blocksize = None
522 522
523 523 getByBlock = False
524 524
525 525 def __init__(self):
526 526
527 527 raise NotImplementedError
528 528
529 529 def run(self):
530 530
531 531 raise NotImplementedError
532 532
533 533 def getDtypeWidth(self):
534 534
535 535 dtype_index = get_dtype_index(self.dtype)
536 536 dtype_width = get_dtype_width(dtype_index)
537 537
538 538 return dtype_width
539 539
540 540 def getAllowedArgs(self):
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544 544
545 545
546 546 online = 0
547 547
548 548 realtime = 0
549 549
550 550 nReadBlocks = 0
551 551
552 552 delay = 10 #number of seconds waiting a new file
553 553
554 554 nTries = 3 #quantity tries
555 555
556 556 nFiles = 3 #number of files for searching
557 557
558 558 path = None
559 559
560 560 foldercounter = 0
561 561
562 562 flagNoMoreFiles = 0
563 563
564 564 datetimeList = []
565 565
566 566 __isFirstTimeOnline = 1
567 567
568 568 __printInfo = True
569 569
570 570 profileIndex = None
571 571
572 572 nTxs = 1
573 573
574 574 txIndex = None
575 575
576 576 #Added--------------------
577 577
578 578 selBlocksize = None
579 579
580 580 selBlocktime = None
581 581
582 582
583 583 def __init__(self):
584 584
585 585 """
586 586 This class is used to find data files
587 587
588 588 Example:
589 589 reader = JRODataReader()
590 590 fileList = reader.findDataFiles()
591 591
592 592 """
593 593 pass
594 594
595 595
596 596 def createObjByDefault(self):
597 597 """
598 598
599 599 """
600 600 raise NotImplementedError
601 601
602 602 def getBlockDimension(self):
603 603
604 604 raise NotImplementedError
605 605
606 606 def __searchFilesOffLine(self,
607 607 path,
608 608 startDate=None,
609 609 endDate=None,
610 610 startTime=datetime.time(0,0,0),
611 611 endTime=datetime.time(23,59,59),
612 612 set=None,
613 613 expLabel='',
614 614 ext='.r',
615 615 queue=None,
616 616 cursor=None,
617 617 skip=None,
618 618 walk=True):
619 619
620 620 self.filenameList = []
621 621 self.datetimeList = []
622 622
623 623 pathList = []
624 624
625 625 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
626 626
627 627 if dateList == []:
628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
629 629 return None, None
630 630
631 631 if len(dateList) > 1:
632 632 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
633 633 else:
634 634 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
635 635
636 636 filenameList = []
637 637 datetimeList = []
638 638
639 639 for thisPath in pathList:
640 # thisPath = pathList[pathDict[file]]
641
640 # thisPath = pathList[pathDict[file]]
641
642 642 fileList = glob.glob1(thisPath, "*%s" %ext)
643 643 fileList.sort()
644 644
645 645 skippedFileList = []
646 646
647 647 if cursor is not None and skip is not None:
648 648 # if cursor*skip > len(fileList):
649 649 if skip == 0:
650 650 if queue is not None:
651 651 queue.put(len(fileList))
652 652 skippedFileList = []
653 653 else:
654 654 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
655 655
656 656 else:
657 657 skippedFileList = fileList
658 658
659 659 for file in skippedFileList:
660 660
661 661 filename = os.path.join(thisPath,file)
662 662
663 663 if not isFileInDateRange(filename, startDate, endDate):
664 664 continue
665 665
666 666 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
667 667
668 668 if not(thisDatetime):
669 669 continue
670 670
671 671 filenameList.append(filename)
672 672 datetimeList.append(thisDatetime)
673 673
674 674 if not(filenameList):
675 675 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
676 676 return None, None
677 677
678 678 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
679 679 print
680 680
681 681 for i in range(len(filenameList)):
682 682 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
683 683
684 684 self.filenameList = filenameList
685 685 self.datetimeList = datetimeList
686 686
687 687 return pathList, filenameList
688 688
689 689 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
690 690
691 691 """
692 692 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
693 693 devuelve el archivo encontrado ademas de otros datos.
694 694
695 695 Input:
696 696 path : carpeta donde estan contenidos los files que contiene data
697 697
698 698 expLabel : Nombre del subexperimento (subfolder)
699 699
700 700 ext : extension de los files
701 701
702 702 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
703 703
704 704 Return:
705 705 directory : eL directorio donde esta el file encontrado
706 706 filename : el ultimo file de una determinada carpeta
707 707 year : el anho
708 708 doy : el numero de dia del anho
709 709 set : el set del archivo
710 710
711 711
712 712 """
713 713 if not os.path.isdir(path):
714 714 return None, None, None, None, None, None
715 715
716 716 dirList = []
717 717
718 718 if not walk:
719 719 fullpath = path
720 720 foldercounter = 0
721 721 else:
722 722 #Filtra solo los directorios
723 723 for thisPath in os.listdir(path):
724 724 if not os.path.isdir(os.path.join(path,thisPath)):
725 725 continue
726 726 if not isRadarFolder(thisPath):
727 727 continue
728 728
729 729 dirList.append(thisPath)
730 730
731 731 if not(dirList):
732 732 return None, None, None, None, None, None
733 733
734 734 dirList = sorted( dirList, key=str.lower )
735 735
736 736 doypath = dirList[-1]
737 737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 738 fullpath = os.path.join(path, doypath, expLabel)
739 739
740 740
741 741 print "[Reading] %s folder was found: " %(fullpath )
742 742
743 743 if set == None:
744 744 filename = getlastFileFromPath(fullpath, ext)
745 745 else:
746 746 filename = getFileFromSet(fullpath, ext, set)
747 747
748 748 if not(filename):
749 749 return None, None, None, None, None, None
750 750
751 751 print "[Reading] %s file was found" %(filename)
752 752
753 753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 754 return None, None, None, None, None, None
755 755
756 756 year = int( filename[1:5] )
757 757 doy = int( filename[5:8] )
758 758 set = int( filename[8:11] )
759 759
760 760 return fullpath, foldercounter, filename, year, doy, set
761 761
762 762 def __setNextFileOffline(self):
763 763
764 764 idFile = self.fileIndex
765 765
766 766 while (True):
767 767 idFile += 1
768 768 if not(idFile < len(self.filenameList)):
769 769 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
770 # print "[Reading] No more Files"
771 771 return 0
772 772
773 773 filename = self.filenameList[idFile]
774 774
775 775 if not(self.__verifyFile(filename)):
776 776 continue
777 777
778 778 fileSize = os.path.getsize(filename)
779 779 fp = open(filename,'rb')
780 780 break
781 781
782 782 self.flagIsNewFile = 1
783 783 self.fileIndex = idFile
784 784 self.filename = filename
785 785 self.fileSize = fileSize
786 786 self.fp = fp
787 787
788 # print "[Reading] Setting the file: %s"%self.filename
788 # print "[Reading] Setting the file: %s"%self.filename
789 789
790 790 return 1
791 791
792 792 def __setNextFileOnline(self):
793 793 """
794 794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 796 siguientes.
797 797
798 798 Affected:
799 799 self.flagIsNewFile
800 800 self.filename
801 801 self.fileSize
802 802 self.fp
803 803 self.set
804 804 self.flagNoMoreFiles
805 805
806 806 Return:
807 807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 808 1 : si el file fue abierto con exito y esta listo a ser leido
809 809
810 810 Excepciones:
811 811 Si un determinado file no puede ser abierto
812 812 """
813 813 nFiles = 0
814 814 fileOk_flag = False
815 815 firstTime_flag = True
816 816
817 817 self.set += 1
818 818
819 819 if self.set > 999:
820 820 self.set = 0
821 821 self.foldercounter += 1
822 822
823 823 #busca el 1er file disponible
824 824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 825 if fullfilename:
826 826 if self.__verifyFile(fullfilename, False):
827 827 fileOk_flag = True
828 828
829 829 #si no encuentra un file entonces espera y vuelve a buscar
830 830 if not(fileOk_flag):
831 831 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
832 832
833 833 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
834 834 tries = self.nTries
835 835 else:
836 836 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
837 837
838 838 for nTries in range( tries ):
839 839 if firstTime_flag:
840 840 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
841 841 sleep( self.delay )
842 842 else:
843 843 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
844 844
845 845 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
846 846 if fullfilename:
847 847 if self.__verifyFile(fullfilename):
848 848 fileOk_flag = True
849 849 break
850 850
851 851 if fileOk_flag:
852 852 break
853 853
854 854 firstTime_flag = False
855 855
856 856 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
857 857 self.set += 1
858 858
859 859 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
860 860 self.set = 0
861 861 self.doy += 1
862 862 self.foldercounter = 0
863 863
864 864 if fileOk_flag:
865 865 self.fileSize = os.path.getsize( fullfilename )
866 866 self.filename = fullfilename
867 867 self.flagIsNewFile = 1
868 868 if self.fp != None: self.fp.close()
869 869 self.fp = open(fullfilename, 'rb')
870 870 self.flagNoMoreFiles = 0
871 # print '[Reading] Setting the file: %s' % fullfilename
871 # print '[Reading] Setting the file: %s' % fullfilename
872 872 else:
873 873 self.fileSize = 0
874 874 self.filename = None
875 875 self.flagIsNewFile = 0
876 876 self.fp = None
877 877 self.flagNoMoreFiles = 1
878 # print '[Reading] No more files to read'
878 # print '[Reading] No more files to read'
879 879
880 880 return fileOk_flag
881 881
882 882 def setNextFile(self):
883 883 if self.fp != None:
884 884 self.fp.close()
885 885
886 886 if self.online:
887 887 newFile = self.__setNextFileOnline()
888 888 else:
889 889 newFile = self.__setNextFileOffline()
890 890
891 891 if not(newFile):
892 892 print '[Reading] No more files to read'
893 893 return 0
894 894
895 895 if self.verbose:
896 896 print '[Reading] Setting the file: %s' % self.filename
897 897
898 898 self.__readFirstHeader()
899 899 self.nReadBlocks = 0
900 900 return 1
901 901
902 902 def __waitNewBlock(self):
903 903 """
904 904 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
905 905
906 906 Si el modo de lectura es OffLine siempre retorn 0
907 907 """
908 908 if not self.online:
909 909 return 0
910 910
911 911 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
912 912 return 0
913 913
914 914 currentPointer = self.fp.tell()
915 915
916 916 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
917 917
918 918 for nTries in range( self.nTries ):
919 919
920 920 self.fp.close()
921 921 self.fp = open( self.filename, 'rb' )
922 922 self.fp.seek( currentPointer )
923 923
924 924 self.fileSize = os.path.getsize( self.filename )
925 925 currentSize = self.fileSize - currentPointer
926 926
927 927 if ( currentSize >= neededSize ):
928 928 self.basicHeaderObj.read(self.fp)
929 929 return 1
930 930
931 931 if self.fileSize == self.fileSizeByHeader:
932 # self.flagEoF = True
932 # self.flagEoF = True
933 933 return 0
934 934
935 935 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
936 936 sleep( self.delay )
937 937
938 938
939 939 return 0
940 940
941 941 def waitDataBlock(self,pointer_location):
942 942
943 943 currentPointer = pointer_location
944 944
945 945 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
946 946
947 947 for nTries in range( self.nTries ):
948 948 self.fp.close()
949 949 self.fp = open( self.filename, 'rb' )
950 950 self.fp.seek( currentPointer )
951 951
952 952 self.fileSize = os.path.getsize( self.filename )
953 953 currentSize = self.fileSize - currentPointer
954 954
955 955 if ( currentSize >= neededSize ):
956 956 return 1
957 957
958 958 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
959 959 sleep( self.delay )
960 960
961 961 return 0
962 962
963 963 def __jumpToLastBlock(self):
964 964
965 965 if not(self.__isFirstTimeOnline):
966 966 return
967 967
968 968 csize = self.fileSize - self.fp.tell()
969 969 blocksize = self.processingHeaderObj.blockSize
970 970
971 971 #salta el primer bloque de datos
972 972 if csize > self.processingHeaderObj.blockSize:
973 973 self.fp.seek(self.fp.tell() + blocksize)
974 974 else:
975 975 return
976 976
977 977 csize = self.fileSize - self.fp.tell()
978 978 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
979 979 while True:
980 980
981 981 if self.fp.tell()<self.fileSize:
982 982 self.fp.seek(self.fp.tell() + neededsize)
983 983 else:
984 984 self.fp.seek(self.fp.tell() - neededsize)
985 985 break
986
987 # csize = self.fileSize - self.fp.tell()
988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
989 # factor = int(csize/neededsize)
990 # if factor > 0:
991 # self.fp.seek(self.fp.tell() + factor*neededsize)
992
986
987 # csize = self.fileSize - self.fp.tell()
988 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
989 # factor = int(csize/neededsize)
990 # if factor > 0:
991 # self.fp.seek(self.fp.tell() + factor*neededsize)
992
993 993 self.flagIsNewFile = 0
994 994 self.__isFirstTimeOnline = 0
995 995
996 996 def __setNewBlock(self):
997 997 #if self.server is None:
998 998 if self.fp == None:
999 999 return 0
1000
1001 # if self.online:
1002 # self.__jumpToLastBlock()
1003
1000
1001 # if self.online:
1002 # self.__jumpToLastBlock()
1003
1004 1004 if self.flagIsNewFile:
1005 1005 self.lastUTTime = self.basicHeaderObj.utc
1006 1006 return 1
1007 1007
1008 1008 if self.realtime:
1009 1009 self.flagDiscontinuousBlock = 1
1010 1010 if not(self.setNextFile()):
1011 1011 return 0
1012 1012 else:
1013 1013 return 1
1014 1014 #if self.server is None:
1015 1015 currentSize = self.fileSize - self.fp.tell()
1016 1016 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1017 1017 if (currentSize >= neededSize):
1018 1018 self.basicHeaderObj.read(self.fp)
1019 1019 self.lastUTTime = self.basicHeaderObj.utc
1020 1020 return 1
1021 1021 # else:
1022 1022 # self.basicHeaderObj.read(self.zHeader)
1023 1023 # self.lastUTTime = self.basicHeaderObj.utc
1024 1024 # return 1
1025 1025 if self.__waitNewBlock():
1026 1026 self.lastUTTime = self.basicHeaderObj.utc
1027 1027 return 1
1028 1028 #if self.server is None:
1029 1029 if not(self.setNextFile()):
1030 1030 return 0
1031 1031
1032 1032 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1033 1033 self.lastUTTime = self.basicHeaderObj.utc
1034 1034
1035 1035 self.flagDiscontinuousBlock = 0
1036 1036
1037 1037 if deltaTime > self.maxTimeStep:
1038 1038 self.flagDiscontinuousBlock = 1
1039 1039
1040 1040 return 1
1041 1041
1042 1042 def readNextBlock(self):
1043 1043
1044 1044 #Skip block out of startTime and endTime
1045 1045 while True:
1046 1046 if not(self.__setNewBlock()):
1047 1047 print 'returning'
1048 1048 return 0
1049 1049
1050 1050 if not(self.readBlock()):
1051 1051 return 0
1052 1052
1053 1053 self.getBasicHeader()
1054 1054
1055 1055 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1056 1056
1057 1057 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1058 1058 self.processingHeaderObj.dataBlocksPerFile,
1059 1059 self.dataOut.datatime.ctime())
1060 1060 continue
1061 1061
1062 1062 break
1063 1063
1064 1064 if self.verbose:
1065 1065 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1066 self.processingHeaderObj.dataBlocksPerFile,
1067 self.dataOut.datatime.ctime())
1066 self.processingHeaderObj.dataBlocksPerFile,
1067 self.dataOut.datatime.ctime())
1068 1068 return 1
1069 1069
1070 1070 def __readFirstHeader(self):
1071 1071
1072 1072 self.basicHeaderObj.read(self.fp)
1073 1073 self.systemHeaderObj.read(self.fp)
1074 1074 self.radarControllerHeaderObj.read(self.fp)
1075 1075 self.processingHeaderObj.read(self.fp)
1076 1076
1077 1077 self.firstHeaderSize = self.basicHeaderObj.size
1078 1078
1079 1079 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1080 1080 if datatype == 0:
1081 1081 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1082 1082 elif datatype == 1:
1083 1083 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1084 1084 elif datatype == 2:
1085 1085 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1086 1086 elif datatype == 3:
1087 1087 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1088 1088 elif datatype == 4:
1089 1089 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1090 1090 elif datatype == 5:
1091 1091 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1092 1092 else:
1093 1093 raise ValueError, 'Data type was not defined'
1094 1094
1095 1095 self.dtype = datatype_str
1096 1096 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1097 1097 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1098 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1099 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1098 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1099 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1100 1100 self.getBlockDimension()
1101 1101
1102 1102 def __verifyFile(self, filename, msgFlag=True):
1103 1103
1104 1104 msg = None
1105 1105
1106 1106 try:
1107 1107 fp = open(filename, 'rb')
1108 1108 except IOError:
1109 1109
1110 1110 if msgFlag:
1111 1111 print "[Reading] File %s can't be opened" % (filename)
1112 1112
1113 1113 return False
1114 1114
1115 1115 currentPosition = fp.tell()
1116 1116 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1117 1117
1118 1118 if neededSize == 0:
1119 1119 basicHeaderObj = BasicHeader(LOCALTIME)
1120 1120 systemHeaderObj = SystemHeader()
1121 1121 radarControllerHeaderObj = RadarControllerHeader()
1122 1122 processingHeaderObj = ProcessingHeader()
1123 1123
1124 1124 if not( basicHeaderObj.read(fp) ):
1125 1125 fp.close()
1126 1126 return False
1127 1127
1128 1128 if not( systemHeaderObj.read(fp) ):
1129 1129 fp.close()
1130 1130 return False
1131 1131
1132 1132 if not( radarControllerHeaderObj.read(fp) ):
1133 1133 fp.close()
1134 1134 return False
1135 1135
1136 1136 if not( processingHeaderObj.read(fp) ):
1137 1137 fp.close()
1138 1138 return False
1139 1139
1140 1140 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1141 1141 else:
1142 1142 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1143 1143
1144 1144 fp.close()
1145 1145
1146 1146 fileSize = os.path.getsize(filename)
1147 1147 currentSize = fileSize - currentPosition
1148 1148
1149 1149 if currentSize < neededSize:
1150 1150 if msgFlag and (msg != None):
1151 1151 print msg
1152 1152 return False
1153 1153
1154 1154 return True
1155 1155
1156 1156 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1157 1157
1158 1158 path_empty = True
1159 1159
1160 1160 dateList = []
1161 1161 pathList = []
1162 1162
1163 1163 multi_path = path.split(',')
1164 1164
1165 1165 if not walk:
1166 1166
1167 1167 for single_path in multi_path:
1168 1168
1169 1169 if not os.path.isdir(single_path):
1170 1170 continue
1171 1171
1172 1172 fileList = glob.glob1(single_path, "*"+ext)
1173 1173
1174 1174 if not fileList:
1175 1175 continue
1176 1176
1177 1177 path_empty = False
1178 1178
1179 1179 fileList.sort()
1180 1180
1181 1181 for thisFile in fileList:
1182 1182
1183 1183 if not os.path.isfile(os.path.join(single_path, thisFile)):
1184 1184 continue
1185 1185
1186 1186 if not isRadarFile(thisFile):
1187 1187 continue
1188 1188
1189 1189 if not isFileInDateRange(thisFile, startDate, endDate):
1190 1190 continue
1191 1191
1192 1192 thisDate = getDateFromRadarFile(thisFile)
1193 1193
1194 1194 if thisDate in dateList:
1195 1195 continue
1196 1196
1197 1197 dateList.append(thisDate)
1198 1198 pathList.append(single_path)
1199 1199
1200 1200 else:
1201 1201 for single_path in multi_path:
1202 1202
1203 1203 if not os.path.isdir(single_path):
1204 1204 continue
1205 1205
1206 1206 dirList = []
1207 1207
1208 1208 for thisPath in os.listdir(single_path):
1209 1209
1210 1210 if not os.path.isdir(os.path.join(single_path,thisPath)):
1211 1211 continue
1212 1212
1213 1213 if not isRadarFolder(thisPath):
1214 1214 continue
1215 1215
1216 1216 if not isFolderInDateRange(thisPath, startDate, endDate):
1217 1217 continue
1218 1218
1219 1219 dirList.append(thisPath)
1220 1220
1221 1221 if not dirList:
1222 1222 continue
1223 1223
1224 1224 dirList.sort()
1225 1225
1226 1226 for thisDir in dirList:
1227 1227
1228 1228 datapath = os.path.join(single_path, thisDir, expLabel)
1229 1229 fileList = glob.glob1(datapath, "*"+ext)
1230 1230
1231 1231 if not fileList:
1232 1232 continue
1233 1233
1234 1234 path_empty = False
1235 1235
1236 1236 thisDate = getDateFromRadarFolder(thisDir)
1237 1237
1238 1238 pathList.append(datapath)
1239 1239 dateList.append(thisDate)
1240 1240
1241 1241 dateList.sort()
1242 1242
1243 1243 if walk:
1244 1244 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1245 1245 else:
1246 1246 pattern_path = multi_path[0]
1247 1247
1248 1248 if path_empty:
1249 1249 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1250 1250 else:
1251 1251 if not dateList:
1252 1252 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1253 1253
1254 1254 if include_path:
1255 1255 return dateList, pathList
1256 1256
1257 1257 return dateList
1258 1258
1259 1259 def setup(self,
1260 1260 path=None,
1261 1261 startDate=None,
1262 1262 endDate=None,
1263 1263 startTime=datetime.time(0,0,0),
1264 1264 endTime=datetime.time(23,59,59),
1265 1265 set=None,
1266 1266 expLabel = "",
1267 1267 ext = None,
1268 1268 online = False,
1269 1269 delay = 60,
1270 1270 walk = True,
1271 1271 getblock = False,
1272 1272 nTxs = 1,
1273 1273 realtime=False,
1274 1274 blocksize=None,
1275 1275 blocktime=None,
1276 1276 queue=None,
1277 1277 skip=None,
1278 1278 cursor=None,
1279 1279 warnings=True,
1280 1280 verbose=True,
1281 1281 server=None):
1282 1282 if server is not None:
1283 1283 if 'tcp://' in server:
1284 1284 address = server
1285 1285 else:
1286 1286 address = 'ipc:///tmp/%s' % server
1287 1287 self.server = address
1288 1288 self.context = zmq.Context()
1289 1289 self.receiver = self.context.socket(zmq.PULL)
1290 1290 self.receiver.connect(self.server)
1291 1291 time.sleep(0.5)
1292 1292 print '[Starting] ReceiverData from {}'.format(self.server)
1293 1293 else:
1294 1294 self.server = None
1295 1295 if path == None:
1296 1296 raise ValueError, "[Reading] The path is not valid"
1297 1297
1298 1298 if ext == None:
1299 1299 ext = self.ext
1300 1300
1301 1301 if online:
1302 1302 print "[Reading] Searching files in online mode..."
1303 1303
1304 1304 for nTries in range( self.nTries ):
1305 1305 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1306 1306
1307 1307 if fullpath:
1308 1308 break
1309 1309
1310 1310 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1311 1311 sleep( self.delay )
1312 1312
1313 1313 if not(fullpath):
1314 1314 print "[Reading] There 'isn't any valid file in %s" % path
1315 1315 return
1316 1316
1317 1317 self.year = year
1318 1318 self.doy = doy
1319 1319 self.set = set - 1
1320 1320 self.path = path
1321 1321 self.foldercounter = foldercounter
1322 1322 last_set = None
1323 1323 else:
1324 1324 print "[Reading] Searching files in offline mode ..."
1325 1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 1326 startTime=startTime, endTime=endTime,
1327 1327 set=set, expLabel=expLabel, ext=ext,
1328 1328 walk=walk, cursor=cursor,
1329 1329 skip=skip, queue=queue)
1330 1330
1331 1331 if not(pathList):
1332 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1333 # datetime.datetime.combine(startDate,startTime).ctime(),
1334 # datetime.datetime.combine(endDate,endTime).ctime())
1332 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1333 # datetime.datetime.combine(startDate,startTime).ctime(),
1334 # datetime.datetime.combine(endDate,endTime).ctime())
1335 1335
1336 # sys.exit(-1)
1336 # sys.exit(-1)
1337 1337
1338 1338 self.fileIndex = -1
1339 1339 self.pathList = []
1340 1340 self.filenameList = []
1341 1341 return
1342 1342
1343 1343 self.fileIndex = -1
1344 1344 self.pathList = pathList
1345 1345 self.filenameList = filenameList
1346 1346 file_name = os.path.basename(filenameList[-1])
1347 1347 basename, ext = os.path.splitext(file_name)
1348 1348 last_set = int(basename[-3:])
1349 1349
1350 1350 self.online = online
1351 1351 self.realtime = realtime
1352 1352 self.delay = delay
1353 1353 ext = ext.lower()
1354 1354 self.ext = ext
1355 1355 self.getByBlock = getblock
1356 1356 self.nTxs = nTxs
1357 1357 self.startTime = startTime
1358 1358 self.endTime = endTime
1359 1359
1360 1360 #Added-----------------
1361 1361 self.selBlocksize = blocksize
1362 1362 self.selBlocktime = blocktime
1363 1363
1364 1364 # Verbose-----------
1365 1365 self.verbose = verbose
1366 1366 self.warnings = warnings
1367 1367
1368 1368 if not(self.setNextFile()):
1369 1369 if (startDate!=None) and (endDate!=None):
1370 1370 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1371 1371 elif startDate != None:
1372 1372 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1373 1373 else:
1374 1374 print "[Reading] No files"
1375 1375
1376 1376 self.fileIndex = -1
1377 1377 self.pathList = []
1378 1378 self.filenameList = []
1379 1379 return
1380 1380
1381 # self.getBasicHeader()
1381 # self.getBasicHeader()
1382 1382
1383 if last_set != None:
1384 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1383 if last_set != None:
1384 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1385 1385 return
1386 1386
1387 1387 def getBasicHeader(self):
1388 1388
1389 1389 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1390 1390
1391 1391 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1392 1392
1393 1393 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1394 1394
1395 1395 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1396 1396
1397 1397 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1398 1398
1399 1399 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1400 1400
1401 1401 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1402 1402
1403 1403 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1404 1404
1405 1405
1406 1406 def getFirstHeader(self):
1407 1407
1408 1408 raise NotImplementedError
1409 1409
1410 1410 def getData(self):
1411 1411
1412 1412 raise NotImplementedError
1413 1413
1414 1414 def hasNotDataInBuffer(self):
1415 1415
1416 1416 raise NotImplementedError
1417 1417
1418 1418 def readBlock(self):
1419 1419
1420 1420 raise NotImplementedError
1421 1421
1422 1422 def isEndProcess(self):
1423 1423
1424 1424 return self.flagNoMoreFiles
1425 1425
1426 1426 def printReadBlocks(self):
1427 1427
1428 1428 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1429 1429
1430 1430 def printTotalBlocks(self):
1431 1431
1432 1432 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1433 1433
1434 1434 def printNumberOfBlock(self):
1435 1435
1436 1436 if self.flagIsNewBlock:
1437 1437 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1438 1438 self.processingHeaderObj.dataBlocksPerFile,
1439 1439 self.dataOut.datatime.ctime())
1440 1440
1441 1441 def printInfo(self):
1442 1442
1443 1443 if self.__printInfo == False:
1444 1444 return
1445 1445
1446 1446 self.basicHeaderObj.printInfo()
1447 1447 self.systemHeaderObj.printInfo()
1448 1448 self.radarControllerHeaderObj.printInfo()
1449 1449 self.processingHeaderObj.printInfo()
1450 1450
1451 1451 self.__printInfo = False
1452 1452
1453 1453
1454 1454 def run(self,
1455 1455 path=None,
1456 1456 startDate=None,
1457 1457 endDate=None,
1458 1458 startTime=datetime.time(0,0,0),
1459 1459 endTime=datetime.time(23,59,59),
1460 1460 set=None,
1461 1461 expLabel = "",
1462 1462 ext = None,
1463 1463 online = False,
1464 1464 delay = 60,
1465 1465 walk = True,
1466 1466 getblock = False,
1467 1467 nTxs = 1,
1468 1468 realtime=False,
1469 1469 blocksize=None,
1470 1470 blocktime=None,
1471 1471 queue=None,
1472 1472 skip=None,
1473 1473 cursor=None,
1474 1474 warnings=True,
1475 1475 server=None,
1476 1476 verbose=True, **kwargs):
1477 1477
1478 1478 if not(self.isConfig):
1479 1479 # self.dataOut = dataOut
1480 1480 self.setup( path=path,
1481 1481 startDate=startDate,
1482 1482 endDate=endDate,
1483 1483 startTime=startTime,
1484 1484 endTime=endTime,
1485 1485 set=set,
1486 1486 expLabel=expLabel,
1487 1487 ext=ext,
1488 1488 online=online,
1489 1489 delay=delay,
1490 1490 walk=walk,
1491 1491 getblock=getblock,
1492 1492 nTxs=nTxs,
1493 1493 realtime=realtime,
1494 1494 blocksize=blocksize,
1495 1495 blocktime=blocktime,
1496 1496 queue=queue,
1497 1497 skip=skip,
1498 1498 cursor=cursor,
1499 1499 warnings=warnings,
1500 1500 server=server,
1501 1501 verbose=verbose)
1502 1502 self.isConfig = True
1503 1503 if server is None:
1504 1504 self.getData()
1505 1505 else:
1506 1506 self.getFromServer()
1507 1507
1508 1508 class JRODataWriter(JRODataIO):
1509 1509
1510 1510 """
1511 1511 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1512 1512 de los datos siempre se realiza por bloques.
1513 1513 """
1514 1514
1515 1515 blockIndex = 0
1516 1516
1517 1517 path = None
1518 1518
1519 1519 setFile = None
1520 1520
1521 1521 profilesPerBlock = None
1522 1522
1523 1523 blocksPerFile = None
1524 1524
1525 1525 nWriteBlocks = 0
1526 1526
1527 1527 fileDate = None
1528 1528
1529 1529 def __init__(self, dataOut=None):
1530 1530 raise NotImplementedError
1531 1531
1532 1532
1533 1533 def hasAllDataInBuffer(self):
1534 1534 raise NotImplementedError
1535 1535
1536 1536
1537 1537 def setBlockDimension(self):
1538 1538 raise NotImplementedError
1539 1539
1540 1540
1541 1541 def writeBlock(self):
1542 1542 raise NotImplementedError
1543 1543
1544 1544
1545 1545 def putData(self):
1546 1546 raise NotImplementedError
1547 1547
1548 1548
1549 1549 def getProcessFlags(self):
1550 1550
1551 1551 processFlags = 0
1552 1552
1553 1553 dtype_index = get_dtype_index(self.dtype)
1554 1554 procflag_dtype = get_procflag_dtype(dtype_index)
1555 1555
1556 1556 processFlags += procflag_dtype
1557 1557
1558 1558 if self.dataOut.flagDecodeData:
1559 1559 processFlags += PROCFLAG.DECODE_DATA
1560 1560
1561 1561 if self.dataOut.flagDeflipData:
1562 1562 processFlags += PROCFLAG.DEFLIP_DATA
1563 1563
1564 1564 if self.dataOut.code is not None:
1565 1565 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1566 1566
1567 1567 if self.dataOut.nCohInt > 1:
1568 1568 processFlags += PROCFLAG.COHERENT_INTEGRATION
1569 1569
1570 1570 if self.dataOut.type == "Spectra":
1571 1571 if self.dataOut.nIncohInt > 1:
1572 1572 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1573 1573
1574 1574 if self.dataOut.data_dc is not None:
1575 1575 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1576 1576
1577 1577 if self.dataOut.flagShiftFFT:
1578 1578 processFlags += PROCFLAG.SHIFT_FFT_DATA
1579 1579
1580 1580 return processFlags
1581 1581
1582 1582 def setBasicHeader(self):
1583 1583
1584 1584 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1585 1585 self.basicHeaderObj.version = self.versionFile
1586 1586 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1587 1587
1588 1588 utc = numpy.floor(self.dataOut.utctime)
1589 1589 milisecond = (self.dataOut.utctime - utc)* 1000.0
1590 1590
1591 1591 self.basicHeaderObj.utc = utc
1592 1592 self.basicHeaderObj.miliSecond = milisecond
1593 1593 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1594 1594 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1595 1595 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1596 1596
1597 1597 def setFirstHeader(self):
1598 1598 """
1599 1599 Obtiene una copia del First Header
1600 1600
1601 1601 Affected:
1602 1602
1603 1603 self.basicHeaderObj
1604 1604 self.systemHeaderObj
1605 1605 self.radarControllerHeaderObj
1606 1606 self.processingHeaderObj self.
1607 1607
1608 1608 Return:
1609 1609 None
1610 1610 """
1611 1611
1612 1612 raise NotImplementedError
1613 1613
1614 1614 def __writeFirstHeader(self):
1615 1615 """
1616 1616 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1617 1617
1618 1618 Affected:
1619 1619 __dataType
1620 1620
1621 1621 Return:
1622 1622 None
1623 1623 """
1624 1624
1625 1625 # CALCULAR PARAMETROS
1626 1626
1627 1627 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1628 1628 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1629 1629
1630 1630 self.basicHeaderObj.write(self.fp)
1631 1631 self.systemHeaderObj.write(self.fp)
1632 1632 self.radarControllerHeaderObj.write(self.fp)
1633 1633 self.processingHeaderObj.write(self.fp)
1634 1634
1635 1635 def __setNewBlock(self):
1636 1636 """
1637 1637 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1638 1638
1639 1639 Return:
1640 1640 0 : si no pudo escribir nada
1641 1641 1 : Si escribio el Basic el First Header
1642 1642 """
1643 1643 if self.fp == None:
1644 1644 self.setNextFile()
1645 1645
1646 1646 if self.flagIsNewFile:
1647 1647 return 1
1648 1648
1649 1649 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1650 1650 self.basicHeaderObj.write(self.fp)
1651 1651 return 1
1652 1652
1653 1653 if not( self.setNextFile() ):
1654 1654 return 0
1655 1655
1656 1656 return 1
1657 1657
1658 1658
1659 1659 def writeNextBlock(self):
1660 1660 """
1661 1661 Selecciona el bloque siguiente de datos y los escribe en un file
1662 1662
1663 1663 Return:
1664 1664 0 : Si no hizo pudo escribir el bloque de datos
1665 1665 1 : Si no pudo escribir el bloque de datos
1666 1666 """
1667 1667 if not( self.__setNewBlock() ):
1668 1668 return 0
1669 1669
1670 1670 self.writeBlock()
1671 1671
1672 1672 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1673 1673 self.processingHeaderObj.dataBlocksPerFile)
1674 1674
1675 1675 return 1
1676 1676
1677 1677 def setNextFile(self):
1678 1678 """
1679 1679 Determina el siguiente file que sera escrito
1680 1680
1681 1681 Affected:
1682 1682 self.filename
1683 1683 self.subfolder
1684 1684 self.fp
1685 1685 self.setFile
1686 1686 self.flagIsNewFile
1687 1687
1688 1688 Return:
1689 1689 0 : Si el archivo no puede ser escrito
1690 1690 1 : Si el archivo esta listo para ser escrito
1691 1691 """
1692 1692 ext = self.ext
1693 1693 path = self.path
1694 1694
1695 1695 if self.fp != None:
1696 1696 self.fp.close()
1697 1697
1698 1698 timeTuple = time.localtime( self.dataOut.utctime)
1699 1699 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1700 1700
1701 1701 fullpath = os.path.join( path, subfolder )
1702 1702 setFile = self.setFile
1703 1703
1704 1704 if not( os.path.exists(fullpath) ):
1705 1705 os.mkdir(fullpath)
1706 1706 setFile = -1 #inicializo mi contador de seteo
1707 1707 else:
1708 1708 filesList = os.listdir( fullpath )
1709 1709 if len( filesList ) > 0:
1710 1710 filesList = sorted( filesList, key=str.lower )
1711 1711 filen = filesList[-1]
1712 1712 # el filename debera tener el siguiente formato
1713 1713 # 0 1234 567 89A BCDE (hex)
1714 1714 # x YYYY DDD SSS .ext
1715 1715 if isNumber( filen[8:11] ):
1716 1716 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1717 1717 else:
1718 1718 setFile = -1
1719 1719 else:
1720 1720 setFile = -1 #inicializo mi contador de seteo
1721 1721
1722 1722 setFile += 1
1723 1723
1724 1724 #If this is a new day it resets some values
1725 1725 if self.dataOut.datatime.date() > self.fileDate:
1726 1726 setFile = 0
1727 1727 self.nTotalBlocks = 0
1728 1728
1729 1729 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1730 1730
1731 1731 filename = os.path.join( path, subfolder, filen )
1732 1732
1733 1733 fp = open( filename,'wb' )
1734 1734
1735 1735 self.blockIndex = 0
1736 1736
1737 1737 #guardando atributos
1738 1738 self.filename = filename
1739 1739 self.subfolder = subfolder
1740 1740 self.fp = fp
1741 1741 self.setFile = setFile
1742 1742 self.flagIsNewFile = 1
1743 1743 self.fileDate = self.dataOut.datatime.date()
1744 1744
1745 1745 self.setFirstHeader()
1746 1746
1747 1747 print '[Writing] Opening file: %s'%self.filename
1748 1748
1749 1749 self.__writeFirstHeader()
1750 1750
1751 1751 return 1
1752 1752
1753 1753 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1754 1754 """
1755 1755 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1756 1756
1757 1757 Inputs:
1758 1758 path : directory where data will be saved
1759 1759 profilesPerBlock : number of profiles per block
1760 1760 set : initial file set
1761 1761 datatype : An integer number that defines data type:
1762 1762 0 : int8 (1 byte)
1763 1763 1 : int16 (2 bytes)
1764 1764 2 : int32 (4 bytes)
1765 1765 3 : int64 (8 bytes)
1766 1766 4 : float32 (4 bytes)
1767 1767 5 : double64 (8 bytes)
1768 1768
1769 1769 Return:
1770 1770 0 : Si no realizo un buen seteo
1771 1771 1 : Si realizo un buen seteo
1772 1772 """
1773 1773
1774 1774 if ext == None:
1775 1775 ext = self.ext
1776 1776
1777 1777 self.ext = ext.lower()
1778 1778
1779 1779 self.path = path
1780 1780
1781 1781 if set is None:
1782 1782 self.setFile = -1
1783 1783 else:
1784 1784 self.setFile = set - 1
1785 1785
1786 1786 self.blocksPerFile = blocksPerFile
1787 1787
1788 1788 self.profilesPerBlock = profilesPerBlock
1789 1789
1790 1790 self.dataOut = dataOut
1791 1791 self.fileDate = self.dataOut.datatime.date()
1792 1792 #By default
1793 1793 self.dtype = self.dataOut.dtype
1794 1794
1795 1795 if datatype is not None:
1796 1796 self.dtype = get_numpy_dtype(datatype)
1797 1797
1798 1798 if not(self.setNextFile()):
1799 1799 print "[Writing] There isn't a next file"
1800 1800 return 0
1801 1801
1802 1802 self.setBlockDimension()
1803 1803
1804 1804 return 1
1805 1805
1806 1806 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1807 1807
1808 1808 if not(self.isConfig):
1809 1809
1810 1810 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1811 1811 self.isConfig = True
1812 1812
1813 1813 self.putData()
Show file before | Show file after | Hide comments
@@ -1,1007 +1,1009
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13 13
14 14 import time
15 15
16 16 import threading
17 17 Thread = threading.Thread
18 18
19 19 # try:
20 20 # from gevent import sleep
21 21 # except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25
26 26 class Remote(Thread):
27 27 """
28 28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 29 used to upload or download files remotely.
30 30
31 31 Non-standard Python modules used:
32 32 None
33 33
34 34 Written by:
35 35
36 36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37 37
38 38 """
39 39
40 40 server = None
41 41 username = None
42 42 password = None
43 43 remotefolder = None
44 44
45 45 period = 60
46 46 fileList = []
47 47 bussy = False
48 48
49 49 def __init__(self, server, username, password, remotefolder, period=60):
50 50
51 51 Thread.__init__(self)
52 52
53 53 self.setDaemon(True)
54 54
55 55 self.status = 0
56 56
57 57 self.__server = server
58 58 self.__username = username
59 59 self.__password = password
60 60 self.__remotefolder = remotefolder
61 61
62 62 self.period = period
63 63
64 64 self.fileList = []
65 65 self.bussy = False
66 66
67 67 self.stopFlag = False
68 68
69 69 print "[Remote Server] Opening server: %s" %self.__server
70 70 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 71 print "[Remote Server] %s server was opened successfully" %self.__server
72 72
73 73 self.close()
74 74
75 75 self.mutex = threading.Lock()
76 76
77 77 def stop(self):
78 78
79 79 self.stopFlag = True
80 80 self.join(10)
81 81
82 82 def open(self):
83 83 """
84 84 Connect to server and create a connection class (FTP or SSH) to remote server.
85 85 """
86 86 raise NotImplementedError, "Implement this method in child class"
87 87
88 88 def close(self):
89 89 """
90 90 Close connection to server
91 91 """
92 92 raise NotImplementedError, "Implement this method in child class"
93 93
94 94 def mkdir(self, remotefolder):
95 95 """
96 96 Create a folder remotely
97 97 """
98 98 raise NotImplementedError, "Implement this method in child class"
99 99
100 100 def cd(self, remotefolder):
101 101 """
102 102 Change working directory in remote server
103 103 """
104 104 raise NotImplementedError, "Implement this method in child class"
105 105
106 106 def download(self, filename, localfolder=None):
107 107 """
108 108 Download a file from server to local host
109 109 """
110 110 raise NotImplementedError, "Implement this method in child class"
111 111
112 112 def sendFile(self, fullfilename):
113 113 """
114 114 sendFile method is used to upload a local file to the current directory in remote server
115 115
116 116 Inputs:
117 117 fullfilename - full path name of local file to store in remote directory
118 118
119 119 Returns:
120 120 0 in error case else 1
121 121 """
122 122 raise NotImplementedError, "Implement this method in child class"
123 123
124 124 def upload(self, fullfilename, remotefolder=None):
125 125 """
126 126 upload method is used to upload a local file to remote directory. This method changes
127 127 working directory before sending a file.
128 128
129 129 Inputs:
130 130 fullfilename - full path name of local file to store in remote directory
131 131
132 132 remotefolder - remote directory
133 133
134 134 Returns:
135 135 0 in error case else 1
136 136 """
137 137 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
138 138
139 139 if not self.status:
140 140 return 0
141 141
142 142 if remotefolder == None:
143 143 remotefolder = self.remotefolder
144 144
145 145 if not self.cd(remotefolder):
146 146 return 0
147 147
148 148 if not self.sendFile(fullfilename):
149 149 print "[Remote Server] Error uploading file %s" %fullfilename
150 150 return 0
151 151
152 152 print "[Remote Server] upload finished successfully"
153 153
154 154 return 1
155 155
156 156 def delete(self, filename):
157 157 """
158 158 Remove a file from remote server
159 159 """
160 160 pass
161 161
162 162 def updateFileList(self, fileList):
163 163 """
164 164 Remove a file from remote server
165 165 """
166 166
167 167 if fileList == self.fileList:
168 168 return 0
169 169
170 170 self.mutex.acquire()
171 171 # init = time.time()
172 172 #
173 173 # while(self.bussy):
174 174 # sleep(0.1)
175 175 # if time.time() - init > 2*self.period:
176 176 # return 0
177 177
178 178 self.fileList = fileList
179 179 self.mutex.release()
180 180 return 1
181 181
182 182 def run(self):
183 183
184 184 if not self.status:
185 185 print "Finishing FTP service"
186 186 return
187 187
188 188 if not self.cd(self.remotefolder):
189 189 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
190 190
191 191 while True:
192 192
193 193 for i in range(self.period):
194 194 if self.stopFlag:
195 195 break
196 196 sleep(1)
197 197
198 198 if self.stopFlag:
199 199 break
200 200
201 201 # self.bussy = True
202 202 self.mutex.acquire()
203 203
204 204 print "[Remote Server] Opening %s" %self.__server
205 205 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 206 self.mutex.release()
207 207 continue
208 208
209 209 for thisFile in self.fileList:
210 210 self.upload(thisFile, self.remotefolder)
211 211
212 212 print "[Remote Server] Closing %s" %self.__server
213 213 self.close()
214 214
215 215 self.mutex.release()
216 216 # self.bussy = False
217 217
218 218 print "[Remote Server] Thread stopped successfully"
219 219
220 220 class FTPClient(Remote):
221 221
222 222 __ftpClientObj = None
223 223
224 224 def __init__(self, server, username, password, remotefolder, period=60):
225 225 """
226 226 """
227 227 Remote.__init__(self, server, username, password, remotefolder, period)
228 228
229 229 def open(self, server, username, password, remotefolder):
230 230
231 231 """
232 232 This method is used to set FTP parameters and establish a connection to remote server
233 233
234 234 Inputs:
235 235 server - remote server IP Address
236 236
237 237 username - remote server Username
238 238
239 239 password - remote server password
240 240
241 241 remotefolder - remote server current working directory
242 242
243 243 Return:
244 244 Boolean - Returns 1 if a connection has been established, 0 otherwise
245 245
246 246 Affects:
247 247 self.status - in case of error or fail connection this parameter is set to 0 else 1
248 248
249 249 """
250 250
251 251 if server == None:
252 252 raise ValueError, "FTP server should be defined"
253 253
254 254 if username == None:
255 255 raise ValueError, "FTP username should be defined"
256 256
257 257 if password == None:
258 258 raise ValueError, "FTP password should be defined"
259 259
260 260 if remotefolder == None:
261 261 raise ValueError, "FTP remote folder should be defined"
262 262
263 263 try:
264 264 ftpClientObj = ftplib.FTP(server)
265 265 except ftplib.all_errors, e:
266 266 print "[FTP Server]: FTP server connection fail: %s" %server
267 267 print "[FTP Server]:", e
268 268 self.status = 0
269 269 return 0
270 270
271 271 try:
272 272 ftpClientObj.login(username, password)
273 273 except ftplib.all_errors:
274 274 print "[FTP Server]: FTP username or password are incorrect"
275 275 self.status = 0
276 276 return 0
277 277
278 278 if remotefolder == None:
279 279 remotefolder = ftpClientObj.pwd()
280 280 else:
281 281 try:
282 282 ftpClientObj.cwd(remotefolder)
283 283 except ftplib.all_errors:
284 284 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
285 285 remotefolder = ftpClientObj.pwd()
286 286
287 287 self.server = server
288 288 self.username = username
289 289 self.password = password
290 290 self.remotefolder = remotefolder
291 291 self.__ftpClientObj = ftpClientObj
292 292 self.status = 1
293 293
294 294 return 1
295 295
296 296 def close(self):
297 297 """
298 298 Close connection to remote server
299 299 """
300 300 if not self.status:
301 301 return 0
302 302
303 303 self.__ftpClientObj.close()
304 304
305 305 def mkdir(self, remotefolder):
306 306 """
307 307 mkdir is used to make a new directory in remote server
308 308
309 309 Input:
310 310 remotefolder - directory name
311 311
312 312 Return:
313 313 0 in error case else 1
314 314 """
315 315 if not self.status:
316 316 return 0
317 317
318 318 try:
319 319 self.__ftpClientObj.mkd(dirname)
320 320 except ftplib.all_errors:
321 321 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
322 322 return 0
323 323
324 324 return 1
325 325
326 326 def cd(self, remotefolder):
327 327 """
328 328 cd is used to change remote working directory on server
329 329
330 330 Input:
331 331 remotefolder - current working directory
332 332
333 333 Affects:
334 334 self.remotefolder
335 335
336 336 Return:
337 337 0 in case of error else 1
338 338 """
339 339 if not self.status:
340 340 return 0
341 341
342 342 if remotefolder == self.remotefolder:
343 343 return 1
344 344
345 345 try:
346 346 self.__ftpClientObj.cwd(remotefolder)
347 347 except ftplib.all_errors:
348 348 print '[FTP Server]: Error changing to %s' %remotefolder
349 349 print '[FTP Server]: Trying to create remote folder'
350 350
351 351 if not self.mkdir(remotefolder):
352 352 print '[FTP Server]: Remote folder could not be created'
353 353 return 0
354 354
355 355 try:
356 356 self.__ftpClientObj.cwd(remotefolder)
357 357 except ftplib.all_errors:
358 358 return 0
359 359
360 360 self.remotefolder = remotefolder
361 361
362 362 return 1
363 363
364 364 def sendFile(self, fullfilename):
365 365
366 366 if not self.status:
367 367 return 0
368 368
369 369 fp = open(fullfilename, 'rb')
370 370
371 371 filename = os.path.basename(fullfilename)
372 372
373 373 command = "STOR %s" %filename
374 374
375 375 try:
376 376 self.__ftpClientObj.storbinary(command, fp)
377 377 except ftplib.all_errors, e:
378 378 print "[FTP Server]:", e
379 379 return 0
380 380
381 381 try:
382 382 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 383 except ftplib.all_errors, e:
384 384 print "[FTP Server]:", e
385 385
386 386 fp.close()
387 387
388 388 return 1
389 389
390 390 class SSHClient(Remote):
391 391
392 392 __sshClientObj = None
393 393 __scpClientObj = None
394 394
395 395 def __init__(self, server, username, password, remotefolder, period=60):
396 396 """
397 397 """
398 398 Remote.__init__(self, server, username, password, remotefolder, period)
399 399
400 400 def open(self, server, username, password, remotefolder, port=22):
401 401
402 402 """
403 403 This method is used to set SSH parameters and establish a connection to a remote server
404 404
405 405 Inputs:
406 406 server - remote server IP Address
407 407
408 408 username - remote server Username
409 409
410 410 password - remote server password
411 411
412 412 remotefolder - remote server current working directory
413 413
414 414 Return: void
415 415
416 416 Affects:
417 417 self.status - in case of error or fail connection this parameter is set to 0 else 1
418 418
419 419 """
420 420 import socket
421 421
422 422 if server == None:
423 423 raise ValueError, "SSH server should be defined"
424 424
425 425 if username == None:
426 426 raise ValueError, "SSH username should be defined"
427 427
428 428 if password == None:
429 429 raise ValueError, "SSH password should be defined"
430 430
431 431 if remotefolder == None:
432 432 raise ValueError, "SSH remote folder should be defined"
433 433
434 434 sshClientObj = paramiko.SSHClient()
435 435
436 436 sshClientObj.load_system_host_keys()
437 437 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
438 438
439 439 self.status = 0
440 440 try:
441 441 sshClientObj.connect(server, username=username, password=password, port=port)
442 442 except paramiko.AuthenticationException, e:
443 443 # print "SSH username or password are incorrect: %s"
444 444 print "[SSH Server]:", e
445 445 return 0
446 446 except SSHException, e:
447 447 print "[SSH Server]:", e
448 448 return 0
449 449 except socket.error:
450 450 self.status = 0
451 451 print "[SSH Server]:", e
452 452 return 0
453 453
454 454 self.status = 1
455 455 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
456 456
457 457 if remotefolder == None:
458 458 remotefolder = self.pwd()
459 459
460 460 self.server = server
461 461 self.username = username
462 462 self.password = password
463 463 self.__sshClientObj = sshClientObj
464 464 self.__scpClientObj = scpClientObj
465 465 self.status = 1
466 466
467 467 if not self.cd(remotefolder):
468 468 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
469 469 return 0
470 470
471 471 self.remotefolder = remotefolder
472 472
473 473 return 1
474 474
475 475 def close(self):
476 476 """
477 477 Close connection to remote server
478 478 """
479 479 if not self.status:
480 480 return 0
481 481
482 482 self.__scpClientObj.close()
483 483 self.__sshClientObj.close()
484 484
485 485 def __execute(self, command):
486 486 """
487 487 __execute a command on remote server
488 488
489 489 Input:
490 490 command - Exmaple 'ls -l'
491 491
492 492 Return:
493 493 0 in error case else 1
494 494 """
495 495 if not self.status:
496 496 return 0
497 497
498 498 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
499 499
500 500 result = stderr.readlines()
501 501 if len(result) > 1:
502 502 return 0
503 503
504 504 result = stdout.readlines()
505 505 if len(result) > 1:
506 506 return result[0][:-1]
507 507
508 508 return 1
509 509
510 510 def mkdir(self, remotefolder):
511 511 """
512 512 mkdir is used to make a new directory in remote server
513 513
514 514 Input:
515 515 remotefolder - directory name
516 516
517 517 Return:
518 518 0 in error case else 1
519 519 """
520 520
521 521 command = 'mkdir %s' %remotefolder
522 522
523 523 return self.__execute(command)
524 524
525 525 def pwd(self):
526 526
527 527 command = 'pwd'
528 528
529 529 return self.__execute(command)
530 530
531 531 def cd(self, remotefolder):
532 532 """
533 533 cd is used to change remote working directory on server
534 534
535 535 Input:
536 536 remotefolder - current working directory
537 537
538 538 Affects:
539 539 self.remotefolder
540 540
541 541 Return:
542 542 0 in case of error else 1
543 543 """
544 544 if not self.status:
545 545 return 0
546 546
547 547 if remotefolder == self.remotefolder:
548 548 return 1
549 549
550 550 chk_command = "cd %s; pwd" %remotefolder
551 551 mkdir_command = "mkdir %s" %remotefolder
552 552
553 553 if not self.__execute(chk_command):
554 554 if not self.__execute(mkdir_command):
555 555 self.remotefolder = None
556 556 return 0
557 557
558 558 self.remotefolder = remotefolder
559 559
560 560 return 1
561 561
562 562 def sendFile(self, fullfilename):
563 563
564 564 if not self.status:
565 565 return 0
566 566
567 567 try:
568 568 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
569 569 except scp.ScpError, e:
570 570 print "[SSH Server]", str(e)
571 571 return 0
572 572
573 573 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
574 574 command = 'chmod 775 %s' %remotefile
575 575
576 576 return self.__execute(command)
577 577
578 578 class SendToServer(ProcessingUnit):
579 579
580 580 def __init__(self, **kwargs):
581 581
582 582 ProcessingUnit.__init__(self, **kwargs)
583 583
584 584 self.isConfig = False
585 585 self.clientObj = None
586 586
587 587 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
588 588
589 589 self.clientObj = None
590 590 self.localfolder = localfolder
591 591 self.ext = ext
592 592 self.period = period
593 593
594 594 if str.lower(protocol) == 'ftp':
595 595 self.clientObj = FTPClient(server, username, password, remotefolder, period)
596 596
597 597 if str.lower(protocol) == 'ssh':
598 598 self.clientObj = SSHClient(server, username, password, remotefolder, period)
599 599
600 600 if not self.clientObj:
601 601 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
602 602
603 603 self.clientObj.start()
604 604
605 605 def findFiles(self):
606 606
607 607 if not type(self.localfolder) == list:
608 608 folderList = [self.localfolder]
609 609 else:
610 610 folderList = self.localfolder
611 611
612 612 #Remove duplicate items
613 613 folderList = list(set(folderList))
614 614
615 615 fullfilenameList = []
616 616
617 617 for thisFolder in folderList:
618 618
619 619 print "[Remote Server]: Searching files on %s" %thisFolder
620 620
621 621 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
622 622
623 623 if len(filenameList) < 1:
624 624 continue
625 625
626 626 for thisFile in filenameList:
627 627 fullfilename = os.path.join(thisFolder, thisFile)
628 628
629 629 if fullfilename in fullfilenameList:
630 630 continue
631 631
632 632 #Only files modified in the last 30 minutes are considered
633 633 if os.path.getmtime(fullfilename) < time.time() - 30*60:
634 634 continue
635 635
636 636 fullfilenameList.append(fullfilename)
637 637
638 638 return fullfilenameList
639 639
640 640 def run(self, **kwargs):
641
642 641 if not self.isConfig:
643 642 self.init = time.time()
644 643 self.setup(**kwargs)
645 644 self.isConfig = True
646
645
646 if not self.clientObj.is_alive():
647 print "[Remote Server]: Restarting connection "
648 self.setup(**kwargs)
649
647 650 if time.time() - self.init >= self.period:
648 651 fullfilenameList = self.findFiles()
649 652
650 653 if self.clientObj.updateFileList(fullfilenameList):
651 654 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
652
653 655 self.init = time.time()
654 656
655 657 def close(self):
656 658 print "[Remote Server] Stopping thread"
657 659 self.clientObj.stop()
658 660
659 661
660 662 class FTP(object):
661 663 """
662 664 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
663 665
664 666 Non-standard Python modules used: None
665 667
666 668 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
667 669 """
668 670
669 671 def __init__(self,server = None, username=None, password=None, remotefolder=None):
670 672 """
671 673 This method is used to setting parameters for FTP and establishing connection to remote server
672 674
673 675 Inputs:
674 676 server - remote server IP Address
675 677
676 678 username - remote server Username
677 679
678 680 password - remote server password
679 681
680 682 remotefolder - remote server current working directory
681 683
682 684 Return: void
683 685
684 686 Affects:
685 687 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
686 688
687 689 self.folderList - sub-folder list of remote folder
688 690
689 691 self.fileList - file list of remote folder
690 692
691 693
692 694 """
693 695
694 696 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
695 697 server, username, password, remotefolder = self.parmsByDefault()
696 698
697 699 self.server = server
698 700 self.username = username
699 701 self.password = password
700 702 self.remotefolder = remotefolder
701 703 self.file = None
702 704 self.ftp = None
703 705 self.status = 0
704 706
705 707 try:
706 708 self.ftp = ftplib.FTP(self.server)
707 709 self.ftp.login(self.username,self.password)
708 710 self.ftp.cwd(self.remotefolder)
709 711 # print 'Connect to FTP Server: Successfully'
710 712
711 713 except ftplib.all_errors:
712 714 print 'Error FTP Service'
713 715 self.status = 1
714 716 return
715 717
716 718
717 719
718 720 self.dirList = []
719 721
720 722 try:
721 723 self.dirList = self.ftp.nlst()
722 724
723 725 except ftplib.error_perm, resp:
724 726 if str(resp) == "550 No files found":
725 727 print "no files in this directory"
726 728 self.status = 1
727 729 return
728 730
729 731 except ftplib.all_errors:
730 732 print 'Error Displaying Dir-Files'
731 733 self.status = 1
732 734 return
733 735
734 736 self.fileList = []
735 737 self.folderList = []
736 738 #only for test
737 739 for f in self.dirList:
738 740 name, ext = os.path.splitext(f)
739 741 if ext != '':
740 742 self.fileList.append(f)
741 743 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
742 744
743 745 def parmsByDefault(self):
744 746 server = 'jro-app.igp.gob.pe'
745 747 username = 'wmaster'
746 748 password = 'mst2010vhf'
747 749 remotefolder = '/home/wmaster/graficos'
748 750
749 751 return server, username, password, remotefolder
750 752
751 753
752 754 def mkd(self,dirname):
753 755 """
754 756 mkd is used to make directory in remote server
755 757
756 758 Input:
757 759 dirname - directory name
758 760
759 761 Return:
760 762 1 in error case else 0
761 763 """
762 764 try:
763 765 self.ftp.mkd(dirname)
764 766 except:
765 767 print 'Error creating remote folder:%s'%dirname
766 768 return 1
767 769
768 770 return 0
769 771
770 772
771 773 def delete(self,filename):
772 774 """
773 775 delete is used to delete file in current working directory of remote server
774 776
775 777 Input:
776 778 filename - filename to delete in remote folder
777 779
778 780 Return:
779 781 1 in error case else 0
780 782 """
781 783
782 784 try:
783 785 self.ftp.delete(filename)
784 786 except:
785 787 print 'Error deleting remote file:%s'%filename
786 788 return 1
787 789
788 790 return 0
789 791
790 792 def download(self,filename,localfolder):
791 793 """
792 794 download is used to downloading file from remote folder into local folder
793 795
794 796 Inputs:
795 797 filename - filename to donwload
796 798
797 799 localfolder - directory local to store filename
798 800
799 801 Returns:
800 802 self.status - 1 in error case else 0
801 803 """
802 804
803 805 self.status = 0
804 806
805 807
806 808 if not(filename in self.fileList):
807 809 print 'filename:%s not exists'%filename
808 810 self.status = 1
809 811 return self.status
810 812
811 813 newfilename = os.path.join(localfolder,filename)
812 814
813 815 self.file = open(newfilename, 'wb')
814 816
815 817 try:
816 818 print 'Download: ' + filename
817 819 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
818 820 print 'Download Complete'
819 821 except ftplib.all_errors:
820 822 print 'Error Downloading ' + filename
821 823 self.status = 1
822 824 return self.status
823 825
824 826 self.file.close()
825 827
826 828 return self.status
827 829
828 830
829 831 def __handleDownload(self,block):
830 832 """
831 833 __handleDownload is used to handle writing file
832 834 """
833 835 self.file.write(block)
834 836
835 837
836 838 def upload(self,filename,remotefolder=None):
837 839 """
838 840 upload is used to uploading local file to remote directory
839 841
840 842 Inputs:
841 843 filename - full path name of local file to store in remote directory
842 844
843 845 remotefolder - remote directory
844 846
845 847 Returns:
846 848 self.status - 1 in error case else 0
847 849 """
848 850
849 851 if remotefolder == None:
850 852 remotefolder = self.remotefolder
851 853
852 854 self.status = 0
853 855
854 856 try:
855 857 self.ftp.cwd(remotefolder)
856 858
857 859 self.file = open(filename, 'rb')
858 860
859 861 (head, tail) = os.path.split(filename)
860 862
861 863 command = "STOR " + tail
862 864
863 865 print 'Uploading: ' + tail
864 866 self.ftp.storbinary(command, self.file)
865 867 print 'Upload Completed'
866 868
867 869 except ftplib.all_errors:
868 870 print 'Error Uploading ' + tail
869 871 self.status = 1
870 872 return self.status
871 873
872 874 self.file.close()
873 875
874 876 #back to initial directory in __init__()
875 877 self.ftp.cwd(self.remotefolder)
876 878
877 879 return self.status
878 880
879 881
880 882 def dir(self,remotefolder):
881 883 """
882 884 dir is used to change working directory of remote server and get folder and file list
883 885
884 886 Input:
885 887 remotefolder - current working directory
886 888
887 889 Affects:
888 890 self.fileList - file list of working directory
889 891
890 892 Return:
891 893 infoList - list with filenames and size of file in bytes
892 894
893 895 self.folderList - folder list
894 896 """
895 897
896 898 self.remotefolder = remotefolder
897 899 print 'Change to ' + self.remotefolder
898 900 try:
899 901 self.ftp.cwd(remotefolder)
900 902 except ftplib.all_errors:
901 903 print 'Error Change to ' + self.remotefolder
902 904 infoList = None
903 905 self.folderList = None
904 906 return infoList,self.folderList
905 907
906 908 self.dirList = []
907 909
908 910 try:
909 911 self.dirList = self.ftp.nlst()
910 912
911 913 except ftplib.error_perm, resp:
912 914 if str(resp) == "550 No files found":
913 915 print "no files in this directory"
914 916 infoList = None
915 917 self.folderList = None
916 918 return infoList,self.folderList
917 919 except ftplib.all_errors:
918 920 print 'Error Displaying Dir-Files'
919 921 infoList = None
920 922 self.folderList = None
921 923 return infoList,self.folderList
922 924
923 925 infoList = []
924 926 self.fileList = []
925 927 self.folderList = []
926 928 for f in self.dirList:
927 929 name,ext = os.path.splitext(f)
928 930 if ext != '':
929 931 self.fileList.append(f)
930 932 value = (f,self.ftp.size(f))
931 933 infoList.append(value)
932 934
933 935 if ext == '':
934 936 self.folderList.append(f)
935 937
936 938 return infoList,self.folderList
937 939
938 940
939 941 def close(self):
940 942 """
941 943 close is used to close and end FTP connection
942 944
943 945 Inputs: None
944 946
945 947 Return: void
946 948
947 949 """
948 950 self.ftp.close()
949 951
950 952 class SendByFTP(Operation):
951 953
952 954 def __init__(self, **kwargs):
953 955 Operation.__init__(self, **kwargs)
954 956 self.status = 1
955 957 self.counter = 0
956 958
957 959 def error_print(self, ValueError):
958 960
959 961 print ValueError, 'Error FTP'
960 962 print "don't worry the program is running..."
961 963
962 964 def worker_ftp(self, server, username, password, remotefolder, filenameList):
963 965
964 966 self.ftpClientObj = FTP(server, username, password, remotefolder)
965 967 for filename in filenameList:
966 968 self.ftpClientObj.upload(filename)
967 969 self.ftpClientObj.close()
968 970
969 971 def ftp_thread(self, server, username, password, remotefolder):
970 972 if not(self.status):
971 973 return
972 974
973 975 import multiprocessing
974 976
975 977 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
976 978 p.start()
977 979
978 980 p.join(3)
979 981
980 982 if p.is_alive():
981 983 p.terminate()
982 984 p.join()
983 985 print 'killing ftp process...'
984 986 self.status = 0
985 987 return
986 988
987 989 self.status = 1
988 990 return
989 991
990 992 def filterByExt(self, ext, localfolder):
991 993 fnameList = glob.glob1(localfolder,ext)
992 994 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
993 995
994 996 if len(self.filenameList) == 0:
995 997 self.status = 0
996 998
997 999 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
998 1000
999 1001 self.counter += 1
1000 1002 if self.counter >= period:
1001 1003 self.filterByExt(ext, localfolder)
1002 1004
1003 1005 self.ftp_thread(server, username, password, remotefolder)
1004 1006
1005 1007 self.counter = 0
1006 1008
1007 1009 self.status = 1
Show file before | Show file after | Hide comments
@@ -1,113 +1,117
1 1 import h5py
2 2 import numpy
3 3 import matplotlib.pyplot as plt
4 4 import glob
5 5 import os
6 6
7 7 #---------------------- Functions ---------------------
8 8
9 9 def findFiles(path):
10 10
11 11 dirList = []
12 12 fileList = []
13 13
14 14 for thisPath in os.listdir(path):
15 15 dirList.append(os.path.join(path,thisPath))
16 16 dirList.sort()
17 17
18 18 for thisDirectory in dirList:
19 19 files = glob.glob1(thisDirectory, "*.hdf5")
20 20 files.sort()
21 21 for thisFile in files:
22 22 fileList.append(os.path.join(thisDirectory,thisFile))
23 23
24 24 return fileList
25 25
26 26 def readFiles(fileList):
27 27
28 28 meteors_array = numpy.zeros((1,4))
29 29
30 30 for thisFile in fileList:
31 31
32 32 #Leer
33 33 f1 = h5py.File(thisFile,'r')
34 34 grp1 = f1['Data']
35 35 grp2 = grp1['data_output']
36 36 meteors1 = grp2['table0'][:]
37 37 meteors_array = numpy.vstack((meteors_array,meteors1))
38 38 #cerrar
39 39 f1.close()
40 40
41 41 meteors_array = numpy.delete(meteors_array, 0, axis=0)
42 42 meteors_list = [meteors_array[:,0],meteors_array[:,1],meteors_array[:,2],meteors_array[:,3]]
43 43 return meteors_list
44 44
45 45 def estimateMean(offset_list):
46 46
47 47 mean_off = []
48 48 axisY_off = []
49 49 axisX_off = []
50 50
51 51 for thisOffset in offset_list:
52 52 mean_aux = numpy.mean(thisOffset, axis = 0)
53 53 mean_off.append(mean_aux)
54 54 axisX_off.append(numpy.array([0,numpy.size(thisOffset)]))
55 55 axisY_off.append(numpy.array([mean_aux,mean_aux]))
56 56
57 57 return mean_off, axisY_off, axisX_off
58 58
59 59 def plotPhases(offset0, axisY0, axisX0, title):
60 60 f, axarr = plt.subplots(4, sharey=True)
61 61 color = ['b','g','r','c']
62 62 # plt.grid()
63 63 for i in range(len(offset0)):
64 64 thisMeteor = offset0[i]
65 65 thisY = axisY0[i]
66 66 thisX = axisX0[i]
67 67 thisColor = color[i]
68 68
69 69 opt = thisColor + 'o'
70 70 axarr[i].plot(thisMeteor,opt)
71 71 axarr[i].plot(thisX, thisY, thisColor)
72 72 axarr[i].set_ylabel('Offset ' + str(i))
73 73
74 74 plt.ylim((-180,180))
75 75 axarr[0].set_title(title + ' Offsets')
76 76 axarr[3].set_xlabel('Number of estimations')
77 77
78 78 return
79 79
80 80 def filterOffsets(offsets0, stdvLimit):
81 81 offsets1 = []
82 82
83 83 for thisOffset in offsets0:
84 84 pstd = numpy.std(thisOffset)*stdvLimit
85 85 pmean = numpy.mean(thisOffset)
86 86 outlier1 = thisOffset > pmean - pstd
87 87 outlier2 = thisOffset < pmean + pstd
88 88 not_outlier = numpy.logical_and(outlier1,outlier2)
89 89 thisOffset1 = thisOffset[not_outlier]
90 90 offsets1.append(thisOffset1)
91 91
92 92 return offsets1
93 93
94 94 #---------------------- Setup ---------------------------
95 95
96 <<<<<<< HEAD
96 97 path = '/home/nanosat/Pictures/JASMET30_mp/201608/phase'
98 =======
99 path = '/home/jespinoza/Pictures/JASMET30/201608/phase'
100 >>>>>>> master
97 101 stdvLimit = 0.5
98 102
99 103 #---------------------- Script ---------------------------
100 104
101 105 fileList = findFiles(path)
102 106 offsets0 = readFiles(fileList)
103 107 mean0, axisY0, axisX0 = estimateMean(offsets0)
104 108 plotPhases(offsets0, axisY0, axisX0, 'Original')
105 109
106 110 offsets1 = filterOffsets(offsets0, stdvLimit)
107 111 mean1, axisY1, axisX1 = estimateMean(offsets1)
108 112 plotPhases(offsets1, axisY1, axisX1, 'Filtered')
109 113
110 114 print "Original Offsets: %.2f, %.2f, %.2f, %.2f" % (mean0[0],mean0[1],mean0[2],mean0[3])
111 115 print "Filtered Offsets: %.2f, %.2f, %.2f, %.2f" % (mean1[0],mean1[1],mean1[2],mean1[3])
112 116
113 117 plt.show()
General Comments 0
You need to be logged in to leave comments. Login now