##// END OF EJS Templates
schain-jc
RSS
setup actualizado para no necesitar numpy previamente
José Chávez -
r1026:cfaa3fd99bda
parent child
Show More
Side by Side Unified
Context file:
r1026:cfaa3fd99bda Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,108 +1,108
1 1 # Byte-compiled / optimized / DLL files
2 2 __pycache__/
3 3 *.py[cod]
4 4 *$py.class
5 5
6 6 # C extensions
7 7 *.so
8 8
9 9 # Distribution / packaging
10 10 .Python
11 11 env/
12 12 build/
13 13 develop-eggs/
14 14 dist/
15 15 downloads/
16 16 eggs/
17 17 .eggs/
18 18 lib/
19 19 lib64/
20 20 parts/
21 21 sdist/
22 22 var/
23 23 wheels/
24 24 *.egg-info/
25 25 .installed.cfg
26 26 *.egg
27 27
28 28 # PyInstaller
29 29 # Usually these files are written by a python script from a template
30 30 # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 31 *.manifest
32 32 *.spec
33 33
34 34 # Installer logs
35 35 pip-log.txt
36 36 pip-delete-this-directory.txt
37 37
38 38 # Unit test / coverage reports
39 39 htmlcov/
40 40 .tox/
41 41 .coverage
42 42 .coverage.*
43 43 .cache
44 44 nosetests.xml
45 45 coverage.xml
46 46 *,cover
47 47 .hypothesis/
48 48
49 49 # Translations
50 50 *.mo
51 51 *.pot
52 52
53 53 # Django stuff:
54 54 *.log
55 55 local_settings.py
56 56
57 57 # Flask stuff:
58 58 instance/
59 59 .webassets-cache
60 60
61 61 # Scrapy stuff:
62 62 .scrapy
63 63
64 64 # Sphinx documentation
65 65 docs/_build/
66 66
67 67 # PyBuilder
68 68 target/
69 69
70 70 # Jupyter Notebook
71 71 .ipynb_checkpoints
72 72
73 73 # pyenv
74 74 .python-version
75 75
76 76 # celery beat schedule file
77 77 celerybeat-schedule
78 78
79 79 # SageMath parsed files
80 80 *.sage.py
81 81
82 82 # dotenv
83 83 .env
84 84
85 85 # virtualenv
86 86 .venv
87 87 venv/
88 88 ENV/
89 89
90 90 # Spyder project settings
91 91 .spyderproject
92 92 .spyproject
93 93
94 94 # Rope project settings
95 95 .ropeproject
96 96
97 97 # mkdocs documentation
98 98 /site
99 99
100 100 # eclipse
101 101 .project
102 102 .pydevproject
103 103 .svn/
104 104 *.png
105 105 *.pyc
106 106 schainpy/scripts
107
107 .vscode
108 108 schaingui/node_modules No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1012 +1,1008
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4 import os
5 5 import glob
6 6 import ftplib
7 7
8 8 try:
9 9 import paramiko
10 10 import scp
11 11 except:
12 12 print "You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server"
13 13
14 14 import time
15 15
16 16 import threading
17 17 Thread = threading.Thread
18 18
19 19 # try:
20 20 # from gevent import sleep
21 21 # except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 25
26 26 class Remote(Thread):
27 27 """
28 28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 29 used to upload or download files remotely.
30 30
31 31 Non-standard Python modules used:
32 32 None
33 33
34 34 Written by:
35
36 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
37
35 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
38 36 """
39 37
40 38 server = None
41 39 username = None
42 40 password = None
43 41 remotefolder = None
44 42
45 43 period = 60
46 44 fileList = []
47 45 bussy = False
48 46
49 47 def __init__(self, server, username, password, remotefolder, period=60):
50 48
51 49 Thread.__init__(self)
52 50
53 51 self.setDaemon(True)
54 52
55 53 self.status = 0
56 54
57 55 self.__server = server
58 56 self.__username = username
59 57 self.__password = password
60 58 self.__remotefolder = remotefolder
61 59
62 60 self.period = period
63 61
64 62 self.fileList = []
65 63 self.bussy = False
66 64
67 65 self.stopFlag = False
68 66
69 67 print "[Remote Server] Opening server: %s" %self.__server
70 68 if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
71 69 print "[Remote Server] %s server was opened successfully" %self.__server
72 70
73 71 self.close()
74 72
75 73 self.mutex = threading.Lock()
76 74
77 75 def stop(self):
78 76
79 77 self.stopFlag = True
80 78 self.join(10)
81 79
82 80 def open(self):
83 81 """
84 82 Connect to server and create a connection class (FTP or SSH) to remote server.
85 83 """
86 84 raise NotImplementedError, "Implement this method in child class"
87 85
88 86 def close(self):
89 87 """
90 88 Close connection to server
91 89 """
92 90 raise NotImplementedError, "Implement this method in child class"
93 91
94 92 def mkdir(self, remotefolder):
95 93 """
96 94 Create a folder remotely
97 95 """
98 96 raise NotImplementedError, "Implement this method in child class"
99 97
100 98 def cd(self, remotefolder):
101 99 """
102 100 Change working directory in remote server
103 101 """
104 102 raise NotImplementedError, "Implement this method in child class"
105 103
106 104 def download(self, filename, localfolder=None):
107 105 """
108 106 Download a file from server to local host
109 107 """
110 108 raise NotImplementedError, "Implement this method in child class"
111 109
112 110 def sendFile(self, fullfilename):
113 111 """
114 112 sendFile method is used to upload a local file to the current directory in remote server
115 113
116 114 Inputs:
117 115 fullfilename - full path name of local file to store in remote directory
118 116
119 117 Returns:
120 118 0 in error case else 1
121 119 """
122 120 raise NotImplementedError, "Implement this method in child class"
123 121
124 122 def upload(self, fullfilename, remotefolder=None):
125 123 """
126 124 upload method is used to upload a local file to remote directory. This method changes
127 125 working directory before sending a file.
128 126
129 127 Inputs:
130 128 fullfilename - full path name of local file to store in remote directory
131 129
132 130 remotefolder - remote directory
133 131
134 132 Returns:
135 133 0 in error case else 1
136 134 """
137 135 print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder)
138 136
139 137 if not self.status:
140 138 return 0
141 139
142 140 if remotefolder == None:
143 141 remotefolder = self.remotefolder
144 142
145 143 if not self.cd(remotefolder):
146 144 return 0
147 145
148 146 if not self.sendFile(fullfilename):
149 147 print "[Remote Server] Error uploading file %s" %fullfilename
150 148 return 0
151 149
152 150 print "[Remote Server] upload finished successfully"
153 151
154 152 return 1
155 153
156 154 def delete(self, filename):
157 155 """
158 156 Remove a file from remote server
159 157 """
160 158 pass
161 159
162 160 def updateFileList(self, fileList):
163 161 """
164 162 Remove a file from remote server
165 163 """
166 164
167 165 if fileList == self.fileList:
168 166 return 0
169 167
170 168 self.mutex.acquire()
171 # init = time.time()
172 #
173 # while(self.bussy):
174 # sleep(0.1)
175 # if time.time() - init > 2*self.period:
176 # return 0
177
169 # init = time.time()
170 #
171 # while(self.bussy):
172 # sleep(0.1)
173 # if time.time() - init > 2*self.period:
174 # return 0
175
178 176 self.fileList = fileList
179 177 self.mutex.release()
180 178 return 1
181 179
182 180 def run(self):
183 181
184 182 if not self.status:
185 183 print "Finishing FTP service"
186 184 return
187 185
188 186 if not self.cd(self.remotefolder):
189 187 raise ValueError, "Could not access to the new remote directory: %s" %self.remotefolder
190 188
191 189 while True:
192 190
193 191 for i in range(self.period):
194 192 if self.stopFlag:
195 193 break
196 194 sleep(1)
197 195
198 196 if self.stopFlag:
199 197 break
200 198
201 # self.bussy = True
199 # self.bussy = True
202 200 self.mutex.acquire()
203 201
204 202 print "[Remote Server] Opening %s" %self.__server
205 203 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 204 self.mutex.release()
207 205 continue
208 206
209 207 for thisFile in self.fileList:
210 208 self.upload(thisFile, self.remotefolder)
211 209
212 210 print "[Remote Server] Closing %s" %self.__server
213 211 self.close()
214 212
215 213 self.mutex.release()
216 214 # self.bussy = False
217 215
218 216 print "[Remote Server] Thread stopped successfully"
219 217
220 218 class FTPClient(Remote):
221 219
222 220 __ftpClientObj = None
223 221
224 222 def __init__(self, server, username, password, remotefolder, period=60):
225 223 """
226 224 """
227 225 Remote.__init__(self, server, username, password, remotefolder, period)
228 226
229 227 def open(self, server, username, password, remotefolder):
230 228
231 229 """
232 230 This method is used to set FTP parameters and establish a connection to remote server
233 231
234 232 Inputs:
235 233 server - remote server IP Address
236 234
237 235 username - remote server Username
238 236
239 237 password - remote server password
240 238
241 239 remotefolder - remote server current working directory
242 240
243 241 Return:
244 242 Boolean - Returns 1 if a connection has been established, 0 otherwise
245 243
246 244 Affects:
247 245 self.status - in case of error or fail connection this parameter is set to 0 else 1
248 246
249 247 """
250 248
251 249 if server == None:
252 250 raise ValueError, "FTP server should be defined"
253 251
254 252 if username == None:
255 253 raise ValueError, "FTP username should be defined"
256 254
257 255 if password == None:
258 256 raise ValueError, "FTP password should be defined"
259 257
260 258 if remotefolder == None:
261 259 raise ValueError, "FTP remote folder should be defined"
262 260
263 261 try:
264 262 ftpClientObj = ftplib.FTP(server)
265 263 except ftplib.all_errors, e:
266 264 print "[FTP Server]: FTP server connection fail: %s" %server
267 265 print "[FTP Server]:", e
268 266 self.status = 0
269 267 return 0
270 268
271 269 try:
272 270 ftpClientObj.login(username, password)
273 271 except ftplib.all_errors:
274 272 print "[FTP Server]: FTP username or password are incorrect"
275 273 self.status = 0
276 274 return 0
277 275
278 276 if remotefolder == None:
279 277 remotefolder = ftpClientObj.pwd()
280 278 else:
281 279 try:
282 280 ftpClientObj.cwd(remotefolder)
283 281 except ftplib.all_errors:
284 282 print "[FTP Server]: FTP remote folder is invalid: %s" %remotefolder
285 283 remotefolder = ftpClientObj.pwd()
286 284
287 285 self.server = server
288 286 self.username = username
289 287 self.password = password
290 288 self.remotefolder = remotefolder
291 289 self.__ftpClientObj = ftpClientObj
292 290 self.status = 1
293 291
294 292 return 1
295 293
296 294 def close(self):
297 295 """
298 296 Close connection to remote server
299 297 """
300 298 if not self.status:
301 299 return 0
302 300
303 301 self.__ftpClientObj.close()
304 302
305 303 def mkdir(self, remotefolder):
306 304 """
307 305 mkdir is used to make a new directory in remote server
308 306
309 307 Input:
310 308 remotefolder - directory name
311 309
312 310 Return:
313 311 0 in error case else 1
314 312 """
315 313 if not self.status:
316 314 return 0
317 315
318 316 try:
319 317 self.__ftpClientObj.mkd(dirname)
320 318 except ftplib.all_errors:
321 319 print "[FTP Server]: Error creating remote folder: %s" %remotefolder
322 320 return 0
323 321
324 322 return 1
325 323
326 324 def cd(self, remotefolder):
327 325 """
328 326 cd is used to change remote working directory on server
329 327
330 328 Input:
331 329 remotefolder - current working directory
332 330
333 331 Affects:
334 332 self.remotefolder
335 333
336 334 Return:
337 335 0 in case of error else 1
338 336 """
339 337 if not self.status:
340 338 return 0
341 339
342 340 if remotefolder == self.remotefolder:
343 341 return 1
344 342
345 343 try:
346 344 self.__ftpClientObj.cwd(remotefolder)
347 345 except ftplib.all_errors:
348 346 print '[FTP Server]: Error changing to %s' %remotefolder
349 347 print '[FTP Server]: Trying to create remote folder'
350 348
351 349 if not self.mkdir(remotefolder):
352 350 print '[FTP Server]: Remote folder could not be created'
353 351 return 0
354 352
355 353 try:
356 354 self.__ftpClientObj.cwd(remotefolder)
357 355 except ftplib.all_errors:
358 356 return 0
359 357
360 358 self.remotefolder = remotefolder
361 359
362 360 return 1
363 361
364 362 def sendFile(self, fullfilename):
365 363
366 364 if not self.status:
367 365 return 0
368 366
369 367 fp = open(fullfilename, 'rb')
370 368
371 369 filename = os.path.basename(fullfilename)
372 370
373 371 command = "STOR %s" %filename
374 372
375 373 try:
376 374 self.__ftpClientObj.storbinary(command, fp)
377 375 except ftplib.all_errors, e:
378 376 print "[FTP Server]:", e
379 377 return 0
380 378
381 379 try:
382 380 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 381 except ftplib.all_errors, e:
384 382 print "[FTP Server]:", e
385 383
386 384 fp.close()
387 385
388 386 return 1
389 387
390 388 class SSHClient(Remote):
391 389
392 390 __sshClientObj = None
393 391 __scpClientObj = None
394 392
395 393 def __init__(self, server, username, password, remotefolder, period=60):
396 394 """
397 395 """
398 396 Remote.__init__(self, server, username, password, remotefolder, period)
399 397
400 398 def open(self, server, username, password, remotefolder, port=22):
401 399
402 400 """
403 This method is used to set SSH parameters and establish a connection to a remote server
404
405 Inputs:
406 server - remote server IP Address
401 This method is used to set SSH parameters and establish a connection to a remote server
407 402
408 username - remote server Username
403 Inputs:
404 server - remote server IP Address
405
406 username - remote server Username
407
408 password - remote server password
409
410 remotefolder - remote server current working directory
409 411
410 password - remote server password
412 Return: void
411 413
412 remotefolder - remote server current working directory
413
414 Return: void
415
416 Affects:
417 self.status - in case of error or fail connection this parameter is set to 0 else 1
414 Affects:
415 self.status - in case of error or fail connection this parameter is set to 0 else 1
418 416
419 417 """
420 418 import socket
421 419
422 420 if server == None:
423 421 raise ValueError, "SSH server should be defined"
424 422
425 423 if username == None:
426 424 raise ValueError, "SSH username should be defined"
427 425
428 426 if password == None:
429 427 raise ValueError, "SSH password should be defined"
430 428
431 429 if remotefolder == None:
432 430 raise ValueError, "SSH remote folder should be defined"
433 431
434 432 sshClientObj = paramiko.SSHClient()
435 433
436 434 sshClientObj.load_system_host_keys()
437 435 sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
438 436
439 437 self.status = 0
440 438 try:
441 439 sshClientObj.connect(server, username=username, password=password, port=port)
442 440 except paramiko.AuthenticationException, e:
443 # print "SSH username or password are incorrect: %s"
441 # print "SSH username or password are incorrect: %s"
444 442 print "[SSH Server]:", e
445 443 return 0
446 444 except SSHException, e:
447 445 print "[SSH Server]:", e
448 446 return 0
449 447 except socket.error:
450 448 self.status = 0
451 449 print "[SSH Server]:", e
452 450 return 0
453 451
454 452 self.status = 1
455 453 scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
456 454
457 455 if remotefolder == None:
458 456 remotefolder = self.pwd()
459 457
460 458 self.server = server
461 459 self.username = username
462 460 self.password = password
463 461 self.__sshClientObj = sshClientObj
464 462 self.__scpClientObj = scpClientObj
465 463 self.status = 1
466 464
467 465 if not self.cd(remotefolder):
468 466 raise ValueError, "[SSH Server]: Could not access to remote folder: %s" %remotefolder
469 467 return 0
470 468
471 469 self.remotefolder = remotefolder
472 470
473 471 return 1
474 472
475 473 def close(self):
476 474 """
477 Close connection to remote server
475 Close connection to remote server
478 476 """
479 477 if not self.status:
480 478 return 0
481 479
482 480 self.__scpClientObj.close()
483 481 self.__sshClientObj.close()
484 482
485 483 def __execute(self, command):
486 484 """
487 __execute a command on remote server
488
489 Input:
490 command - Exmaple 'ls -l'
491
492 Return:
493 0 in error case else 1
485 __execute a command on remote server
486
487 Input:
488 command - Exmaple 'ls -l'
489
490 Return:
491 0 in error case else 1
494 492 """
495 493 if not self.status:
496 494 return 0
497 495
498 496 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
499 497
500 498 result = stderr.readlines()
501 499 if len(result) > 1:
502 500 return 0
503 501
504 502 result = stdout.readlines()
505 503 if len(result) > 1:
506 504 return result[0][:-1]
507 505
508 506 return 1
509 507
510 508 def mkdir(self, remotefolder):
511 509 """
512 mkdir is used to make a new directory in remote server
513
514 Input:
515 remotefolder - directory name
516
517 Return:
518 0 in error case else 1
510 mkdir is used to make a new directory in remote server
511
512 Input:
513 remotefolder - directory name
514
515 Return:
516 0 in error case else 1
519 517 """
520 518
521 519 command = 'mkdir %s' %remotefolder
522 520
523 521 return self.__execute(command)
524 522
525 523 def pwd(self):
526 524
527 525 command = 'pwd'
528 526
529 527 return self.__execute(command)
530 528
531 529 def cd(self, remotefolder):
532 530 """
533 cd is used to change remote working directory on server
534
535 Input:
536 remotefolder - current working directory
531 cd is used to change remote working directory on server
537 532
538 Affects:
539 self.remotefolder
540
541 Return:
542 0 in case of error else 1
533 Input:
534 remotefolder - current working directory
535
536 Affects:
537 self.remotefolder
538
539 Return:
540 0 in case of error else 1
543 541 """
544 542 if not self.status:
545 543 return 0
546 544
547 545 if remotefolder == self.remotefolder:
548 546 return 1
549 547
550 548 chk_command = "cd %s; pwd" %remotefolder
551 549 mkdir_command = "mkdir %s" %remotefolder
552 550
553 551 if not self.__execute(chk_command):
554 552 if not self.__execute(mkdir_command):
555 553 self.remotefolder = None
556 554 return 0
557 555
558 556 self.remotefolder = remotefolder
559 557
560 558 return 1
561 559
562 560 def sendFile(self, fullfilename):
563 561
564 562 if not self.status:
565 563 return 0
566 564
567 565 try:
568 566 self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
569 567 except scp.ScpError, e:
570 568 print "[SSH Server]", str(e)
571 569 return 0
572 570
573 571 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
574 572 command = 'chmod 775 %s' %remotefile
575 573
576 574 return self.__execute(command)
577 575
578 576 class SendToServer(ProcessingUnit):
579 577
580 578 def __init__(self):
581 579
582 580 ProcessingUnit.__init__(self)
583 581
584 582 self.isConfig = False
585 self.clientObj = None
586
587
583 self.clientObj = None
588 584
589 585 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
590 586
591 587 self.clientObj = None
592 588 self.localfolder = localfolder
593 589 self.ext = ext
594 590 self.period = period
595 591
596 592 if str.lower(protocol) == 'ftp':
597 593 self.clientObj = FTPClient(server, username, password, remotefolder, period)
598 594
599 595 if str.lower(protocol) == 'ssh':
600 596 self.clientObj = SSHClient(server, username, password, remotefolder, period)
601 597
602 598 if not self.clientObj:
603 599 raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol
604 600
605 601 self.clientObj.start()
606 602
607 603 def findFiles(self):
608 604
609 605 if not type(self.localfolder) == list:
610 606 folderList = [self.localfolder]
611 607 else:
612 608 folderList = self.localfolder
613 609
614 610 #Remove duplicate items
615 611 folderList = list(set(folderList))
616 612
617 613 fullfilenameList = []
618 614
619 615 for thisFolder in folderList:
620 616
621 617 print "[Remote Server]: Searching files on %s" %thisFolder
622 618
623 619 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
624 620
625 621 if len(filenameList) < 1:
626 622 continue
627 623
628 624 for thisFile in filenameList:
629 625 fullfilename = os.path.join(thisFolder, thisFile)
630 626
631 627 if fullfilename in fullfilenameList:
632 628 continue
633 629
634 630 #Only files modified in the last 30 minutes are considered
635 631 if os.path.getmtime(fullfilename) < time.time() - 30*60:
636 632 continue
637 633
638 634 fullfilenameList.append(fullfilename)
639 635
640 636 return fullfilenameList
641 637
642 638 def run(self, **kwargs):
643 639 if not self.isConfig:
644 640 self.init = time.time()
645 641 self.setup(**kwargs)
646 642 self.isConfig = True
647 643
648 644 if not self.clientObj.is_alive():
649 645 print "[Remote Server]: Restarting connection "
650 646 self.setup(**kwargs)
651 647
652 648 if time.time() - self.init >= self.period:
653 649 fullfilenameList = self.findFiles()
654 650
655 651 if self.clientObj.updateFileList(fullfilenameList):
656 652 print "[Remote Server]: Sending the next files ", str(fullfilenameList)
657 653 self.init = time.time()
658 654
659 655 def close(self):
660 656 print "[Remote Server] Stopping thread"
661 657 self.clientObj.stop()
662 658
663 659
664 660 class FTP(object):
665 661 """
666 662 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
667 663
668 664 Non-standard Python modules used: None
669 665
670 666 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
671 667 """
672 668
673 669 def __init__(self,server = None, username=None, password=None, remotefolder=None):
674 670 """
675 671 This method is used to setting parameters for FTP and establishing connection to remote server
676 672
677 673 Inputs:
678 674 server - remote server IP Address
679 675
680 676 username - remote server Username
681 677
682 678 password - remote server password
683 679
684 680 remotefolder - remote server current working directory
685 681
686 682 Return: void
687 683
688 684 Affects:
689 685 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
690 686
691 687 self.folderList - sub-folder list of remote folder
692 688
693 689 self.fileList - file list of remote folder
694 690
695 691
696 692 """
697 693
698 694 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
699 695 server, username, password, remotefolder = self.parmsByDefault()
700 696
701 697 self.server = server
702 698 self.username = username
703 699 self.password = password
704 700 self.remotefolder = remotefolder
705 701 self.file = None
706 702 self.ftp = None
707 703 self.status = 0
708 704
709 705 try:
710 706 self.ftp = ftplib.FTP(self.server)
711 707 self.ftp.login(self.username,self.password)
712 708 self.ftp.cwd(self.remotefolder)
713 # print 'Connect to FTP Server: Successfully'
709 # print 'Connect to FTP Server: Successfully'
714 710
715 711 except ftplib.all_errors:
716 712 print 'Error FTP Service'
717 713 self.status = 1
718 714 return
719 715
720 716
721 717
722 718 self.dirList = []
723 719
724 720 try:
725 721 self.dirList = self.ftp.nlst()
726 722
727 723 except ftplib.error_perm, resp:
728 724 if str(resp) == "550 No files found":
729 725 print "no files in this directory"
730 726 self.status = 1
731 727 return
732 728
733 729 except ftplib.all_errors:
734 730 print 'Error Displaying Dir-Files'
735 731 self.status = 1
736 732 return
737 733
738 734 self.fileList = []
739 735 self.folderList = []
740 736 #only for test
741 737 for f in self.dirList:
742 738 name, ext = os.path.splitext(f)
743 739 if ext != '':
744 740 self.fileList.append(f)
745 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
741 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
746 742
747 743 def parmsByDefault(self):
748 744 server = 'jro-app.igp.gob.pe'
749 745 username = 'wmaster'
750 746 password = 'mst2010vhf'
751 747 remotefolder = '/home/wmaster/graficos'
752 748
753 749 return server, username, password, remotefolder
754 750
755 751
756 752 def mkd(self,dirname):
757 753 """
758 754 mkd is used to make directory in remote server
759 755
760 756 Input:
761 757 dirname - directory name
762 758
763 759 Return:
764 760 1 in error case else 0
765 761 """
766 762 try:
767 763 self.ftp.mkd(dirname)
768 764 except:
769 765 print 'Error creating remote folder:%s'%dirname
770 766 return 1
771 767
772 768 return 0
773 769
774 770
775 771 def delete(self,filename):
776 772 """
777 773 delete is used to delete file in current working directory of remote server
778 774
779 775 Input:
780 776 filename - filename to delete in remote folder
781 777
782 778 Return:
783 779 1 in error case else 0
784 780 """
785 781
786 782 try:
787 783 self.ftp.delete(filename)
788 784 except:
789 785 print 'Error deleting remote file:%s'%filename
790 786 return 1
791 787
792 788 return 0
793 789
794 790 def download(self,filename,localfolder):
795 791 """
796 792 download is used to downloading file from remote folder into local folder
797 793
798 794 Inputs:
799 795 filename - filename to donwload
800 796
801 797 localfolder - directory local to store filename
802 798
803 799 Returns:
804 800 self.status - 1 in error case else 0
805 801 """
806 802
807 803 self.status = 0
808 804
809 805
810 806 if not(filename in self.fileList):
811 807 print 'filename:%s not exists'%filename
812 808 self.status = 1
813 809 return self.status
814 810
815 811 newfilename = os.path.join(localfolder,filename)
816 812
817 813 self.file = open(newfilename, 'wb')
818 814
819 815 try:
820 816 print 'Download: ' + filename
821 817 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
822 818 print 'Download Complete'
823 819 except ftplib.all_errors:
824 820 print 'Error Downloading ' + filename
825 821 self.status = 1
826 822 return self.status
827 823
828 824 self.file.close()
829 825
830 826 return self.status
831 827
832 828
833 829 def __handleDownload(self,block):
834 830 """
835 831 __handleDownload is used to handle writing file
836 832 """
837 833 self.file.write(block)
838 834
839 835
840 836 def upload(self,filename,remotefolder=None):
841 837 """
842 838 upload is used to uploading local file to remote directory
843 839
844 840 Inputs:
845 841 filename - full path name of local file to store in remote directory
846 842
847 843 remotefolder - remote directory
848 844
849 845 Returns:
850 846 self.status - 1 in error case else 0
851 847 """
852 848
853 849 if remotefolder == None:
854 850 remotefolder = self.remotefolder
855 851
856 852 self.status = 0
857 853
858 854 try:
859 855 self.ftp.cwd(remotefolder)
860 856
861 857 self.file = open(filename, 'rb')
862 858
863 859 (head, tail) = os.path.split(filename)
864 860
865 861 command = "STOR " + tail
866 862
867 863 print 'Uploading: ' + tail
868 864 self.ftp.storbinary(command, self.file)
869 865 print 'Upload Completed'
870 866
871 867 except ftplib.all_errors:
872 868 print 'Error Uploading ' + tail
873 869 self.status = 1
874 870 return self.status
875 871
876 872 self.file.close()
877 873
878 874 #back to initial directory in __init__()
879 875 self.ftp.cwd(self.remotefolder)
880 876
881 877 return self.status
882 878
883 879
884 880 def dir(self,remotefolder):
885 881 """
886 882 dir is used to change working directory of remote server and get folder and file list
887 883
888 884 Input:
889 885 remotefolder - current working directory
890 886
891 887 Affects:
892 888 self.fileList - file list of working directory
893 889
894 890 Return:
895 891 infoList - list with filenames and size of file in bytes
896 892
897 893 self.folderList - folder list
898 894 """
899 895
900 896 self.remotefolder = remotefolder
901 897 print 'Change to ' + self.remotefolder
902 898 try:
903 899 self.ftp.cwd(remotefolder)
904 900 except ftplib.all_errors:
905 901 print 'Error Change to ' + self.remotefolder
906 902 infoList = None
907 903 self.folderList = None
908 904 return infoList,self.folderList
909 905
910 906 self.dirList = []
911 907
912 908 try:
913 909 self.dirList = self.ftp.nlst()
914 910
915 911 except ftplib.error_perm, resp:
916 912 if str(resp) == "550 No files found":
917 913 print "no files in this directory"
918 914 infoList = None
919 915 self.folderList = None
920 916 return infoList,self.folderList
921 917 except ftplib.all_errors:
922 918 print 'Error Displaying Dir-Files'
923 919 infoList = None
924 920 self.folderList = None
925 921 return infoList,self.folderList
926 922
927 923 infoList = []
928 924 self.fileList = []
929 925 self.folderList = []
930 926 for f in self.dirList:
931 927 name,ext = os.path.splitext(f)
932 928 if ext != '':
933 929 self.fileList.append(f)
934 930 value = (f,self.ftp.size(f))
935 931 infoList.append(value)
936 932
937 933 if ext == '':
938 934 self.folderList.append(f)
939 935
940 936 return infoList,self.folderList
941 937
942 938
943 939 def close(self):
944 940 """
945 941 close is used to close and end FTP connection
946 942
947 943 Inputs: None
948 944
949 945 Return: void
950 946
951 947 """
952 948 self.ftp.close()
953 949
954 950 class SendByFTP(Operation):
955 951
956 952 def __init__(self):
957 953
958 954 self.status = 1
959 955 self.counter = 0
960 956
961 957 def error_print(self, ValueError):
962 958
963 959 print ValueError, 'Error FTP'
964 960 print "don't worry the program is running..."
965 961
966 962 def worker_ftp(self, server, username, password, remotefolder, filenameList):
967 963
968 964 self.ftpClientObj = FTP(server, username, password, remotefolder)
969 965 for filename in filenameList:
970 966 self.ftpClientObj.upload(filename)
971 967 self.ftpClientObj.close()
972 968
973 969 def ftp_thread(self, server, username, password, remotefolder):
974 970 if not(self.status):
975 971 return
976 972
977 973 import multiprocessing
978 974
979 975 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
980 976 p.start()
981 977
982 978 p.join(3)
983 979
984 980 if p.is_alive():
985 981 p.terminate()
986 982 p.join()
987 983 print 'killing ftp process...'
988 984 self.status = 0
989 985 return
990 986
991 987 self.status = 1
992 988 return
993 989
994 990 def filterByExt(self, ext, localfolder):
995 991 fnameList = glob.glob1(localfolder,ext)
996 992 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
997 993
998 994 if len(self.filenameList) == 0:
999 995 self.status = 0
1000 996
1001 997 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
1002 998
1003 999 self.counter += 1
1004 1000 if self.counter >= period:
1005 1001 self.filterByExt(ext, localfolder)
1006 1002
1007 1003 self.ftp_thread(server, username, password, remotefolder)
1008 1004
1009 1005 self.counter = 0
1010 1006
1011 1007 self.status = 1
1012 1008
Show file before | Show file after | Hide comments
@@ -1,48 +1,59
1 1 '''
2 2 Created on Jul 16, 2014
3 3
4 4 @author: Miguel Urco
5 5 '''
6 import numpy
7 6 from setuptools import setup, Extension
7 from setuptools.command.build_ext import build_ext as _build_ext
8 8 from schainpy import __version__
9 9
10 class build_ext(_build_ext):
11 def finalize_options(self):
12 _build_ext.finalize_options(self)
13 # Prevent numpy from thinking it is still in its setup process:
14 __builtins__.__NUMPY_SETUP__ = False
15 import numpy
16 self.include_dirs.append(numpy.get_include())
17
18
10 19 setup(name="schainpy",
11 20 version=__version__,
12 21 description="Python tools to read, write and process Jicamarca data",
13 22 author="Miguel Urco",
14 23 author_email="miguel.urco@jro.igp.gob.pe",
15 24 url="http://jro.igp.gob.pe",
16 25 packages = {'schainpy',
17 26 'schainpy.model',
18 27 'schainpy.model.data',
19 28 'schainpy.model.graphics',
20 29 'schainpy.model.io',
21 30 'schainpy.model.proc',
22 31 'schainpy.model.serializer',
23 32 'schainpy.model.utils',
24 33 'schainpy.gui',
25 34 'schainpy.gui.figures',
26 35 'schainpy.gui.viewcontroller',
27 36 'schainpy.gui.viewer',
28 37 'schainpy.gui.viewer.windows'},
29 38 ext_package='schainpy',
30 39 py_modules=[''],
31 40 package_data={'': ['schain.conf.template'],
32 41 'schainpy.gui.figures': ['*.png','*.jpg'],
33 42 },
34 43 include_package_data=False,
35 44 scripts =['schainpy/gui/schainGUI',
36 45 'schainpy/scripts/schain'],
37 ext_modules=[Extension("cSchain", ["schainpy/model/proc/extensions.c"], include_dirs=[numpy.get_include()])],
46 ext_modules=[
47 Extension("cSchain", ["schainpy/model/proc/extensions.c"]
48 )],
49 cmdclass={'build_ext':build_ext},
50 setup_requires=["numpy >= 1.11.2"],
38 51 install_requires=[
39 52 "scipy >= 0.14.0",
40 53 "h5py >= 2.2.1",
41 54 "matplotlib >= 1.4.2",
42 55 "pyfits >= 3.4",
43 "numpy >= 1.11.2",
44 "paramiko >= 2.1.2",
45 56 "paho-mqtt >= 1.2",
46 57 "zmq",
47 58 ],
48 59 ) No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now