##// END OF EJS Templates
realtime_web
RSS
new fix
Armandex123 -
r84:a831c0e60bea
parent child
Show More
Show file before | Show file after | Hide comments
1 NO CONTENT: new file 100644, binary diff hidden
NO CONTENT: new file 100644, binary diff hidden
Show file before | Show file after | Hide comments
1 NO CONTENT: new file 100644, binary diff hidden
NO CONTENT: new file 100644, binary diff hidden
Show file before | Show file after | Hide comments
@@ -1,121 +1,127
1 version: '3'
1 version: '3'
2
2
3 services:
3 services:
4
4
5 ofelia-cron:
5 ofelia-cron:
6 container_name: 'ofelia-cron'
6 container_name: 'ofelia-cron'
7 image: mcuadros/ofelia:v0.3.6
7 image: mcuadros/ofelia:v0.3.6
8 command: daemon --docker
8 command: daemon --docker
9 depends_on:
9 depends_on:
10 - web
10 - web
11 volumes:
11 volumes:
12 - /var/run/docker.sock:/var/run/docker.sock:ro
12 - /var/run/docker.sock:/var/run/docker.sock:ro
13 logging:
13 logging:
14 driver: "json-file"
14 driver: "json-file"
15 options:
15 options:
16 max-size: "12m"
16 max-size: "12m"
17
17
18
18 # web-muf:
19 # container_name: 'web-muf'
20 # build: ./volumes/app/
21 # volumes:
22 # - './volumes/app:${APP_DIR}'
23 # - './volumes/app/others/data_pgw:/app/others/data_pgw'
24 # command: python -u /app/others/data_pgw/refresh_pgw.py
19
25
20 web:
26 web:
21 container_name: 'realtime'
27 container_name: 'realtime'
22 build: ./volumes/app/
28 build: ./volumes/app/
23 restart: always
29 restart: always
24 image: realtime
30 image: realtime
25 command: sh -c "daphne -b 0.0.0.0 -p 8080 realtime.asgi:application && python ./others/data_pgw/refresh_pgw.py"
31 command: daphne -b 0.0.0.0 -p 8080 realtime.asgi:application
26 env_file: .env
32 env_file: .env
27 links:
33 links:
28 - redis
34 - redis
29 - mongo
35 - mongo
30 volumes:
36 volumes:
31 - './volumes/app:${APP_DIR}'
37 - './volumes/app:${APP_DIR}'
32 - './volumes/app/others/data_pgw:/app/others/data_pgw'
38 - './volumes/app/others/data_pgw:/app/others/data_pgw'
33 depends_on:
39 depends_on:
34 - redis
40 - redis
35 - mongo
41 - mongo
36 logging:
42 logging:
37 options:
43 options:
38 max-size: 10m
44 max-size: 10m
39 max-file: "3"
45 max-file: "3"
40
46
41 labels:
47 labels:
42 ofelia.enabled: "true"
48 ofelia.enabled: "true"
43 ofelia.job-exec.refresh_pgw.command: "python /app/others/data_pgw/refresh_pgw.py"
49 ofelia.job-exec.refresh_pgw.command: "python /app/others/data_pgw/refresh_pgw.py"
44 ofelia.job-exec.refresh_pgw.schedule: "@every 60m"
50 ofelia.job-exec.refresh_pgw.schedule: "@every 30m"
45 ofelia.job-exec.refresh_pgw.no-overlap: "true"
51 ofelia.job-exec.refresh_pgw.no-overlap: "true"
46
52
47
53
48
54
49
55
50
56
51
57
52
58
53 zmq_server:
59 zmq_server:
54 container_name: 'realtime_zmq'
60 container_name: 'realtime_zmq'
55 restart: always
61 restart: always
56 image: 'realtime'
62 image: 'realtime'
57 ports:
63 ports:
58 - '${ZMQ_PORT}:4444'
64 - '${ZMQ_PORT}:4444'
59 command: 'python -u scripts/server.py'
65 command: 'python -u scripts/server.py'
60 env_file: .env
66 env_file: .env
61 links:
67 links:
62 - redis
68 - redis
63 - mongo
69 - mongo
64 volumes:
70 volumes:
65 - './volumes/app:${APP_DIR}'
71 - './volumes/app:${APP_DIR}'
66 depends_on:
72 depends_on:
67 - web
73 - web
68 logging:
74 logging:
69 options:
75 options:
70 max-size: 10m
76 max-size: 10m
71 max-file: "3"
77 max-file: "3"
72
78
73 time_server:
79 time_server:
74 container_name: 'realtime_time'
80 container_name: 'realtime_time'
75 restart: always
81 restart: always
76 image: 'realtime'
82 image: 'realtime'
77 command: 'python -u scripts/update_times.py'
83 command: 'python -u scripts/update_times.py'
78 env_file: .env
84 env_file: .env
79 links:
85 links:
80 - redis
86 - redis
81 - mongo
87 - mongo
82 volumes:
88 volumes:
83 - './volumes/app:${APP_DIR}'
89 - './volumes/app:${APP_DIR}'
84 depends_on:
90 depends_on:
85 - web
91 - web
86 logging:
92 logging:
87 options:
93 options:
88 max-size: 10m
94 max-size: 10m
89 max-file: "3"
95 max-file: "3"
90
96
91 redis:
97 redis:
92 container_name: 'realtime_redis'
98 container_name: 'realtime_redis'
93 restart: always
99 restart: always
94 image: 'redis'
100 image: 'redis'
95 volumes:
101 volumes:
96 - '${REDIS_DATA}:/data'
102 - '${REDIS_DATA}:/data'
97
103
98 mongo:
104 mongo:
99 container_name: 'realtime_mongo'
105 container_name: 'realtime_mongo'
100 restart: always
106 restart: always
101 image: 'mongo:4.0'
107 image: 'mongo:4.0'
102 command: '--storageEngine wiredTiger'
108 command: '--storageEngine wiredTiger'
103 volumes:
109 volumes:
104 - '${MONGO_DATA}:/data/db'
110 - '${MONGO_DATA}:/data/db'
105
111
106 nginx:
112 nginx:
107 container_name: 'realtime_nginx'
113 container_name: 'realtime_nginx'
108 restart: always
114 restart: always
109 image: nginx:1.15-alpine
115 image: nginx:1.15-alpine
110 volumes:
116 volumes:
111 - ./volumes/app/static_files:/static
117 - ./volumes/app/static_files:/static
112 - ./nginx:/etc/nginx/conf.d
118 - ./nginx:/etc/nginx/conf.d
113 ports:
119 ports:
114 - "${HTTP_PORT}:8000"
120 - "${HTTP_PORT}:8000"
115 depends_on:
121 depends_on:
116 - web
122 - web
117 command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
123 command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
118
124
119 #volumes:
125 #volumes:
120 # mongo:
126 # mongo:
121 # redis:
127 # redis:
Show file before | Show file after | Hide comments
@@ -1,226 +1,226
1 import os
1 import os
2 import pytz
2 import pytz
3 import shutil
3 import shutil
4 import urllib
4 import urllib
5 import contextlib
5 import contextlib
6 import collections
6 import collections
7 import urllib.request
7 import urllib.request
8
8
9
9
10 from keys import *
10 from keys import *
11 from datetime import datetime
11 from datetime import datetime
12
12
13
13
14
14
15 class downloader(object):
15 class downloader(object):
16
16
17 def __init__(self,**kwargs):
17 def __init__(self,**kwargs):
18
18
19 print("ruta actual",os.path.abspath(__file__))
19 print("ruta actual",os.path.abspath(__file__))
20
20
21 self.verbose = kwargs.get("verbose",False)
21 self.verbose = kwargs.get("verbose",True)
22
22
23
23
24 self.__download_kp(2020)
24 self.__download_kp(2020)
25
25
26 self.__download_ae(2020)
26 self.__download_ae(2020)
27
27
28 #change this line when dst and kp data from peru stations is ready
28 #change this line when dst and kp data from peru stations is ready
29 self.__download_dst(2020)
29 self.__download_dst(2020)
30
30
31
31
32
32
33 def __download_kp(self,year0,year1=None):
33 def __download_kp(self,year0,year1=None):
34
34
35
35
36 if year1 is None:
36 if year1 is None:
37 year1 = datetime.now(pytz.utc).year
37 year1 = datetime.now(pytz.utc).year
38
38
39 for year in range(year0,year1+1):
39 for year in range(year0,year1+1):
40
40
41 src="ftp://ftp.gfz-potsdam.de/pub/home/obs/Kp_ap_Ap_SN_F107/"\
41 src="ftp://ftp.gfz-potsdam.de/pub/home/obs/Kp_ap_Ap_SN_F107/"\
42 + "Kp_ap_Ap_SN_F107_%4i.txt" %(year)
42 + "Kp_ap_Ap_SN_F107_%4i.txt" %(year)
43
43
44
44
45 name_file = "%4i" % (year)
45 name_file = "%4i" % (year)
46
46
47
47
48
48
49 path_des = os.path.join(PYGLOW_FILES,'kp',name_file)
49 path_des = os.path.join(PYGLOW_FILES,'kp',name_file)
50
50
51 if not os.path.isdir(os.path.dirname(path_des)):
51 if not os.path.isdir(os.path.dirname(path_des)):
52
52
53 os.makedirs(os.path.dirname(path_des))
53 os.makedirs(os.path.dirname(path_des))
54
54
55 listname = os.listdir(os.path.join(PYGLOW_FILES,'kp'))
55 listname = os.listdir(os.path.join(PYGLOW_FILES,'kp'))
56
56
57 if (not(name_file in listname)) or (year1==datetime.now(pytz.utc).year):
57 if (not(name_file in listname)) or (year1==datetime.now(pytz.utc).year):
58 try:
58 try:
59 with contextlib.closing(urllib.request.urlopen(src)) as r:
59 with contextlib.closing(urllib.request.urlopen(src)) as r:
60 with open(path_des, 'wb') as f:
60 with open(path_des, 'wb') as f:
61 shutil.copyfileobj(r, f)
61 shutil.copyfileobj(r, f)
62
62
63
63
64 except IOError as e:
64 except:
65
65
66 print("Failed downloading data for year {}."
66 print("Failed downloading data for year {}."
67 "File does not exist ({})".format(
67 "File does not exist ({})".format(
68 year,
68 year,
69 str(e),
69 str(e),
70 ),
70 ),
71 )
71 )
72
72
73 def __download_ae(self,year0,year1=None):
73 def __download_ae(self,year0,year1=None):
74
74
75 pyglow_dir = os.path.join(PYGLOW_FILES,'ae')
75 pyglow_dir = os.path.join(PYGLOW_FILES,'ae')
76
76
77 if not os.path.isdir(pyglow_dir):
77 if not os.path.isdir(pyglow_dir):
78 os.makedirs(pyglow_dir)
78 os.makedirs(pyglow_dir)
79
79
80
80
81 if year1 is None:
81 if year1 is None:
82 year1 = datetime.now(pytz.utc).year
82 year1 = datetime.now(pytz.utc).year
83
83
84 for year in range(year0,year1+1):
84 for year in range(year0,year1+1):
85 for month in range(1,13):
85 for month in range(1,13):
86 filename = '{:4d}{:02d}'.format(year,month)
86 filename = '{:4d}{:02d}'.format(year,month)
87
87
88 des = os.path.join(pyglow_dir, filename)
88 des = os.path.join(pyglow_dir, filename)
89
89
90 self.__process_ae(year,month,des)
90 self.__process_ae(year,month,des)
91
91
92
92
93 def __process_ae(self,year, month, des):
93 def __process_ae(self,year, month, des):
94 '''
94 '''
95 Helper function to earch for the appropriate location
95 Helper function to earch for the appropriate location
96 and download the AE index file from WDC Kyoto for the
96 and download the AE index file from WDC Kyoto for the
97 given month and year. Save it to the specified file "des".
97 given month and year. Save it to the specified file "des".
98 Return True if successful, False if not.
98 Return True if successful, False if not.
99 '''
99 '''
100 # There are three possible sources of data. Search for
100 # There are three possible sources of data. Search for
101 # them in the following order:
101 # them in the following order:
102 # 1) Final
102 # 1) Final
103 # 2) Provisional
103 # 2) Provisional
104 # 3) Realtime
104 # 3) Realtime
105 year_month = '%i%02i' % (year, month)
105 year_month = '%i%02i' % (year, month)
106 wgdc_fn = 'ae%s%02i.for.request' % (str(year)[2:], month)
106 wgdc_fn = 'ae%s%02i.for.request' % (str(year)[2:], month)
107 src_provisional = \
107 src_provisional = \
108 'http://wdc.kugi.kyoto-u.ac.jp/ae_provisional/%s/%s' % \
108 'http://wdc.kugi.kyoto-u.ac.jp/ae_provisional/%s/%s' % \
109 (year_month, wgdc_fn)
109 (year_month, wgdc_fn)
110 src_realtime = 'http://wdc.kugi.kyoto-u.ac.jp/ae_realtime/%s/%s' % \
110 src_realtime = 'http://wdc.kugi.kyoto-u.ac.jp/ae_realtime/%s/%s' % \
111 (year_month, wgdc_fn)
111 (year_month, wgdc_fn)
112
112
113 success = False
113 success = False
114
114
115 for src in [src_provisional, src_realtime]:
115 for src in [src_provisional, src_realtime]:
116 try:
116 try:
117 with contextlib.closing(urllib.request.urlopen(src)) as r:
117 with contextlib.closing(urllib.request.urlopen(src)) as r:
118 contents = r.readlines()
118 contents = r.readlines()
119 # If that succeeded, then the file exists
119 # If that succeeded, then the file exists
120 if self.verbose:
120 if self.verbose:
121 print(
121 print(
122 "\nDownloading\n{src}\nto\n{des}".format(
122 "\nDownloading\n{src}\nto\n{des}".format(
123 src=src,
123 src=src,
124 des=des,
124 des=des,
125 )
125 )
126 )
126 )
127
127
128 with open(des, 'w') as f:
128 with open(des, 'w') as f:
129 # this shrinks the filesize to hourly
129 # this shrinks the filesize to hourly
130 for c in contents:
130 for c in contents:
131 c = c.decode('utf8')
131 c = c.decode('utf8')
132 f.write(
132 f.write(
133 "%s%s%s\n" % (c[12:18], c[19:21], c[394:400])
133 "%s%s%s\n" % (c[12:18], c[19:21], c[394:400])
134 )
134 )
135 success = True
135 success = True
136 break
136 break
137 except Exception as e:
137 except:
138 print("No se pudo descargar archivo ae del enlace de descarga {}".format(src))
138 print("No se pudo descargar archivo ae del enlace de descarga {}".format(src))
139
139
140 return success
140 return success
141
141
142 def __download_dst(self,year0,year1=None):
142 def __download_dst(self,year0,year1=None):
143 """
143 """
144 Update the Dst index files used in pyglow.
144 Update the Dst index files used in pyglow.
145 The files will be downloaded from WDC Kyoto
145 The files will be downloaded from WDC Kyoto
146 to your pyglow installation directory.
146 to your pyglow installation directory.
147
147
148 _update_dst(years=None)
148 _update_dst(years=None)
149
149
150 :param years: (optional) a list of years to download.
150 :param years: (optional) a list of years to download.
151 If this input is not provided, the full
151 If this input is not provided, the full
152 range of years starting from 2005 to the
152 range of years starting from 2005 to the
153 current year will be downloaded. Pre-2005
153 current year will be downloaded. Pre-2005
154 files are shipped with pyglow.
154 files are shipped with pyglow.
155 """
155 """
156
156
157 pyglow_dir = os.path.join(PYGLOW_FILES,'dst')
157 pyglow_dir = os.path.join(PYGLOW_FILES,'dst')
158
158
159 if year1 is None:
159 if year1 is None:
160 year1 = datetime.now(pytz.utc).year
160 year1 = datetime.now(pytz.utc).year
161
161
162 if not os.path.isdir(pyglow_dir):
162 if not os.path.isdir(pyglow_dir):
163 os.makedirs(pyglow_dir)
163 os.makedirs(pyglow_dir)
164
164
165 for year in range(year0,year1+1):
165 for year in range(year0,year1+1):
166
166
167 for month in range(1,13):
167 for month in range(1,13):
168 filename = '{:4d}{:02d}'.format(year,month)
168 filename = '{:4d}{:02d}'.format(year,month)
169
169
170 des = os.path.join(pyglow_dir, filename)
170 des = os.path.join(pyglow_dir, filename)
171
171
172 self.__process_dst(year, month, des)
172 self.__process_dst(year, month, des)
173
173
174 return
174 return
175
175
176 def __process_dst(self,year, month, des):
176 def __process_dst(self,year, month, des):
177
177
178
178
179 """
179 """
180 Helper function to earch for the appropriate location
180 Helper function to earch for the appropriate location
181 and download the DST index file from WDC Kyoto for the
181 and download the DST index file from WDC Kyoto for the
182 given month and year. Save it to the specified file "des".
182 given month and year. Save it to the specified file "des".
183 Return True if successful, False if not.
183 Return True if successful, False if not.
184 """
184 """
185 # There are three possible sources of data. Search for
185 # There are three possible sources of data. Search for
186 # them in the following order:
186 # them in the following order:
187 # 1) Final
187 # 1) Final
188 # 2) Provisional
188 # 2) Provisional
189 # 3) Realtime
189 # 3) Realtime
190
190
191 success = False
191 success = False
192
192
193
193
194
194
195 year_month = '%i%02i' % (year, month)
195 year_month = '%i%02i' % (year, month)
196 wgdc_fn = 'dst%s%02i.for.request' % (str(year)[2:], month)
196 wgdc_fn = 'dst%s%02i.for.request' % (str(year)[2:], month)
197 src_final = 'http://wdc.kugi.kyoto-u.ac.jp/dst_final/%s/%s' % \
197 src_final = 'http://wdc.kugi.kyoto-u.ac.jp/dst_final/%s/%s' % \
198 (year_month, wgdc_fn)
198 (year_month, wgdc_fn)
199 src_provisional = \
199 src_provisional = \
200 'http://wdc.kugi.kyoto-u.ac.jp/dst_provisional/%s/%s' % \
200 'http://wdc.kugi.kyoto-u.ac.jp/dst_provisional/%s/%s' % \
201 (year_month, wgdc_fn)
201 (year_month, wgdc_fn)
202 src_realtime = 'http://wdc.kugi.kyoto-u.ac.jp/dst_realtime/%s/%s' % \
202 src_realtime = 'http://wdc.kugi.kyoto-u.ac.jp/dst_realtime/%s/%s' % \
203 (year_month, wgdc_fn)
203 (year_month, wgdc_fn)
204
204
205
205
206 for src in [src_final, src_provisional, src_realtime]:
206 for src in [src_final, src_provisional, src_realtime]:
207 try:
207 try:
208 with contextlib.closing(urllib.request.urlopen(src)) as r:
208 with contextlib.closing(urllib.request.urlopen(src)) as r:
209 contents = r.read().decode('utf8')
209 contents = r.read().decode('utf8')
210 # If that succeeded, then the file exists
210 # If that succeeded, then the file exists
211 print(
211 print(
212 "\nDownloading\n{src}\nto\n{des}".format(
212 "\nDownloading\n{src}\nto\n{des}".format(
213 src=src,
213 src=src,
214 des=des
214 des=des
215 )
215 )
216 )
216 )
217 with open(des, 'w') as f:
217 with open(des, 'w') as f:
218 f.write(contents)
218 f.write(contents)
219 success = True
219 success = True
220 break
220 break
221 except urllib.error.HTTPError:
221 except:
222 pass
222 pass
223
223
224 return success
224 return success
225
225
226
226
General Comments 0
You need to be logged in to leave comments. Login now