##// END OF EJS Templates
realtime_web
RSS
Reorganize models, smaller docker, others
jespinoza -
r2:ccdf3a6b9025
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,110
1 [
2 {
3 "code" : 100,
4 "name" : "HF JRO"
5 },
6 {
7 "code" : 101,
8 "name" : "HF Huancayo"
9 },
10 {
11 "code" : 102,
12 "name" : "HF Ica"
13 },
14 {
15 "code" : 110,
16 "name" : "ISR EW Drifts"
17 },
18 {
19 "code" : 111,
20 "name" : "ISR Faraday"
21 },
22 {
23 "code" : 112,
24 "name" : "ISR Imaging"
25 },
26 {
27 "code" : 120,
28 "name" : "Faraday"
29 },
30 {
31 "code" : 121,
32 "name" : "Faraday Long Pulse"
33 },
34 {
35 "code" : 122,
36 "name" : "Faraday Codded LP"
37 },
38 {
39 "code" : 123,
40 "name" : "Faraday Double Pulse"
41 },
42 {
43 "code" : 124,
44 "name" : "Faraday AC"
45 },
46 {
47 "code" : 125,
48 "name" : "Faraday Differential Phase"
49 },
50 {
51 "code" : 150,
52 "name" : "JASMET 50"
53 },
54 {
55 "code" : 151,
56 "name" : "JASMET 30"
57 },
58 {
59 "code" : 170,
60 "name" : "BLTR Huancayo"
61 },
62 {
63 "code" : 171,
64 "name" : "CIRI Huancayo"
65 },
66 {
67 "code" : 172,
68 "name" : "CLAIRE Huancayo"
69 },
70 {
71 "code" : 180,
72 "name" : "MST"
73 },
74 {
75 "code" : 181,
76 "name" : "ISR"
77 },
78 {
79 "code" : 182,
80 "name" : "EEJ"
81 },
82 {
83 "code" : 190,
84 "name" : "JULIA 150km (Natalia)"
85 },
86 {
87 "code" : 191,
88 "name" : "JULIA SpreadF (Natalia)"
89 },
90 {
91 "code" : 200,
92 "name" : "JULIA 150km"
93 },
94 {
95 "code" : 201,
96 "name" : "JULIA EEJ"
97 },
98 {
99 "code" : 202,
100 "name" : "JULIA SpreadF"
101 },
102 {
103 "code" : 203,
104 "name" : "JULIA Imaging"
105 },
106 {
107 "code" : 203,
108 "name" : "JULIA Bistatic"
109 }
110 ] No newline at end of file
Show file before | Show file after | Hide comments
@@ -6,7 +6,7
6 g++ \
6 g++ \
7 gcc \
7 gcc \
8 && pip install -r requirements.txt \
8 && pip install -r requirements.txt \
9 && apt-get purge -y --auto-remove gcc g++\
9 && apt-get purge -y --auto-remove gcc g++ \
10 && rm -rf /var/lib/apt/lists/*
10 && rm -rf /var/lib/apt/lists/*
11 COPY . /app/
11 COPY . /app/
12 No newline at end of file
12
Show file before | Show file after | Hide comments
@@ -22,7 +22,7 services:
22 container_name: 'zmq_server'
22 container_name: 'zmq_server'
23 image: 'realtime'
23 image: 'realtime'
24 ports:
24 ports:
25 - '127.0.0.1:4444:4444'
25 - '4444:4444'
26 command: 'python -u scripts/server.py'
26 command: 'python -u scripts/server.py'
27 env_file: .env
27 env_file: .env
28 links:
28 links:
Show file before | Show file after | Hide comments
@@ -1,12 +1,10
1 import os
1 import os
2 import json
2 import json
3 import numpy as np
3
4 from datetime import datetime
4 from datetime import datetime
5
5
6 from pymongo import MongoClient
6 from pymongo import MongoClient
7
7
8 from models import Experiment, Data
9
10 from channels.handler import AsgiHandler
8 from channels.handler import AsgiHandler
11 from channels.auth import channel_session_user
9 from channels.auth import channel_session_user
12 from channels import Group
10 from channels import Group
@@ -17,32 +15,35 DB = CLIENT['dbplots']
17
15
18 # Connected to websocket.connect
16 # Connected to websocket.connect
19 def ws_connect(message, code, plot):
17 def ws_connect(message, code, plot):
20 # Accept the incoming connection
21 message.reply_channel.send({'accept': True})
18 message.reply_channel.send({'accept': True})
22 # Add them to the chat group
19 pk = message.content['query_string'].split('=')[1]
23 Group('{}_{}'.format(code, plot)).add(message.reply_channel)
20 Group('{}_{}'.format(pk, plot)).add(message.reply_channel)
21 print('New connection from: {}, creating Group: {}_{}'.format(message.content['client'][0], pk, plot))
24
22
25 def ws_message(message, code, plot):
23 def ws_message(message, code, plot):
26 # Accept the incoming connection
24 # Accept the incoming connection
27
25 print 'incoming message'
28 dt = datetime.strptime(str(json.loads(message.content['text'])['date']), '%d/%m/%Y')
26 dt = datetime.strptime(str(json.loads(message.content['text'])['date']), '%d/%m/%Y')
29 e = DB.experiment.find_one({'date': dt})
27 exp = DB.exp_meta.find_one({'code': int(code), 'date': dt})
30 if e:
28 print exp
29 if exp:
31 if plot == 'spc':
30 if plot == 'spc':
32 datas = DB.data.find({'experiment': e['_id']}, ['time', 'data']).sort('time', -1).limit(1)[0]
31 datas = DB.exp_data.find({'expmeta': exp['_id']}, ['time', 'data']).sort('time', -1).limit(1)[0]
33 e['time'] = [datas['time']]
32 exp['time'] = [datas['time']]
34 e['spc'] = datas['data']['spc']
33 exp['spc'] = datas['data']['spc']
35 e['rti'] = datas['data']['rti']
34 exp['rti'] = datas['data']['rti']
36 e['noise'] = datas['data']['noise']
35 exp['noise'] = datas['data']['noise']
37 else:
36 else:
38 datas = DB.data.find({'experiment': e['_id']}, ['time', 'data']).sort('time', 1)
37 datas = DB.exp_data.find({'expmeta': exp['_id']}, ['time', 'data']).sort('time', 1)
38 print 'Counting...'
39 print datas.count()
39 dum = [(d['time'], d['data'][plot]) for d in datas]
40 dum = [(d['time'], d['data'][plot]) for d in datas]
40 e['time'] = [d[0] for d in dum]
41 exp['time'] = [d[0] for d in dum]
41 dum = [d[1] for d in dum]
42 dum = [d[1] for d in dum]
42 e[plot] = map(list, zip(*dum))
43 exp[plot] = map(list, zip(*dum))
43 e.pop('date', None)
44 exp.pop('date', None)
44 e.pop('_id', None)
45 exp.pop('_id', None)
45 message.reply_channel.send({'text': json.dumps(e)})
46 message.reply_channel.send({'text': json.dumps(exp)})
46 else:
47 else:
47 message.reply_channel.send({'text': json.dumps({'interval': 0})})
48 message.reply_channel.send({'text': json.dumps({'interval': 0})})
48
49
Show file before | Show file after | Hide comments
@@ -5,27 +5,29 from django.db import models
5 from mongoengine import *
5 from mongoengine import *
6
6
7 class Experiment(Document):
7 class Experiment(Document):
8 code = IntField(unique=True)
9 name = StringField(max_length=40)
10
11 class ExpMeta(Document):
8 code = IntField()
12 code = IntField()
9 name = StringField(max_length=40, default='Experiment')
10 date = DateTimeField()
13 date = DateTimeField()
11 pairs = ListField(default=list)
14 pairs = ListField(default=list)
12 yrange = ListField(FloatField())
15 yrange = ListField(FloatField())
13 xrange = ListField(FloatField())
16 xrange = ListField(FloatField())
14 interval = FloatField()
17 interval = FloatField()
18 plots = ListField(StringField())
15 localtime = BooleanField()
19 localtime = BooleanField()
16
20
17 meta = {
21 meta = {
18 'indexes': [[("code", 1), ("date", 1)]]
22 'indexes': [[("code", 1), ("date", 1)]]
19 }
23 }
20
24
21 class Data(Document):
25 class ExpData(Document):
22 experiment = LazyReferenceField(Experiment)
26 expmeta = LazyReferenceField(ExpMeta)
23 time = FloatField()
27 time = FloatField()
24 data = DictField()
28 data = DictField()
25
29
26 meta = {
30 meta = {
27 'indexes': ["experiment", "+time"]
31 'indexes': ["expmeta", "+time"]
28 }
32 }
29
33
30 # connect('dbplots')
31 # Experiment.drop_collection()
Show file before | Show file after | Hide comments
@@ -371,7 +371,7 class Scatter{
371 last = last+interval;
371 last = last+interval;
372 x.push(last);
372 x.push(last);
373 for (var j = 0; j < N; j++){
373 for (var j = 0; j < N; j++){
374 z[j].push(null);
374 y[j].push(null);
375 }
375 }
376 // Avoid infinite loop
376 // Avoid infinite loop
377 if (cnt==100){break;}
377 if (cnt==100){break;}
Show file before | Show file after | Hide comments
@@ -80,7 +80,9
80 document.location = '/' + $("#id_experiment").val() + '/' + $("#id_plot").val() + '/?date=' + $("#id_date").val();
80 document.location = '/' + $("#id_experiment").val() + '/' + $("#id_plot").val() + '/?date=' + $("#id_date").val();
81 });
81 });
82
82
83 var socket = new WebSocket('ws://' + window.location.host + '/{{code}}/{{plot}}/');
83 {% if date %}
84 $("#loader").css("display", "block");
85 var socket = new WebSocket('ws://' + window.location.host + '/{{code}}/{{plot}}/?' + 'pk={{id}}');
84 socket.onopen = function open() {
86 socket.onopen = function open() {
85 console.log('WebSockets connection created.');
87 console.log('WebSockets connection created.');
86 {% if date %}
88 {% if date %}
@@ -92,13 +94,14
92
94
93 socket.onmessage = function message(event) {
95 socket.onmessage = function message(event) {
94 var data = JSON.parse(event.data);
96 var data = JSON.parse(event.data);
97 console.log(data);
95 if (data.interval == 0) {
98 if (data.interval == 0) {
96 $("#loader").removeClass("loader").addClass("no-data");
99 $("#loader").removeClass("loader").addClass("no-data");
97 $("#loader").html("No data found");
100 $("#loader").html("No data found");
98 } else {
101 } else {
99 var first = plot(data);
102 var first = plot(data);
100 if (first == true) {
103 if (first == true) {
101 $("#loader").css("display", "none");
104 $("#loader").css("display", "none");
102 }
105 }
103 }
106 }
104 }
107 }
@@ -106,6 +109,9
106 if (socket.readyState == WebSocket.OPEN) {
109 if (socket.readyState == WebSocket.OPEN) {
107 socket.onopen();
110 socket.onopen();
108 }
111 }
112 {% else %}
113 $("#loader").css("display", "none");
114 {% endif %}
109 </script> {% block script %}{% endblock script %}
115 </script> {% block script %}{% endblock script %}
110 </body>
116 </body>
111
117
Show file before | Show file after | Hide comments
@@ -11,12 +11,12 from django.shortcuts import render
11
11
12 from bootstrap3_datetime.widgets import DateTimePicker
12 from bootstrap3_datetime.widgets import DateTimePicker
13
13
14 from pymongo import MongoClient
14 import mongoengine
15
15
16 host = os.environ.get('HOST_MONGO', 'localhost')
16 from plotter.models import Experiment, ExpMeta
17
17
18 CLIENT = MongoClient('{}:27017'.format(host))
18 host = os.environ.get('HOST_MONGO', 'localhost')
19 DB = CLIENT['dbplots']
19 mongoengine.connect('dbplots', host=host, port=27017)
20
20
21 # Forms
21 # Forms
22 class SearchForm(forms.Form):
22 class SearchForm(forms.Form):
@@ -47,27 +47,30 def main(request, code=None, plot=None):
47 if plot is not None:
47 if plot is not None:
48 initial['plot'] = plot
48 initial['plot'] = plot
49
49
50 print 'hola'
50 exps = [(q['code'], q['name']) for q in Experiment.objects.all()]
51 codes = DB.experiment.find().distinct('code')
51
52 print codes
53 exps = [DB.experiment.find_one({'code': c}, ['name']) for c in codes]
54 print exps
55 names = [q['name'] for q in exps]
56 print names
57 form = SearchForm(
52 form = SearchForm(
58 initial = initial,
53 initial = initial,
59 exp_choices = [(e[0], e[1]) for e in zip(codes, names)]
54 exp_choices = [(e[0], e[1]) for e in exps]
60 )
55 )
61
56
57 try:
58 exp = ExpMeta.objects.get(code=int(code), date=datetime.strptime(date, '%d/%m/%Y'))
59 exp_id = exp.id
60 except:
61 exp_id = 0
62
63
62 kwargs = {
64 kwargs = {
63 'code': code,
65 'code': code,
64 'plot': plot,
66 'plot': plot,
65 'date': date,
67 'date': date,
66 'form': form,
68 'form': form,
69 'id': exp_id
67 }
70 }
68
71
69 if code and codes:
72 if code and exps:
70 kwargs['title'] = [t[1] for t in zip(codes, names) if t[0]==int(code)][0]
73 kwargs['title'] = [t[1] for t in exps if t[0]==int(code)][0]
71 else:
74 else:
72 kwargs['title'] = 'JRO'
75 kwargs['title'] = 'JRO'
73
76
Show file before | Show file after | Hide comments
@@ -5,4 +5,4 channel_routing = [
5 route("websocket.connect", ws_connect, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
5 route("websocket.connect", ws_connect, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
6 route("websocket.receive", ws_message, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
6 route("websocket.receive", ws_message, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
7 route("websocket.disconnect", ws_disconnect, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
7 route("websocket.disconnect", ws_disconnect, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
8 ] No newline at end of file
8 ]
Show file before | Show file after | Hide comments
@@ -122,7 +122,7 USE_TZ = True
122
122
123 STATIC_URL = '/static/'
123 STATIC_URL = '/static/'
124
124
125 host = os.environ.get('HOST_REDIS', 'localhost')
125 host = os.environ.get('HOST_REDIS', '127.0.0.1')
126
126
127 CHANNEL_LAYERS = {
127 CHANNEL_LAYERS = {
128 "default": {
128 "default": {
Show file before | Show file after | Hide comments
@@ -4,7 +4,6 django-bootstrap3==9.1.0
4 django-bootstrap3-datetimepicker-2==2.5.0
4 django-bootstrap3-datetimepicker-2==2.5.0
5 channels==1.1.8
5 channels==1.1.8
6 mongoengine==0.15.0
6 mongoengine==0.15.0
7 numpy==1.13.3
8 pymongo==3.5.1
7 pymongo==3.5.1
9 pyzmq==16.0.3
8 pyzmq==16.0.3
10 redis==2.10.6
9 redis==2.10.6
Show file before | Show file after | Hide comments
@@ -1,5 +1,6
1 import os
1 import os
2 import sys
2 import sys
3 import json
3 import simplejson
4 import simplejson
4 from datetime import datetime
5 from datetime import datetime
5 import zmq
6 import zmq
@@ -10,8 +11,7 import mongoengine
10 sys.path.append('/app')
11 sys.path.append('/app')
11 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "realtime.settings")
12 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "realtime.settings")
12
13
13 from plotter.models import Experiment, Data
14 from plotter.models import Experiment, ExpMeta, ExpData
14
15
15
16 host_mongo = os.environ.get('HOST_MONGO', 'localhost')
16 host_mongo = os.environ.get('HOST_MONGO', 'localhost')
17 mongoengine.connect('dbplots', host=host_mongo, port=27017)
17 mongoengine.connect('dbplots', host=host_mongo, port=27017)
@@ -26,69 +26,59 receiver = context.socket(zmq.SUB)
26 receiver.bind("tcp://0.0.0.0:4444")
26 receiver.bind("tcp://0.0.0.0:4444")
27 receiver.setsockopt(zmq.SUBSCRIBE, '')
27 receiver.setsockopt(zmq.SUBSCRIBE, '')
28
28
29
29 def loaddata():
30
30 print('Loading Experiments...')
31 def update_db(buffer):
31 for tup in json.load(open('scripts/experiments.json')):
32 dt = datetime.utcfromtimestamp(buffer['time'])
32 print(tup['name'])
33 exp = Experiment.objects(code=buffer['exp_code'], date=dt.date()).first()
33 exp = Experiment.objects(code=tup['code']).modify(
34 if exp is None:
34 upsert=True,
35 exp = Experiment(
35 new=True,
36 code=buffer['exp_code'],
36 set__code=tup['code'],
37 date=dt.date(),
37 set__name=tup['name'],
38 yrange = buffer['yrange'],
39 xrange = buffer['xrange'],
40 interval = buffer['interval'],
41 localtime = buffer['localtime'],
42 name = buffer['name'],
43 )
38 )
44 exp.save()
39 exp.save()
40
41 def update(buffer):
42 dt = datetime.utcfromtimestamp(buffer['time'])
43 exp = ExpMeta.objects(code=buffer['exp_code'], date=dt.date()).modify(
44 upsert=True,
45 new=True,
46 set__code=buffer['exp_code'],
47 set__date=dt.date(),
48 set__yrange = buffer['yrange'],
49 set__xrange = buffer['xrange'],
50 set__interval = buffer['interval'],
51 set__localtime = buffer['localtime'],
52 set__plots = buffer['data'].keys()
53 )
54 exp.save()
45
55
46 data = Data.objects(experiment=exp, time=buffer['time']).first()
56 data = ExpData.objects(expmeta=exp, time=buffer['time']).modify(
57 upsert=True,
58 new=True,
59 set__expmeta = exp,
60 set__time = buffer['time'],
61 set__data = buffer['data']
62 )
47
63
48 if data is None:
64 data.save()
49 data = Data(
50 experiment = exp,
51 time = buffer['time'],
52 data = buffer['data']
53 ).save()
54 new = True
55 else:
56 data.data = buffer['data']
57 data.save()
58 new = False
59
65
60 return new
66 return exp.id
61 print 'Starting...'
62 while True:
63 buffer = receiver.recv_json()
64 if 'xrange' not in buffer:
65 buffer['xrange'] = []
66 if 'name' not in buffer:
67 buffer['name'] = 'Experiment'
68 if update_db(buffer):
69 dum = buffer.copy()
70 dum['time'] = [buffer['time']]
71 dum['rti'] = buffer['data']['rti']
72 # dum['noise'] = buffer['data']['noise']
73 dum.pop('data')
74 code = dum.pop('exp_code')
75 channel.send_group(u'{}_rti'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)})
76 print 'Sending...{} - {} bytes'.format('rti', len(str(dum)))
77
67
78 # dum = buffer.copy()
68 def main():
79 # dum['time'] = [buffer['time']]
69 print('Starting ZMQ server...')
80 # dum['rti'] = buffer['data']['rti']
70 while True:
81 # dum['spc'] = buffer['data']['spc']
71 buffer = receiver.recv_json()
82 # dum['noise'] = buffer['data']['noise']
72 code = update(buffer)
83 # dum.pop('data')
73 for plot in buffer['data']:
84 # code = dum.pop('exp_code')
74 dum = buffer.copy()
85 # channel.send_group(u'{}_spc'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)})
75 dum['time'] = [buffer['time']]
86 # print 'Sending...{} - {} bytes'.format('spc', len(str(dum)))
76 dum[plot] = buffer['data'][plot]
77 dum.pop('data')
78 dum.pop('exp_code')
79 channel.send_group(u'{}_{}'.format(code, plot), {'text': simplejson.dumps(dum, ignore_nan=True)})
80 print('Sending...{} - {} bytes'.format(plot, len(str(dum))))
87
81
88 # dum = buffer.copy()
82 if __name__=='__main__':
89 # dum['time'] = [buffer['time']]
83 loaddata()
90 # dum['noise'] = [[x] for x in buffer['data']['noise']]
84 main() No newline at end of file
91 # dum.pop('data')
92 # code = dum.pop('exp_code')
93 # channel.send_group(u'{}_noise'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)})
94 # print 'Sending...{} - {} bytes'.format('noise', len(str(dum)))
General Comments 0
You need to be logged in to leave comments. Login now