##// END OF EJS Templates
realtime_web
RSS
Reorganize models, smaller docker, others
jespinoza -
r2:ccdf3a6b9025
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,110
1 [
2 {
3 "code" : 100,
4 "name" : "HF JRO"
5 },
6 {
7 "code" : 101,
8 "name" : "HF Huancayo"
9 },
10 {
11 "code" : 102,
12 "name" : "HF Ica"
13 },
14 {
15 "code" : 110,
16 "name" : "ISR EW Drifts"
17 },
18 {
19 "code" : 111,
20 "name" : "ISR Faraday"
21 },
22 {
23 "code" : 112,
24 "name" : "ISR Imaging"
25 },
26 {
27 "code" : 120,
28 "name" : "Faraday"
29 },
30 {
31 "code" : 121,
32 "name" : "Faraday Long Pulse"
33 },
34 {
35 "code" : 122,
36 "name" : "Faraday Codded LP"
37 },
38 {
39 "code" : 123,
40 "name" : "Faraday Double Pulse"
41 },
42 {
43 "code" : 124,
44 "name" : "Faraday AC"
45 },
46 {
47 "code" : 125,
48 "name" : "Faraday Differential Phase"
49 },
50 {
51 "code" : 150,
52 "name" : "JASMET 50"
53 },
54 {
55 "code" : 151,
56 "name" : "JASMET 30"
57 },
58 {
59 "code" : 170,
60 "name" : "BLTR Huancayo"
61 },
62 {
63 "code" : 171,
64 "name" : "CIRI Huancayo"
65 },
66 {
67 "code" : 172,
68 "name" : "CLAIRE Huancayo"
69 },
70 {
71 "code" : 180,
72 "name" : "MST"
73 },
74 {
75 "code" : 181,
76 "name" : "ISR"
77 },
78 {
79 "code" : 182,
80 "name" : "EEJ"
81 },
82 {
83 "code" : 190,
84 "name" : "JULIA 150km (Natalia)"
85 },
86 {
87 "code" : 191,
88 "name" : "JULIA SpreadF (Natalia)"
89 },
90 {
91 "code" : 200,
92 "name" : "JULIA 150km"
93 },
94 {
95 "code" : 201,
96 "name" : "JULIA EEJ"
97 },
98 {
99 "code" : 202,
100 "name" : "JULIA SpreadF"
101 },
102 {
103 "code" : 203,
104 "name" : "JULIA Imaging"
105 },
106 {
107 "code" : 203,
108 "name" : "JULIA Bistatic"
109 }
110 ] No newline at end of file
Show file before | Show file after | Hide comments
@@ -6,7 +6,7
6 6 g++ \
7 7 gcc \
8 8 && pip install -r requirements.txt \
9 && apt-get purge -y --auto-remove gcc g++\
9 && apt-get purge -y --auto-remove gcc g++ \
10 10 && rm -rf /var/lib/apt/lists/*
11 11 COPY . /app/
12 12 No newline at end of file
Show file before | Show file after | Hide comments
@@ -22,7 +22,7 services:
22 22 container_name: 'zmq_server'
23 23 image: 'realtime'
24 24 ports:
25 - '127.0.0.1:4444:4444'
25 - '4444:4444'
26 26 command: 'python -u scripts/server.py'
27 27 env_file: .env
28 28 links:
Show file before | Show file after | Hide comments
@@ -1,12 +1,10
1 1 import os
2 2 import json
3 import numpy as np
3
4 4 from datetime import datetime
5 5
6 6 from pymongo import MongoClient
7 7
8 from models import Experiment, Data
9
10 8 from channels.handler import AsgiHandler
11 9 from channels.auth import channel_session_user
12 10 from channels import Group
@@ -17,32 +15,35 DB = CLIENT['dbplots']
17 15
18 16 # Connected to websocket.connect
19 17 def ws_connect(message, code, plot):
20 # Accept the incoming connection
21 18 message.reply_channel.send({'accept': True})
22 # Add them to the chat group
23 Group('{}_{}'.format(code, plot)).add(message.reply_channel)
19 pk = message.content['query_string'].split('=')[1]
20 Group('{}_{}'.format(pk, plot)).add(message.reply_channel)
21 print('New connection from: {}, creating Group: {}_{}'.format(message.content['client'][0], pk, plot))
24 22
25 23 def ws_message(message, code, plot):
26 24 # Accept the incoming connection
27
25 print 'incoming message'
28 26 dt = datetime.strptime(str(json.loads(message.content['text'])['date']), '%d/%m/%Y')
29 e = DB.experiment.find_one({'date': dt})
30 if e:
27 exp = DB.exp_meta.find_one({'code': int(code), 'date': dt})
28 print exp
29 if exp:
31 30 if plot == 'spc':
32 datas = DB.data.find({'experiment': e['_id']}, ['time', 'data']).sort('time', -1).limit(1)[0]
33 e['time'] = [datas['time']]
34 e['spc'] = datas['data']['spc']
35 e['rti'] = datas['data']['rti']
36 e['noise'] = datas['data']['noise']
31 datas = DB.exp_data.find({'expmeta': exp['_id']}, ['time', 'data']).sort('time', -1).limit(1)[0]
32 exp['time'] = [datas['time']]
33 exp['spc'] = datas['data']['spc']
34 exp['rti'] = datas['data']['rti']
35 exp['noise'] = datas['data']['noise']
37 36 else:
38 datas = DB.data.find({'experiment': e['_id']}, ['time', 'data']).sort('time', 1)
37 datas = DB.exp_data.find({'expmeta': exp['_id']}, ['time', 'data']).sort('time', 1)
38 print 'Counting...'
39 print datas.count()
39 40 dum = [(d['time'], d['data'][plot]) for d in datas]
40 e['time'] = [d[0] for d in dum]
41 exp['time'] = [d[0] for d in dum]
41 42 dum = [d[1] for d in dum]
42 e[plot] = map(list, zip(*dum))
43 e.pop('date', None)
44 e.pop('_id', None)
45 message.reply_channel.send({'text': json.dumps(e)})
43 exp[plot] = map(list, zip(*dum))
44 exp.pop('date', None)
45 exp.pop('_id', None)
46 message.reply_channel.send({'text': json.dumps(exp)})
46 47 else:
47 48 message.reply_channel.send({'text': json.dumps({'interval': 0})})
48 49
Show file before | Show file after | Hide comments
@@ -5,27 +5,29 from django.db import models
5 5 from mongoengine import *
6 6
7 7 class Experiment(Document):
8 code = IntField(unique=True)
9 name = StringField(max_length=40)
10
11 class ExpMeta(Document):
8 12 code = IntField()
9 name = StringField(max_length=40, default='Experiment')
10 13 date = DateTimeField()
11 14 pairs = ListField(default=list)
12 15 yrange = ListField(FloatField())
13 16 xrange = ListField(FloatField())
14 17 interval = FloatField()
18 plots = ListField(StringField())
15 19 localtime = BooleanField()
16 20
17 21 meta = {
18 22 'indexes': [[("code", 1), ("date", 1)]]
19 23 }
20 24
21 class Data(Document):
22 experiment = LazyReferenceField(Experiment)
25 class ExpData(Document):
26 expmeta = LazyReferenceField(ExpMeta)
23 27 time = FloatField()
24 28 data = DictField()
25 29
26 30 meta = {
27 'indexes': ["experiment", "+time"]
31 'indexes': ["expmeta", "+time"]
28 32 }
29 33
30 # connect('dbplots')
31 # Experiment.drop_collection()
Show file before | Show file after | Hide comments
@@ -371,7 +371,7 class Scatter{
371 371 last = last+interval;
372 372 x.push(last);
373 373 for (var j = 0; j < N; j++){
374 z[j].push(null);
374 y[j].push(null);
375 375 }
376 376 // Avoid infinite loop
377 377 if (cnt==100){break;}
Show file before | Show file after | Hide comments
@@ -80,7 +80,9
80 80 document.location = '/' + $("#id_experiment").val() + '/' + $("#id_plot").val() + '/?date=' + $("#id_date").val();
81 81 });
82 82
83 var socket = new WebSocket('ws://' + window.location.host + '/{{code}}/{{plot}}/');
83 {% if date %}
84 $("#loader").css("display", "block");
85 var socket = new WebSocket('ws://' + window.location.host + '/{{code}}/{{plot}}/?' + 'pk={{id}}');
84 86 socket.onopen = function open() {
85 87 console.log('WebSockets connection created.');
86 88 {% if date %}
@@ -92,13 +94,14
92 94
93 95 socket.onmessage = function message(event) {
94 96 var data = JSON.parse(event.data);
97 console.log(data);
95 98 if (data.interval == 0) {
96 99 $("#loader").removeClass("loader").addClass("no-data");
97 100 $("#loader").html("No data found");
98 101 } else {
99 102 var first = plot(data);
100 if (first == true) {
101 $("#loader").css("display", "none");
103 if (first == true) {
104 $("#loader").css("display", "none");
102 105 }
103 106 }
104 107 }
@@ -106,6 +109,9
106 109 if (socket.readyState == WebSocket.OPEN) {
107 110 socket.onopen();
108 111 }
112 {% else %}
113 $("#loader").css("display", "none");
114 {% endif %}
109 115 </script> {% block script %}{% endblock script %}
110 116 </body>
111 117
Show file before | Show file after | Hide comments
@@ -11,12 +11,12 from django.shortcuts import render
11 11
12 12 from bootstrap3_datetime.widgets import DateTimePicker
13 13
14 from pymongo import MongoClient
14 import mongoengine
15 15
16 host = os.environ.get('HOST_MONGO', 'localhost')
16 from plotter.models import Experiment, ExpMeta
17 17
18 CLIENT = MongoClient('{}:27017'.format(host))
19 DB = CLIENT['dbplots']
18 host = os.environ.get('HOST_MONGO', 'localhost')
19 mongoengine.connect('dbplots', host=host, port=27017)
20 20
21 21 # Forms
22 22 class SearchForm(forms.Form):
@@ -47,27 +47,30 def main(request, code=None, plot=None):
47 47 if plot is not None:
48 48 initial['plot'] = plot
49 49
50 print 'hola'
51 codes = DB.experiment.find().distinct('code')
52 print codes
53 exps = [DB.experiment.find_one({'code': c}, ['name']) for c in codes]
54 print exps
55 names = [q['name'] for q in exps]
56 print names
50 exps = [(q['code'], q['name']) for q in Experiment.objects.all()]
51
57 52 form = SearchForm(
58 53 initial = initial,
59 exp_choices = [(e[0], e[1]) for e in zip(codes, names)]
54 exp_choices = [(e[0], e[1]) for e in exps]
60 55 )
61 56
57 try:
58 exp = ExpMeta.objects.get(code=int(code), date=datetime.strptime(date, '%d/%m/%Y'))
59 exp_id = exp.id
60 except:
61 exp_id = 0
62
63
62 64 kwargs = {
63 65 'code': code,
64 66 'plot': plot,
65 67 'date': date,
66 68 'form': form,
69 'id': exp_id
67 70 }
68 71
69 if code and codes:
70 kwargs['title'] = [t[1] for t in zip(codes, names) if t[0]==int(code)][0]
72 if code and exps:
73 kwargs['title'] = [t[1] for t in exps if t[0]==int(code)][0]
71 74 else:
72 75 kwargs['title'] = 'JRO'
73 76
Show file before | Show file after | Hide comments
@@ -5,4 +5,4 channel_routing = [
5 5 route("websocket.connect", ws_connect, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
6 6 route("websocket.receive", ws_message, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
7 7 route("websocket.disconnect", ws_disconnect, path=r'^/(?P<code>[0-9]+)/(?P<plot>[a-z]+)/$'),
8 ] No newline at end of file
8 ]
Show file before | Show file after | Hide comments
@@ -122,7 +122,7 USE_TZ = True
122 122
123 123 STATIC_URL = '/static/'
124 124
125 host = os.environ.get('HOST_REDIS', 'localhost')
125 host = os.environ.get('HOST_REDIS', '127.0.0.1')
126 126
127 127 CHANNEL_LAYERS = {
128 128 "default": {
Show file before | Show file after | Hide comments
@@ -4,7 +4,6 django-bootstrap3==9.1.0
4 4 django-bootstrap3-datetimepicker-2==2.5.0
5 5 channels==1.1.8
6 6 mongoengine==0.15.0
7 numpy==1.13.3
8 7 pymongo==3.5.1
9 8 pyzmq==16.0.3
10 9 redis==2.10.6
Show file before | Show file after | Hide comments
@@ -1,5 +1,6
1 1 import os
2 2 import sys
3 import json
3 4 import simplejson
4 5 from datetime import datetime
5 6 import zmq
@@ -10,8 +11,7 import mongoengine
10 11 sys.path.append('/app')
11 12 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "realtime.settings")
12 13
13 from plotter.models import Experiment, Data
14
14 from plotter.models import Experiment, ExpMeta, ExpData
15 15
16 16 host_mongo = os.environ.get('HOST_MONGO', 'localhost')
17 17 mongoengine.connect('dbplots', host=host_mongo, port=27017)
@@ -26,69 +26,59 receiver = context.socket(zmq.SUB)
26 26 receiver.bind("tcp://0.0.0.0:4444")
27 27 receiver.setsockopt(zmq.SUBSCRIBE, '')
28 28
29
30
31 def update_db(buffer):
32 dt = datetime.utcfromtimestamp(buffer['time'])
33 exp = Experiment.objects(code=buffer['exp_code'], date=dt.date()).first()
34 if exp is None:
35 exp = Experiment(
36 code=buffer['exp_code'],
37 date=dt.date(),
38 yrange = buffer['yrange'],
39 xrange = buffer['xrange'],
40 interval = buffer['interval'],
41 localtime = buffer['localtime'],
42 name = buffer['name'],
29 def loaddata():
30 print('Loading Experiments...')
31 for tup in json.load(open('scripts/experiments.json')):
32 print(tup['name'])
33 exp = Experiment.objects(code=tup['code']).modify(
34 upsert=True,
35 new=True,
36 set__code=tup['code'],
37 set__name=tup['name'],
43 38 )
44 39 exp.save()
40
41 def update(buffer):
42 dt = datetime.utcfromtimestamp(buffer['time'])
43 exp = ExpMeta.objects(code=buffer['exp_code'], date=dt.date()).modify(
44 upsert=True,
45 new=True,
46 set__code=buffer['exp_code'],
47 set__date=dt.date(),
48 set__yrange = buffer['yrange'],
49 set__xrange = buffer['xrange'],
50 set__interval = buffer['interval'],
51 set__localtime = buffer['localtime'],
52 set__plots = buffer['data'].keys()
53 )
54 exp.save()
45 55
46 data = Data.objects(experiment=exp, time=buffer['time']).first()
56 data = ExpData.objects(expmeta=exp, time=buffer['time']).modify(
57 upsert=True,
58 new=True,
59 set__expmeta = exp,
60 set__time = buffer['time'],
61 set__data = buffer['data']
62 )
47 63
48 if data is None:
49 data = Data(
50 experiment = exp,
51 time = buffer['time'],
52 data = buffer['data']
53 ).save()
54 new = True
55 else:
56 data.data = buffer['data']
57 data.save()
58 new = False
64 data.save()
59 65
60 return new
61 print 'Starting...'
62 while True:
63 buffer = receiver.recv_json()
64 if 'xrange' not in buffer:
65 buffer['xrange'] = []
66 if 'name' not in buffer:
67 buffer['name'] = 'Experiment'
68 if update_db(buffer):
69 dum = buffer.copy()
70 dum['time'] = [buffer['time']]
71 dum['rti'] = buffer['data']['rti']
72 # dum['noise'] = buffer['data']['noise']
73 dum.pop('data')
74 code = dum.pop('exp_code')
75 channel.send_group(u'{}_rti'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)})
76 print 'Sending...{} - {} bytes'.format('rti', len(str(dum)))
66 return exp.id
77 67
78 # dum = buffer.copy()
79 # dum['time'] = [buffer['time']]
80 # dum['rti'] = buffer['data']['rti']
81 # dum['spc'] = buffer['data']['spc']
82 # dum['noise'] = buffer['data']['noise']
83 # dum.pop('data')
84 # code = dum.pop('exp_code')
85 # channel.send_group(u'{}_spc'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)})
86 # print 'Sending...{} - {} bytes'.format('spc', len(str(dum)))
68 def main():
69 print('Starting ZMQ server...')
70 while True:
71 buffer = receiver.recv_json()
72 code = update(buffer)
73 for plot in buffer['data']:
74 dum = buffer.copy()
75 dum['time'] = [buffer['time']]
76 dum[plot] = buffer['data'][plot]
77 dum.pop('data')
78 dum.pop('exp_code')
79 channel.send_group(u'{}_{}'.format(code, plot), {'text': simplejson.dumps(dum, ignore_nan=True)})
80 print('Sending...{} - {} bytes'.format(plot, len(str(dum))))
87 81
88 # dum = buffer.copy()
89 # dum['time'] = [buffer['time']]
90 # dum['noise'] = [[x] for x in buffer['data']['noise']]
91 # dum.pop('data')
92 # code = dum.pop('exp_code')
93 # channel.send_group(u'{}_noise'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)})
94 # print 'Sending...{} - {} bytes'.format('noise', len(str(dum)))
82 if __name__=='__main__':
83 loaddata()
84 main() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now