diff --git a/Dockerfile b/Dockerfile index 58c293a..76b08b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ g++ \ gcc \ && pip install -r requirements.txt \ - && apt-get purge -y --auto-remove gcc g++\ + && apt-get purge -y --auto-remove gcc g++ \ && rm -rf /var/lib/apt/lists/* COPY . /app/ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 21b2271..9aa2435 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: container_name: 'zmq_server' image: 'realtime' ports: - - '127.0.0.1:4444:4444' + - '4444:4444' command: 'python -u scripts/server.py' env_file: .env links: diff --git a/plotter/consumers.py b/plotter/consumers.py index 39c1d55..c3d2dc2 100644 --- a/plotter/consumers.py +++ b/plotter/consumers.py @@ -1,12 +1,10 @@ import os import json -import numpy as np + from datetime import datetime from pymongo import MongoClient -from models import Experiment, Data - from channels.handler import AsgiHandler from channels.auth import channel_session_user from channels import Group @@ -17,32 +15,35 @@ DB = CLIENT['dbplots'] # Connected to websocket.connect def ws_connect(message, code, plot): - # Accept the incoming connection message.reply_channel.send({'accept': True}) - # Add them to the chat group - Group('{}_{}'.format(code, plot)).add(message.reply_channel) + pk = message.content['query_string'].split('=')[1] + Group('{}_{}'.format(pk, plot)).add(message.reply_channel) + print('New connection from: {}, creating Group: {}_{}'.format(message.content['client'][0], pk, plot)) def ws_message(message, code, plot): # Accept the incoming connection - + print 'incoming message' dt = datetime.strptime(str(json.loads(message.content['text'])['date']), '%d/%m/%Y') - e = DB.experiment.find_one({'date': dt}) - if e: + exp = DB.exp_meta.find_one({'code': int(code), 'date': dt}) + print exp + if exp: if plot == 'spc': - datas = DB.data.find({'experiment': e['_id']}, ['time', 'data']).sort('time', -1).limit(1)[0] - e['time'] = [datas['time']] - e['spc'] = datas['data']['spc'] - e['rti'] = datas['data']['rti'] - e['noise'] = datas['data']['noise'] + datas = DB.exp_data.find({'expmeta': exp['_id']}, ['time', 'data']).sort('time', -1).limit(1)[0] + exp['time'] = [datas['time']] + exp['spc'] = datas['data']['spc'] + exp['rti'] = datas['data']['rti'] + exp['noise'] = datas['data']['noise'] else: - datas = DB.data.find({'experiment': e['_id']}, ['time', 'data']).sort('time', 1) + datas = DB.exp_data.find({'expmeta': exp['_id']}, ['time', 'data']).sort('time', 1) + print 'Counting...' + print datas.count() dum = [(d['time'], d['data'][plot]) for d in datas] - e['time'] = [d[0] for d in dum] + exp['time'] = [d[0] for d in dum] dum = [d[1] for d in dum] - e[plot] = map(list, zip(*dum)) - e.pop('date', None) - e.pop('_id', None) - message.reply_channel.send({'text': json.dumps(e)}) + exp[plot] = map(list, zip(*dum)) + exp.pop('date', None) + exp.pop('_id', None) + message.reply_channel.send({'text': json.dumps(exp)}) else: message.reply_channel.send({'text': json.dumps({'interval': 0})}) diff --git a/plotter/models.py b/plotter/models.py index 1a384ec..17e5b98 100644 --- a/plotter/models.py +++ b/plotter/models.py @@ -5,27 +5,29 @@ from django.db import models from mongoengine import * class Experiment(Document): + code = IntField(unique=True) + name = StringField(max_length=40) + +class ExpMeta(Document): code = IntField() - name = StringField(max_length=40, default='Experiment') date = DateTimeField() pairs = ListField(default=list) yrange = ListField(FloatField()) xrange = ListField(FloatField()) interval = FloatField() + plots = ListField(StringField()) localtime = BooleanField() meta = { 'indexes': [[("code", 1), ("date", 1)]] } -class Data(Document): - experiment = LazyReferenceField(Experiment) +class ExpData(Document): + expmeta = LazyReferenceField(ExpMeta) time = FloatField() data = DictField() meta = { - 'indexes': ["experiment", "+time"] + 'indexes': ["expmeta", "+time"] } -# connect('dbplots') -# Experiment.drop_collection() diff --git a/plotter/static/js/jroplots.js b/plotter/static/js/jroplots.js index c1bdcb2..d419ddc 100644 --- a/plotter/static/js/jroplots.js +++ b/plotter/static/js/jroplots.js @@ -371,7 +371,7 @@ class Scatter{ last = last+interval; x.push(last); for (var j = 0; j < N; j++){ - z[j].push(null); + y[j].push(null); } // Avoid infinite loop if (cnt==100){break;} diff --git a/plotter/templates/base.html b/plotter/templates/base.html index db246fc..e9b8f8c 100644 --- a/plotter/templates/base.html +++ b/plotter/templates/base.html @@ -80,7 +80,9 @@ document.location = '/' + $("#id_experiment").val() + '/' + $("#id_plot").val() + '/?date=' + $("#id_date").val(); }); - var socket = new WebSocket('ws://' + window.location.host + '/{{code}}/{{plot}}/'); + {% if date %} + $("#loader").css("display", "block"); + var socket = new WebSocket('ws://' + window.location.host + '/{{code}}/{{plot}}/?' + 'pk={{id}}'); socket.onopen = function open() { console.log('WebSockets connection created.'); {% if date %} @@ -92,13 +94,14 @@ socket.onmessage = function message(event) { var data = JSON.parse(event.data); + console.log(data); if (data.interval == 0) { $("#loader").removeClass("loader").addClass("no-data"); $("#loader").html("No data found"); } else { var first = plot(data); - if (first == true) { - $("#loader").css("display", "none"); + if (first == true) { + $("#loader").css("display", "none"); } } } @@ -106,6 +109,9 @@ if (socket.readyState == WebSocket.OPEN) { socket.onopen(); } + {% else %} + $("#loader").css("display", "none"); + {% endif %} {% block script %}{% endblock script %} diff --git a/plotter/views.py b/plotter/views.py index 023fba2..7df581c 100644 --- a/plotter/views.py +++ b/plotter/views.py @@ -11,12 +11,12 @@ from django.shortcuts import render from bootstrap3_datetime.widgets import DateTimePicker -from pymongo import MongoClient +import mongoengine -host = os.environ.get('HOST_MONGO', 'localhost') +from plotter.models import Experiment, ExpMeta -CLIENT = MongoClient('{}:27017'.format(host)) -DB = CLIENT['dbplots'] +host = os.environ.get('HOST_MONGO', 'localhost') +mongoengine.connect('dbplots', host=host, port=27017) # Forms class SearchForm(forms.Form): @@ -47,27 +47,30 @@ def main(request, code=None, plot=None): if plot is not None: initial['plot'] = plot - print 'hola' - codes = DB.experiment.find().distinct('code') - print codes - exps = [DB.experiment.find_one({'code': c}, ['name']) for c in codes] - print exps - names = [q['name'] for q in exps] - print names + exps = [(q['code'], q['name']) for q in Experiment.objects.all()] + form = SearchForm( initial = initial, - exp_choices = [(e[0], e[1]) for e in zip(codes, names)] + exp_choices = [(e[0], e[1]) for e in exps] ) + try: + exp = ExpMeta.objects.get(code=int(code), date=datetime.strptime(date, '%d/%m/%Y')) + exp_id = exp.id + except: + exp_id = 0 + + kwargs = { 'code': code, 'plot': plot, 'date': date, 'form': form, + 'id': exp_id } - if code and codes: - kwargs['title'] = [t[1] for t in zip(codes, names) if t[0]==int(code)][0] + if code and exps: + kwargs['title'] = [t[1] for t in exps if t[0]==int(code)][0] else: kwargs['title'] = 'JRO' diff --git a/realtime/routing.py b/realtime/routing.py index 8367da0..4cc6a2f 100644 --- a/realtime/routing.py +++ b/realtime/routing.py @@ -5,4 +5,4 @@ channel_routing = [ route("websocket.connect", ws_connect, path=r'^/(?P[0-9]+)/(?P[a-z]+)/$'), route("websocket.receive", ws_message, path=r'^/(?P[0-9]+)/(?P[a-z]+)/$'), route("websocket.disconnect", ws_disconnect, path=r'^/(?P[0-9]+)/(?P[a-z]+)/$'), -] \ No newline at end of file +] diff --git a/realtime/settings.py b/realtime/settings.py index 78b8353..6738b05 100644 --- a/realtime/settings.py +++ b/realtime/settings.py @@ -122,7 +122,7 @@ USE_TZ = True STATIC_URL = '/static/' -host = os.environ.get('HOST_REDIS', 'localhost') +host = os.environ.get('HOST_REDIS', '127.0.0.1') CHANNEL_LAYERS = { "default": { diff --git a/requirements.txt b/requirements.txt index 9be8471..9ae5d76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ django-bootstrap3==9.1.0 django-bootstrap3-datetimepicker-2==2.5.0 channels==1.1.8 mongoengine==0.15.0 -numpy==1.13.3 pymongo==3.5.1 pyzmq==16.0.3 redis==2.10.6 diff --git a/scripts/experiments.json b/scripts/experiments.json new file mode 100644 index 0000000..247f03d --- /dev/null +++ b/scripts/experiments.json @@ -0,0 +1,110 @@ +[ + { + "code" : 100, + "name" : "HF JRO" + }, + { + "code" : 101, + "name" : "HF Huancayo" + }, + { + "code" : 102, + "name" : "HF Ica" + }, + { + "code" : 110, + "name" : "ISR EW Drifts" + }, + { + "code" : 111, + "name" : "ISR Faraday" + }, + { + "code" : 112, + "name" : "ISR Imaging" + }, + { + "code" : 120, + "name" : "Faraday" + }, + { + "code" : 121, + "name" : "Faraday Long Pulse" + }, + { + "code" : 122, + "name" : "Faraday Codded LP" + }, + { + "code" : 123, + "name" : "Faraday Double Pulse" + }, + { + "code" : 124, + "name" : "Faraday AC" + }, + { + "code" : 125, + "name" : "Faraday Differential Phase" + }, + { + "code" : 150, + "name" : "JASMET 50" + }, + { + "code" : 151, + "name" : "JASMET 30" + }, + { + "code" : 170, + "name" : "BLTR Huancayo" + }, + { + "code" : 171, + "name" : "CIRI Huancayo" + }, + { + "code" : 172, + "name" : "CLAIRE Huancayo" + }, + { + "code" : 180, + "name" : "MST" + }, + { + "code" : 181, + "name" : "ISR" + }, + { + "code" : 182, + "name" : "EEJ" + }, + { + "code" : 190, + "name" : "JULIA 150km (Natalia)" + }, + { + "code" : 191, + "name" : "JULIA SpreadF (Natalia)" + }, + { + "code" : 200, + "name" : "JULIA 150km" + }, + { + "code" : 201, + "name" : "JULIA EEJ" + }, + { + "code" : 202, + "name" : "JULIA SpreadF" + }, + { + "code" : 203, + "name" : "JULIA Imaging" + }, + { + "code" : 203, + "name" : "JULIA Bistatic" + } +] \ No newline at end of file diff --git a/scripts/server.py b/scripts/server.py index 7fcb0dd..7e3dee4 100644 --- a/scripts/server.py +++ b/scripts/server.py @@ -1,5 +1,6 @@ import os import sys +import json import simplejson from datetime import datetime import zmq @@ -10,8 +11,7 @@ import mongoengine sys.path.append('/app') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "realtime.settings") -from plotter.models import Experiment, Data - +from plotter.models import Experiment, ExpMeta, ExpData host_mongo = os.environ.get('HOST_MONGO', 'localhost') mongoengine.connect('dbplots', host=host_mongo, port=27017) @@ -26,69 +26,59 @@ receiver = context.socket(zmq.SUB) receiver.bind("tcp://0.0.0.0:4444") receiver.setsockopt(zmq.SUBSCRIBE, '') - - -def update_db(buffer): - dt = datetime.utcfromtimestamp(buffer['time']) - exp = Experiment.objects(code=buffer['exp_code'], date=dt.date()).first() - if exp is None: - exp = Experiment( - code=buffer['exp_code'], - date=dt.date(), - yrange = buffer['yrange'], - xrange = buffer['xrange'], - interval = buffer['interval'], - localtime = buffer['localtime'], - name = buffer['name'], +def loaddata(): + print('Loading Experiments...') + for tup in json.load(open('scripts/experiments.json')): + print(tup['name']) + exp = Experiment.objects(code=tup['code']).modify( + upsert=True, + new=True, + set__code=tup['code'], + set__name=tup['name'], ) exp.save() + +def update(buffer): + dt = datetime.utcfromtimestamp(buffer['time']) + exp = ExpMeta.objects(code=buffer['exp_code'], date=dt.date()).modify( + upsert=True, + new=True, + set__code=buffer['exp_code'], + set__date=dt.date(), + set__yrange = buffer['yrange'], + set__xrange = buffer['xrange'], + set__interval = buffer['interval'], + set__localtime = buffer['localtime'], + set__plots = buffer['data'].keys() + ) + exp.save() - data = Data.objects(experiment=exp, time=buffer['time']).first() + data = ExpData.objects(expmeta=exp, time=buffer['time']).modify( + upsert=True, + new=True, + set__expmeta = exp, + set__time = buffer['time'], + set__data = buffer['data'] + ) - if data is None: - data = Data( - experiment = exp, - time = buffer['time'], - data = buffer['data'] - ).save() - new = True - else: - data.data = buffer['data'] - data.save() - new = False + data.save() - return new -print 'Starting...' -while True: - buffer = receiver.recv_json() - if 'xrange' not in buffer: - buffer['xrange'] = [] - if 'name' not in buffer: - buffer['name'] = 'Experiment' - if update_db(buffer): - dum = buffer.copy() - dum['time'] = [buffer['time']] - dum['rti'] = buffer['data']['rti'] - # dum['noise'] = buffer['data']['noise'] - dum.pop('data') - code = dum.pop('exp_code') - channel.send_group(u'{}_rti'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)}) - print 'Sending...{} - {} bytes'.format('rti', len(str(dum))) + return exp.id - # dum = buffer.copy() - # dum['time'] = [buffer['time']] - # dum['rti'] = buffer['data']['rti'] - # dum['spc'] = buffer['data']['spc'] - # dum['noise'] = buffer['data']['noise'] - # dum.pop('data') - # code = dum.pop('exp_code') - # channel.send_group(u'{}_spc'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)}) - # print 'Sending...{} - {} bytes'.format('spc', len(str(dum))) +def main(): + print('Starting ZMQ server...') + while True: + buffer = receiver.recv_json() + code = update(buffer) + for plot in buffer['data']: + dum = buffer.copy() + dum['time'] = [buffer['time']] + dum[plot] = buffer['data'][plot] + dum.pop('data') + dum.pop('exp_code') + channel.send_group(u'{}_{}'.format(code, plot), {'text': simplejson.dumps(dum, ignore_nan=True)}) + print('Sending...{} - {} bytes'.format(plot, len(str(dum)))) - # dum = buffer.copy() - # dum['time'] = [buffer['time']] - # dum['noise'] = [[x] for x in buffer['data']['noise']] - # dum.pop('data') - # code = dum.pop('exp_code') - # channel.send_group(u'{}_noise'.format(code), {'text': simplejson.dumps(dum, ignore_nan=True)}) - # print 'Sending...{} - {} bytes'.format('noise', len(str(dum))) +if __name__=='__main__': + loaddata() + main() \ No newline at end of file