views.py
495 lines
| 17.4 KiB
| text/x-python
|
PythonLexer
| r0 | ''' | |||
| views for docs app | ||||
| @author: Bill Rideout | ||||
| @contact: brideout@haystack.mit.edu | ||||
| $Id: views.py 7285 2020-12-04 16:55:17Z brideout $ | ||||
| ''' | ||||
| # standard python imports | ||||
| import os, os.path, sys | ||||
| import random | ||||
| import datetime | ||||
| import re | ||||
| import xml.dom.minidom | ||||
| import hashlib | ||||
| # django imports | ||||
| import django, django.http | ||||
| from django.shortcuts import render | ||||
| from django.views.decorators.csrf import csrf_exempt | ||||
| from . import forms | ||||
| # madrigal imports | ||||
| import madrigal.metadata | ||||
| import madrigal.ui.web | ||||
| import madrigal.openmadrigal | ||||
| import subversion_operations | ||||
| # helper methods | ||||
| def get_exp_info(control_file, rev): | ||||
| """get_exp_info returns a tuple of (exp_desc, cyc_desc, control_file_text) for the input control_file and revision. | ||||
| For now grabs info from Subversion on apollo | ||||
| """ | ||||
| exp_desc = '' | ||||
| cyc_desc = '' | ||||
| control_file_text = '' | ||||
| subObj = subversion_operations.reader() | ||||
| xmlFilename = control_file.replace('.dat', '.xml') | ||||
| output = os.path.join('/tmp', control_file) | ||||
| sub_path = 'millstone_geospace_radar/control/ISRExperiments' | ||||
| try: | ||||
| subObj.getFile(os.path.join(sub_path, control_file), '/tmp', rev) | ||||
| except: | ||||
| try: | ||||
| subObj.getFile(os.path.join(sub_path, control_file), '/tmp') | ||||
| except: | ||||
| return((exp_desc, cyc_desc, control_file_text)) | ||||
| f = open(output) | ||||
| control_file_text = f.read() | ||||
| f.close() | ||||
| try: | ||||
| subObj.getFile(os.path.join(sub_path, xmlFilename), '/tmp', rev) | ||||
| except: | ||||
| try: | ||||
| subObj.getFile(os.path.join(sub_path, xmlFilename), '/tmp') | ||||
| except: | ||||
| return((exp_desc, cyc_desc, control_file_text)) | ||||
| f = open(output) | ||||
| xml_file_text = f.read() | ||||
| f.close() | ||||
| # now build complete xml file | ||||
| cvsText = '<?xml version="1.0"?>\n<ExperimentDescriptionFile>\n' | ||||
| cvsText += xml_file_text | ||||
| # closing tag | ||||
| cvsText += '</ExperimentDescriptionFile>' | ||||
| expDescDoc = xml.dom.minidom.parseString(cvsText) | ||||
| # get all experiments | ||||
| expElemList = expDescDoc.getElementsByTagName("Experiment") | ||||
| for exp in expElemList: | ||||
| # get one exp name from exp list | ||||
| nameEl = exp.getElementsByTagName("ExperimentName")[0] | ||||
| # get text node, convert from unicode to ascii | ||||
| expName = '' | ||||
| for elem in nameEl.childNodes: | ||||
| if elem.nodeType == elem.TEXT_NODE: | ||||
| expName += elem.data.encode('utf-8') | ||||
| # get exp descriptions from exp list | ||||
| expDescElList = exp.getElementsByTagName("ExperimentDescription") | ||||
| for expDescEl in expDescElList: | ||||
| # get text node, convert from unicode to ascii | ||||
| expDesc = '' | ||||
| for elem in expDescEl.childNodes: | ||||
| if elem.nodeType == elem.TEXT_NODE: | ||||
| expDesc += elem.data.encode('utf-8') | ||||
| if len(exp_desc) == 0: | ||||
| exp_desc = expDesc # make sure we get something even if no id match | ||||
| # get one CVS_rev | ||||
| cvsEl = expDescEl.getElementsByTagName("CVS_rev")[0] | ||||
| cvs = '' | ||||
| for elem in cvsEl.childNodes: | ||||
| if elem.nodeType == elem.TEXT_NODE: | ||||
| cvs += elem.data.encode('utf-8') | ||||
| if cvs == rev: | ||||
| # right rev - overwrite | ||||
| if len(expDesc) == 0: | ||||
| exp_desc = expDesc | ||||
| # get cycle descriptions from exp list | ||||
| cycDescElList = exp.getElementsByTagName("CycleDescription") | ||||
| for cycDescEl in cycDescElList: | ||||
| # get text node, convert from unicode to ascii | ||||
| cycDesc = '' | ||||
| for elem in cycDescEl.childNodes: | ||||
| if elem.nodeType == elem.TEXT_NODE: | ||||
| cycDesc += elem.data.encode('utf-8') | ||||
| if len(cyc_desc) == 0: | ||||
| cyc_desc = cycDesc # make sure we get something even if no id match | ||||
| # get one CVS_rev | ||||
| cvsEl = cycDescEl.getElementsByTagName("CVS_rev")[0] | ||||
| cvs = '' | ||||
| for elem in cvsEl.childNodes: | ||||
| if elem.nodeType == elem.TEXT_NODE: | ||||
| cvs += elem.data.encode('utf-8') | ||||
| if cvs == rev: | ||||
| # right rev - overwrite | ||||
| if len(cycDesc) == 0: | ||||
| cyc_desc = cycDesc | ||||
| return((exp_desc, cyc_desc, control_file_text)) | ||||
| @csrf_exempt | ||||
| def get_log_admin(request): | ||||
| """get_log_admin returns the admin access log page. | ||||
| Inputs: | ||||
| request | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| bg_color = madDB.getBackgroundColor() | ||||
| if request.method == 'POST': | ||||
| form =forms.AdminLogForm(request.POST) | ||||
| if form.is_valid(): | ||||
| cleaned_data = form.cleaned_data | ||||
| kinstList = [int(item) for item in eval(cleaned_data['kinst'])] | ||||
| startDT = datetime.datetime(cleaned_data['startDate'].year, cleaned_data['startDate'].month, cleaned_data['startDate'].day) | ||||
| endDT = datetime.datetime(cleaned_data['endDate'].year, cleaned_data['endDate'].month, cleaned_data['endDate'].day) | ||||
| madWebObj = madrigal.ui.web.MadrigalWeb(madDB) | ||||
| stageDir = os.path.join(madDB.getMadroot(), 'experiments/stage') | ||||
| tmpFile = os.path.join(stageDir, 'admin_%i.log' % (random.randint(0, 1000000))) | ||||
| madWebObj.filterLog(tmpFile, kinstList, startDT, endDT) | ||||
| f = open(tmpFile, 'r') | ||||
| thisFile = django.core.files.File(f) | ||||
| response = django.http.HttpResponse(thisFile, content_type='application/x-octet-stream') | ||||
| response['Content-Disposition'] = 'attachment; filename="' + os.path.basename(tmpFile) + '"' | ||||
| response['Content-Length'] = os.path.getsize(tmpFile) | ||||
| response.set_cookie('fileDownload', 'true', path='/', samesite='Strict') | ||||
| return(response) | ||||
| else: | ||||
| form =forms.AdminLogForm() | ||||
| responseDict = {'bg_color': bg_color} | ||||
| responseDict['form'] = form | ||||
| return render(request, 'get_log_admin.html', responseDict) | ||||
| def get_latest_metadata_version(request): | ||||
| """get_latest_metadata_version returns the latest version of the specified metadata file | ||||
| Inputs: | ||||
| request - contains key fullPath - full path to the Madrigal file in Subversion relative to trunk | ||||
| (example: 'madroot/metadata/siteTab.txt') | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| fullPath = request.GET['fullPath'] | ||||
| subObj = subversion_operations.reader('OpenMadrigal') | ||||
| output = os.path.join('/tmp', os.path.basename(fullPath)) | ||||
| path = os.path.join('trunk', fullPath) | ||||
| subObj.getFile(path, '/tmp') | ||||
| f = open(output) | ||||
| fileStr = f.read() | ||||
| f.close() | ||||
| try: | ||||
| os.remove(output) | ||||
| except: | ||||
| pass | ||||
| return(django.http.HttpResponse(fileStr)) | ||||
| def get_all_metadata_versions(request): | ||||
| """get_all_metadata_versions returns the a page listing all versions numbers available for a given metadata | ||||
| file, one line for each version string | ||||
| Inputs: | ||||
| request - contains key fullPath - full path to the Madrigal file in Subversion relative to trunk | ||||
| (example: 'madroot/metadata/siteTab.txt') | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| fullPath = request.GET['fullPath'] | ||||
| subObj = subversion_operations.reader('OpenMadrigal') | ||||
| path = os.path.join('trunk', fullPath) | ||||
| revDict = subObj.getFileRevsDates(path) | ||||
| retStr = '' | ||||
| for k in sorted(revDict): | ||||
| retStr += '%i\n' % (revDict[k]) | ||||
| return(django.http.HttpResponse(retStr)) | ||||
| def get_metadata_version(request): | ||||
| """get_metadata_version returns the specifed version of the specified metadata file | ||||
| Inputs: | ||||
| request - contains key fullPath - full path to the Madrigal file in Subversion relative to trunk | ||||
| (example: 'madroot/metadata/siteTab.txt'), and key version | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| fullPath = request.GET['fullPath'] | ||||
| version = request.GET['version'] | ||||
| subObj = subversion_operations.reader('OpenMadrigal') | ||||
| output = os.path.join('/tmp', os.path.basename(fullPath)) | ||||
| path = os.path.join('trunk', fullPath) | ||||
| subObj.getFile(path, '/tmp', version) | ||||
| f = open(output, 'rb') | ||||
| text = f.read() | ||||
| f.close() | ||||
| try: | ||||
| os.remove(output) | ||||
| except: | ||||
| pass | ||||
| return(django.http.HttpResponse(text)) | ||||
| def get_open_madrigal_shared_files(request): | ||||
| """get_open_madrigal_shared_files returns the specifed version of the shared OpenMadrigal file | ||||
| Inputs: | ||||
| request - contains key filename - in form siteName_siteID/expTab.txt or siteName_siteID/fileTab.txt | ||||
| For now hardcoded path to files | ||||
| """ | ||||
| path = '/usr/local/apache2/htdocs/madrigal/distributionFiles/metadata' | ||||
| path3 = '/usr/local/apache2/htdocs/madrigal/distributionFiles/metadata3' # used because siteTab.txt for mad3 is incompatible | ||||
| filename = request.GET['filename'] | ||||
| if filename == 'siteTab.txt': | ||||
| fullPath = os.path.join(path3, filename) | ||||
| else: | ||||
| fullPath = os.path.join(path, filename) | ||||
| f = open(fullPath, 'r') | ||||
| text = f.read() | ||||
| f.close() | ||||
| return(django.http.HttpResponse(text)) | ||||
| def get_madrigal_videos(request): | ||||
| """get_madrigal_videos returns the Madrigal video tutorials page | ||||
| Inputs: | ||||
| request | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| bg_color = madDB.getBackgroundColor() | ||||
| responseDict = {'bg_color': bg_color} | ||||
| return render(request, 'get_video_tutorials.html', responseDict) | ||||
| def get_exp_notes(request): | ||||
| """get_exp_notes returns the description of the control file - links from catalog in MLH files | ||||
| Inputs: | ||||
| request | ||||
| """ | ||||
| responseDict = {} | ||||
| responseDict['control_file'] = request.GET['exp'] | ||||
| responseDict['rev'] = request.GET['rev'] | ||||
| exp_desc, cyc_desc, control_file_text = get_exp_info(responseDict['control_file'], | ||||
| responseDict['rev']) | ||||
| responseDict['exp_desc'] = exp_desc | ||||
| responseDict['cyc_desc'] = cyc_desc | ||||
| responseDict['control_file_text'] = control_file_text | ||||
| return render(request, 'exp_notes.html', responseDict) | ||||
| def compare_to_archive(request): | ||||
| """compare_to_archive implements the old compareToArchive.py cgi script | ||||
| Now checks newest to oldest, up to 50 max | ||||
| """ | ||||
| filePath = request.GET['filePath'] | ||||
| fileTextMd5 = request.GET['fileTextMd5'] | ||||
| strip = False | ||||
| if 'strip' in request.GET: | ||||
| strip = True | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| openMadObj = madrigal.openmadrigal.OpenMadrigal(madDB) | ||||
| revList = openMadObj.getAllRevisionNumbers(filePath) | ||||
| revList.reverse() # check newest first | ||||
| if len(revList) == 0: | ||||
| return(django.http.HttpResponse('None None')) | ||||
| # loop through all revisions, looking for a match | ||||
| for rev in revList[:50]: | ||||
| thisText = openMadObj.getCvsVersion(filePath, rev) | ||||
| if thisText is None: | ||||
| continue | ||||
| if strip: | ||||
| data = thisText.strip().encode() | ||||
| thisMd5 = hashlib.md5(data) | ||||
| else: | ||||
| thisMd5 = hashlib.md5(thisText.encode()) | ||||
| if thisMd5.hexdigest() == fileTextMd5: | ||||
| return(django.http.HttpResponse('%s %s' % (revList[0], rev))) | ||||
| # no matches found | ||||
| return(django.http.HttpResponse('%s None' % (revList[0]))) | ||||
| def open_madrigal(request): | ||||
| """openmadrigal implements the openmadigal page | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| madSiteObj = madrigal.metadata.MadrigalSite(madDB) | ||||
| responseDict = {} | ||||
| site_list = [] | ||||
| for siteId, siteName in madSiteObj.getSiteList(): | ||||
| if madSiteObj.getSiteServer(siteId).find('http') == -1: | ||||
| url = 'http://' + os.path.join(madSiteObj.getSiteServer(siteId), | ||||
| madSiteObj.getSiteDocRoot(siteId)) | ||||
| else: | ||||
| url = os.path.join(madSiteObj.getSiteServer(siteId), | ||||
| madSiteObj.getSiteDocRoot(siteId)) | ||||
| site_list.append((url, siteName)) | ||||
| responseDict['site_list'] = site_list | ||||
| return(render(request, 'openmadrigal.html', responseDict)) | ||||
| def madrigal_admin(request): | ||||
| """madrigal_admin implements the openmadigal admin page | ||||
| """ | ||||
| responseDict = {} | ||||
| return(render(request, 'admin.html', responseDict)) | ||||
| def madrigal_download(request): | ||||
| """madrigal_download implements the openmadigal download page | ||||
| """ | ||||
| responseDict = {} | ||||
| return(render(request, 'madDownload.html', responseDict)) | ||||
| def get_citation_group_service(request): | ||||
| """get_citation_group returns a list of citations created as a group using a group id | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| id = int(request.GET['id']) | ||||
| citations = madDB.getListFromGroupId(id) | ||||
| text = '' | ||||
| for url in citations: | ||||
| text += '%s\n' % (url) | ||||
| return render(request, 'service.html', {'text': django.utils.safestring.mark_safe(text)}) | ||||
| def create_citation_group_with_list(request): | ||||
| """create_citation_group_with_list create a citation group through passed in group of citations. | ||||
| Returns citation to group | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| try: | ||||
| user_fullname = request.GET['user_fullname'] | ||||
| except KeyError: | ||||
| return render(request, 'service.html', {'text': 'user_fullname argument missing'}) | ||||
| try: | ||||
| user_email = request.GET['user_email'] | ||||
| except KeyError: | ||||
| return render(request, 'service.html', {'text': 'user_email argument missing'}) | ||||
| try: | ||||
| user_affiliation = request.GET['user_affiliation'] | ||||
| except KeyError: | ||||
| return render(request, 'service.html', {'text': 'user_affiliation argument missing'}) | ||||
| citations=request.GET.getlist('url') | ||||
| if len(citations) == 0: | ||||
| return render(request, 'service.html', {'text': 'url argument missing; multiple allowed'}) | ||||
| # verify citations in proper format | ||||
| for citation in citations: | ||||
| if citation[:11] not in ('https://w3i', 'experiments'): | ||||
| return render(request, 'service.html', {'text': 'Illegal url %s' % (citation)}) | ||||
| id = madDB.createGroupIdWithList(user_fullname, user_email, user_affiliation, citations) | ||||
| return render(request, 'service.html', {'text': 'http://cedar.openmadrigal.org/getCitationGroup?id=%i\n' % (id)}) | ||||
| def get_citation_group_with_filters(request): | ||||
| """get_citation_group_with_filters returns a list of citations (one per line) using filters similar to globalDownload. | ||||
| Result can then be used to create citation group using create_citation_group_with_list | ||||
| Request contains to following keys: | ||||
| startDate: start datetime to filter experiments before in YYYY-MM-DD (required) | ||||
| endDate: end datetime to filter experiments after in YYYY-MM-DD (required) | ||||
| inst: a list of instrument codes or names. If not set, all instruments used. | ||||
| For names fnmatch will be used | ||||
| kindat: a list of kind of data codes or names. If not set, all kindats used. | ||||
| For names fnmatch will be used | ||||
| seasonalStartDate: in form MM/DD, rejects all days earlier in year. If not set, | ||||
| implies 01/01 | ||||
| seasonalEndDate: in form MM/DD, rejects all days later in year. If not set, | ||||
| implies 12/31 | ||||
| includeNonDefault: if set, include realtime files when there are no default. If not set, | ||||
| implies only default files. | ||||
| expName: string - filter experiments by the experiment name. fnmatch rules | ||||
| If not set, no filtering by experiment name. | ||||
| excludeExpName: string - exclude experiments by the experiment name. fnmatch rules | ||||
| If not set, no excluding experiments by experiment name. | ||||
| fileDesc: filter files using input file Description string via fnmatch. | ||||
| If not set, no filtering by file name | ||||
| Returns a list with all citations in group, one per line | ||||
| """ | ||||
| madDB = madrigal.metadata.MadrigalDB() | ||||
| madWebObj = madrigal.ui.web.MadrigalWeb(madDB) | ||||
| if not 'startDate' in request.GET: | ||||
| return(django.http.HttpResponse('<p>startDate required for getCitationGroupWithFilters</p>')) | ||||
| startDate = request.GET.get('startDate') | ||||
| startDate = datetime.datetime.strptime(startDate, '%Y-%m-%d') | ||||
| endDate = request.GET.get('endDate') | ||||
| endDate = datetime.datetime.strptime(endDate, '%Y-%m-%d') | ||||
| inst = request.GET.getlist('inst') | ||||
| if len(inst) == 0: | ||||
| inst = None | ||||
| kindat = request.GET.getlist('kindat') | ||||
| if len(kindat) == 0: | ||||
| kindat = None | ||||
| seasonalStartDate = request.GET.get('seasonalStartDate') | ||||
| seasonalEndDate = request.GET.get('seasonalEndDate') | ||||
| if 'includeNonDefault' in request.GET: | ||||
| includeNonDefault = True | ||||
| else: | ||||
| includeNonDefault = False | ||||
| expName = request.GET.get('expName') | ||||
| excludeExpName = request.GET.get('excludeExpName') | ||||
| fileDesc = request.GET.get('fileDesc') | ||||
| citations = madWebObj.global_file_search(startDate, endDate, inst, kindat, | ||||
| seasonalStartDate, seasonalEndDate, | ||||
| includeNonDefault, expName, excludeExpName, | ||||
| fileDesc, returnCitation=True) | ||||
| if len(citations) == 0: | ||||
| return render(request, 'service.html', {'text': 'No citations found with given filters - no group citation created'}) | ||||
| else: | ||||
| text = '' | ||||
| for url in citations: | ||||
| text += '%s\n' % (url) | ||||
| return render(request, 'service.html', {'text': text}) | ||||
