DATABASES/madrigal Commit - r10:ac38b2ff3424

Fix upload files

jespinoza -

r10:ac38b2ff3424

parent child

Context file:

r10:ac38b2ff3424

Collapse all files

.env +3 -2

@@ -1,3 +1,4
1	MADROOT=/madrigal	1	MADROOT=/madrigal
2	EXP_DIR=/~~usr/loc~~al/experiments	2	EXP_DIR=/data/madrigal/experiments
3	MAD_PORT=8080	3	META_DIR=/data/madrigal/metadata
		4	MAD_PORT=80

docker-compose.yml +2 -1

             version: '3'
             services:
               web:
                 container_name: 'madrigal'
                 build: .
                 restart: always
                 image: madrigal
                 working_dir: /madrigal/source/madpy/djangoMad
-                command: gunicorn djangoMad.wsgi --bind 0.0.0.0:8000
+                command: gunicorn djangoMad.wsgi --timeout 600 --bind 0.0.0.0:8000
                 env_file: .env
                 volumes:
                   - '${EXP_DIR}:/madrigal/experiments'
+                  - '${META_DIR}:/madrigal/metadata'
                   - './source/madpy/djangoMad:/madrigal/source/madpy/djangoMad'
               nginx:
                 container_name: 'madrigal-nginx'
                 image: nginx:1.15-alpine
                 volumes:
                     - ./source/madpy/djangoMad/static_files:/static
                     - ./nginx:/etc/nginx/conf.d
                 ports:
                     - "${MAD_PORT}:8080"
                 depends_on:
                     - web
                 command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"

nginx/app.conf +5 -1

             # configuration
             upstream djangomad {
                 server web:8000;
             }
             server {
                 listen 8080;
-                client_max_body_size 50M;
+                client_max_body_size 100M;
                 location /madrigal {
                     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                     proxy_set_header Host $host;
                     proxy_redirect off;
                     proxy_set_header X-Script-Name /madrigal;
                     proxy_pass http://djangomad;
                 }
                 location /madrigal/static/ {
                     alias /static/;
                 }
+                proxy_connect_timeout 600;
+                proxy_send_timeout 600;
+                proxy_read_timeout 600;
+                send_timeout 600;
             }

source/madpy/djangoMad/apps/updata/views.py +1 -1

             from django.contrib.auth.decorators import login_required
             from django.shortcuts import render
             from apps.updata.forms import UpdataForm, ExpForm
             from django.core.files.storage import FileSystemStorage
             from django.contrib import messages
             import os
             # madrigal imports
             import madrigal.metadata
             import madrigal.ui.web
             import madrigal.admin
             @login_required(login_url='/madrigal/accounts/login/')
             def index(request):
                 '''
                 Uploading experiments data view. Allows user to upload experiment files
                 '''
                 dbAdminObj = madrigal.admin.MadrigalDBAdmin()
                 madDB = madrigal.metadata.MadrigalDB()
                 madWebObj = madrigal.ui.web.MadrigalWeb(madDB)
                 siteName, siteList = madWebObj.getSiteInfo()
                 err = False
                 if request.method == 'POST':
                     form = UpdataForm(request.POST, request.FILES)
                     files = request.FILES.getlist('file')
-                    files.sort()
+                    #files.sort()
                     filenames = []
                     choose = request.POST.get('choose')
                     if choose == 'new':
                         instCode = int(request.POST.get('inst'))
                         expTitle = request.POST.get('exp')
                     else:
                         instCode = int(request.POST.get('instruments'))
                         expId = request.POST.get('experiments')
                         madExp = madrigal.metadata.MadrigalExperiment()
                         expTitle = madExp.getExpNameByExpId(expId)
                     description = request.POST.get('description')
                     perm = int(request.POST.get('permission'))
                     optchar = request.POST.get('optchar').strip()
                     first = True
                     for f in files:
                         fs = FileSystemStorage(location='/tmp')
                         fs.save(f.name, f)
                         filename = os.path.join('/tmp', f.name)
                         ext = filename.split('.')[-1]
                         if ext not in ('hdf5', 'h5'):
                             convert = True
                         else:
                             convert = False
                         if first:
                             first = False
                             try:
                                 expDir = dbAdminObj.createMadrigalExperiment(filename, expTitle, perm, description,
                                     instCode, optChar=optchar, updateToMad3=convert)
                             except Exception as e:
                                 err = True
                                 messages.error(request,
                                     'An error occur creating the experiment {}: {}'.format(expTitle, e)
                                     )
                                 break
                         else:
                             try:
                                 dbAdminObj.addMadrigalFile(expDir, filename, perm, description, updateToMad3=convert)
                             except Exception as e:
                                 err = True
                                 messages.error(request,
                                     'An error occur adding file {}: {}'.format(filename.split('/')[-1], e)
                                     )
                         filenames.append(filename.split('/')[-1])
                         os.remove(filename)
                     if not err:
                         messages.success(request,
                             'Experiment {} created succesfully with files: {}'.format(expTitle, filenames)
                         )
                         madInstParams = madrigal.metadata.MadrigalInstrumentParameters()
                         madInstKindats = madrigal.metadata.MadrigalInstrumentKindats()
                         print('*** Updating local metadata ***')
                         dbAdminObj.__updateLocalMetadata__()
                         print('*** Rebuilding instParmTab.txt ***')
                         madInstParams.rebuildInstParmTable()
                         print('*** Rebuilding instKindatTab.txt ***')
                         madInstKindats.rebuildInstKindatTable()
                         form = UpdataForm()
                 else:
                     form = UpdataForm()
                 return render(request, 'updata/index.html', {
                     'form': form,
                     'site_name': siteName,
                     'site_list': siteList,
                 })
             def get_experiments(request):
                 """get_experiments is a Ajax call that returns the experiments select html to support the
                 updata UI.  Called when a user modifies the intruments select field.
                 Inputs:
                     request
                 """
                 form = ExpForm(request.GET)
                 return render(request, 'updata/experiments.html', {'form': form})

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages