|
|
#!PYTHONEXE
|
|
|
|
|
|
import sys
|
|
|
import os
|
|
|
import traceback
|
|
|
|
|
|
def printUsage():
|
|
|
|
|
|
message = """
|
|
|
Usage: tarExperiments [-start <startdate as YYYYMMDD>]
|
|
|
[-end <enddate as YYYYMMDD>]
|
|
|
[-excludePrivData]
|
|
|
[-ignoreDirCon]
|
|
|
[-includeNonDefData]
|
|
|
[-onlyData]
|
|
|
[-quiet]
|
|
|
[-filetype <type>]
|
|
|
-f <outputFile>
|
|
|
|
|
|
where:
|
|
|
-start: set a start date - any folder and all subfolders with an
|
|
|
expTab.txt before YYYYMMDD is excluded.
|
|
|
|
|
|
-end: set a end date - any folder and all subfolders with an expTab.txt
|
|
|
after YYYYMMDD is excluded.
|
|
|
|
|
|
|
|
|
-excludePrivData: if set, omit data marked as private (and the line
|
|
|
from the fileTab.txt to be removed). Default is to use all data, public and
|
|
|
private.
|
|
|
|
|
|
-ignoreDirCon: Has no effect, since directory convention no longer
|
|
|
built into Madrigal. Retained only for backward compatability.
|
|
|
|
|
|
-includeNonDefData: if set, include all files listed in fileTab.txt.
|
|
|
Default is to reject non-default files, and modify fileTab.txt to remove
|
|
|
non-default listings.
|
|
|
|
|
|
-onlyData: if set, reject all files in a directory without a
|
|
|
fileTab.txt. Default is to accept all files in a directory not
|
|
|
mentioned in fileTab.txt.
|
|
|
|
|
|
-quiet: if set, script will run silently. Default is to list
|
|
|
all files being tar'ed.
|
|
|
|
|
|
-filetype: format to save data files as. Default is to leave
|
|
|
present format unchanged. <type> is an integer as follows:
|
|
|
|
|
|
type = 0 Leave present format unchanged (default)
|
|
|
|
|
|
type = 1 Madrigal
|
|
|
|
|
|
type = 2 Blocked Binary
|
|
|
|
|
|
type = 3 Cbf
|
|
|
|
|
|
type = 4 Unblocked binary
|
|
|
|
|
|
type = 5 Ascii
|
|
|
|
|
|
-f <outputFile>: name of output compressed tar file (required)
|
|
|
Extention .tar.gz will be automatically appended if not included.
|
|
|
|
|
|
So the default behavior is to tar every file in madroot/experiments
|
|
|
with a conventional directory path except non-default data files, and
|
|
|
modify fileTab.txt files accordingly. The output file will be saved with
|
|
|
a .tar.gz extension.
|
|
|
"""
|
|
|
|
|
|
print(message)
|
|
|
|
|
|
# catch any exception, and write an appropriate message admin
|
|
|
try:
|
|
|
|
|
|
# check if pythonlibpath env variable exists
|
|
|
# written 'PYTHON' + 'LIBPATH' to stop automatic replacement during setup
|
|
|
temp = os.environ.get('PYTHON' + 'LIBPATH')
|
|
|
if temp != None:
|
|
|
sys.path.append(temp)
|
|
|
|
|
|
# append path madroot/lib (needed only if python not installed by setup)
|
|
|
sys.path.append('MADROOT/lib/python')
|
|
|
|
|
|
# prepare to handle MadrigalError
|
|
|
import madrigal.admin
|
|
|
|
|
|
except ImportError:
|
|
|
|
|
|
# Fatal error - madpy library not found
|
|
|
print("Unable to import the madrigal python library - please alert the sys admin!")
|
|
|
sys.exit(0)
|
|
|
|
|
|
# try to run script, and report all errors to Madrigal sys admin
|
|
|
|
|
|
try:
|
|
|
|
|
|
import madrigal.metadata
|
|
|
|
|
|
# create MadrigalDB obj
|
|
|
madDBObj = madrigal.metadata.MadrigalDB()
|
|
|
|
|
|
# if madroot not set, set it now
|
|
|
if os.environ.get('MAD' + 'ROOT') == None:
|
|
|
os.environ['MAD' + 'ROOT'] = madDBObj.getMadroot()
|
|
|
|
|
|
# set default values of arguments
|
|
|
|
|
|
start = None
|
|
|
end = None
|
|
|
excludePrivData = 0
|
|
|
ignoreDirCon = 1
|
|
|
includeNonDefData = 0
|
|
|
onlyData = 0
|
|
|
quiet = 0
|
|
|
filetype = 0
|
|
|
tarfile = ''
|
|
|
|
|
|
# check number, type of arguments
|
|
|
if len(sys.argv) < 2:
|
|
|
printUsage()
|
|
|
sys.exit(0)
|
|
|
|
|
|
# loop through arguments
|
|
|
count = 0
|
|
|
while count < len(sys.argv):
|
|
|
|
|
|
if sys.argv[count] == '-f':
|
|
|
if len(sys.argv) <= count + 1:
|
|
|
print('A tar file name is required after -f')
|
|
|
printUsage()
|
|
|
sys.exit()
|
|
|
else:
|
|
|
tarfile = sys.argv[count + 1]
|
|
|
count = count + 2
|
|
|
continue
|
|
|
|
|
|
elif sys.argv[count] == '-start':
|
|
|
if len(sys.argv) <= count + 1:
|
|
|
print('A date in form YYYYMMDD is required after -start')
|
|
|
sys.exit()
|
|
|
else:
|
|
|
start = sys.argv[count + 1]
|
|
|
if len(start) != 8:
|
|
|
print('A date in form YYYYMMDD is required after -start')
|
|
|
sys.exit()
|
|
|
try:
|
|
|
int(start)
|
|
|
except:
|
|
|
print('A date in form YYYYMMDD is required after -start')
|
|
|
sys.exit()
|
|
|
if int(start[4:6]) > 12:
|
|
|
print('YYYYMMDD after -start has illegal month')
|
|
|
sys.exit()
|
|
|
if int(start[6:8]) > 31:
|
|
|
print('YYYYMMDD after -start has illegal day')
|
|
|
sys.exit()
|
|
|
count = count + 2
|
|
|
continue
|
|
|
|
|
|
elif sys.argv[count] == '-end':
|
|
|
if len(sys.argv) <= count + 1:
|
|
|
print('A date in form YYYYMMDD is required after -end')
|
|
|
sys.exit()
|
|
|
else:
|
|
|
end = sys.argv[count + 1]
|
|
|
if len(end) != 8:
|
|
|
print('A date in form YYYYMMDD is required after -end')
|
|
|
sys.exit()
|
|
|
try:
|
|
|
int(end)
|
|
|
except:
|
|
|
print('A date in form YYYYMMDD is required after -end')
|
|
|
sys.exit()
|
|
|
if int(end[4:6]) > 12:
|
|
|
print('YYYYMMDD after -end has illegal month')
|
|
|
sys.exit()
|
|
|
if int(end[6:8]) > 31:
|
|
|
print('YYYYMMDD after -end has illegal day')
|
|
|
sys.exit()
|
|
|
count = count + 2
|
|
|
continue
|
|
|
|
|
|
elif sys.argv[count] == '-excludePrivData':
|
|
|
excludePrivData = 1
|
|
|
count = count + 1
|
|
|
|
|
|
elif sys.argv[count] == '-ignoreDirCon':
|
|
|
ignoreDirCon = 1
|
|
|
count = count + 1
|
|
|
|
|
|
elif sys.argv[count] == '-includeNonDefData':
|
|
|
includeNonDefData = 1
|
|
|
count = count + 1
|
|
|
|
|
|
elif sys.argv[count] == '-onlyData':
|
|
|
onlyData = 1
|
|
|
count = count + 1
|
|
|
|
|
|
elif sys.argv[count] == '-quiet':
|
|
|
quiet = 1
|
|
|
count = count + 1
|
|
|
|
|
|
elif sys.argv[count] == '-filetype':
|
|
|
if len(sys.argv) <= count + 1:
|
|
|
print('A filetype between 0 and 5 must be given after -filetype')
|
|
|
sys.exit()
|
|
|
else:
|
|
|
try:
|
|
|
filetype = int(sys.argv[count + 1])
|
|
|
except:
|
|
|
print('A filetype between 0 and 5 must be given after -filetype')
|
|
|
sys.exit()
|
|
|
if filetype < 0 or filetype > 5:
|
|
|
print('A filetype between 0 and 5 must be given after -filetype')
|
|
|
sys.exit()
|
|
|
count = count + 2
|
|
|
continue
|
|
|
|
|
|
elif sys.argv[count][0] == '-':
|
|
|
print('Unknown argument: ' + sys.argv[count])
|
|
|
print()
|
|
|
printUsage()
|
|
|
sys.exit()
|
|
|
|
|
|
else:
|
|
|
# ignore argument
|
|
|
count = count + 1
|
|
|
|
|
|
# check tarfile at least is set
|
|
|
if len(tarfile) == 0:
|
|
|
print('Required argument -f <tarfile> missing')
|
|
|
printUsage()
|
|
|
sys.exit()
|
|
|
|
|
|
# check that tarfile does not bein with a -
|
|
|
if tarfile[0] == '-':
|
|
|
print('Name of tarfile must follow -f option')
|
|
|
printUsage()
|
|
|
sys.exit()
|
|
|
|
|
|
# check tarfile does not have .gz extension
|
|
|
if len(tarfile) > 1:
|
|
|
if tarfile[-3:] == '.gz':
|
|
|
tarfile = tarfile[:-3]
|
|
|
|
|
|
# now check tarfile has .tar extention
|
|
|
if len(tarfile) > 3:
|
|
|
if tarfile[-4:] != '.tar':
|
|
|
tarfile = tarfile + '.tar'
|
|
|
|
|
|
# create start and end times if needed
|
|
|
if start != None:
|
|
|
start = (int(start[0:4]),
|
|
|
int(start[4:6]),
|
|
|
int(start[6:8]),
|
|
|
0, 0, 0, 0, 0, 0)
|
|
|
|
|
|
if end != None:
|
|
|
end = (int(end[0:4]),
|
|
|
int(end[4:6]),
|
|
|
int(end[6:8]),
|
|
|
0, 0, 0, 0, 0, 0)
|
|
|
|
|
|
if quiet == 1:
|
|
|
verbose = 0
|
|
|
else:
|
|
|
verbose = 1
|
|
|
|
|
|
# create tar file
|
|
|
madDBObj.tarExperiments(tarfile,
|
|
|
start,
|
|
|
end,
|
|
|
excludePrivData,
|
|
|
ignoreDirCon,
|
|
|
includeNonDefData,
|
|
|
onlyData,
|
|
|
filetype,
|
|
|
verbose)
|
|
|
|
|
|
if verbose == 1:
|
|
|
print('Running gzip on tarfile...')
|
|
|
|
|
|
# gzip tar file
|
|
|
os.system('gzip ' + tarfile)
|
|
|
|
|
|
|
|
|
except madrigal.admin.MadrigalError as e:
|
|
|
# handle a MadrigalError
|
|
|
|
|
|
|
|
|
errStr = 'Error occurred in tarExperiments.py\n'
|
|
|
|
|
|
errStr = errStr + e.getExceptionHtml()
|
|
|
|
|
|
err = traceback.format_exception(sys.exc_info()[0],
|
|
|
sys.exc_info()[1],
|
|
|
sys.exc_info()[2])
|
|
|
|
|
|
for errItem in err:
|
|
|
errStr = errStr + '\n' + str(errItem)
|
|
|
|
|
|
|
|
|
print(errStr)
|
|
|
|
|
|
|
|
|
except:
|
|
|
# handle a normal error
|
|
|
|
|
|
# if a normal SystemExit, simply terminate
|
|
|
if str(sys.exc_info()[0]).find('exceptions.SystemExit') != -1:
|
|
|
sys.exit(0)
|
|
|
|
|
|
|
|
|
errStr = 'Error occurred in running tarExperiments.py.\n'
|
|
|
|
|
|
|
|
|
err = traceback.format_exception(sys.exc_info()[0],
|
|
|
sys.exc_info()[1],
|
|
|
sys.exc_info()[2])
|
|
|
|
|
|
for errItem in err:
|
|
|
errStr = errStr + '\n' + str(errItem)
|
|
|
|
|
|
|
|
|
print(errStr)
|
|
|
|
|
|
|
|
|
# end script
|
|
|
|
|
|
|
|
|
|
