tarExperiments
330 lines
| 9.7 KiB
| text/plain
|
TextLexer
| r0 | #!PYTHONEXE | |||
| import sys | ||||
| import os | ||||
| import traceback | ||||
| def printUsage(): | ||||
| message = """ | ||||
| Usage: tarExperiments [-start <startdate as YYYYMMDD>] | ||||
| [-end <enddate as YYYYMMDD>] | ||||
| [-excludePrivData] | ||||
| [-ignoreDirCon] | ||||
| [-includeNonDefData] | ||||
| [-onlyData] | ||||
| [-quiet] | ||||
| [-filetype <type>] | ||||
| -f <outputFile> | ||||
| where: | ||||
| -start: set a start date - any folder and all subfolders with an | ||||
| expTab.txt before YYYYMMDD is excluded. | ||||
| -end: set a end date - any folder and all subfolders with an expTab.txt | ||||
| after YYYYMMDD is excluded. | ||||
| -excludePrivData: if set, omit data marked as private (and the line | ||||
| from the fileTab.txt to be removed). Default is to use all data, public and | ||||
| private. | ||||
| -ignoreDirCon: Has no effect, since directory convention no longer | ||||
| built into Madrigal. Retained only for backward compatability. | ||||
| -includeNonDefData: if set, include all files listed in fileTab.txt. | ||||
| Default is to reject non-default files, and modify fileTab.txt to remove | ||||
| non-default listings. | ||||
| -onlyData: if set, reject all files in a directory without a | ||||
| fileTab.txt. Default is to accept all files in a directory not | ||||
| mentioned in fileTab.txt. | ||||
| -quiet: if set, script will run silently. Default is to list | ||||
| all files being tar'ed. | ||||
| -filetype: format to save data files as. Default is to leave | ||||
| present format unchanged. <type> is an integer as follows: | ||||
| type = 0 Leave present format unchanged (default) | ||||
| type = 1 Madrigal | ||||
| type = 2 Blocked Binary | ||||
| type = 3 Cbf | ||||
| type = 4 Unblocked binary | ||||
| type = 5 Ascii | ||||
| -f <outputFile>: name of output compressed tar file (required) | ||||
| Extention .tar.gz will be automatically appended if not included. | ||||
| So the default behavior is to tar every file in madroot/experiments | ||||
| with a conventional directory path except non-default data files, and | ||||
| modify fileTab.txt files accordingly. The output file will be saved with | ||||
| a .tar.gz extension. | ||||
| """ | ||||
| print(message) | ||||
| # catch any exception, and write an appropriate message admin | ||||
| try: | ||||
| # check if pythonlibpath env variable exists | ||||
| # written 'PYTHON' + 'LIBPATH' to stop automatic replacement during setup | ||||
| temp = os.environ.get('PYTHON' + 'LIBPATH') | ||||
| if temp != None: | ||||
| sys.path.append(temp) | ||||
| # append path madroot/lib (needed only if python not installed by setup) | ||||
| sys.path.append('MADROOT/lib/python') | ||||
| # prepare to handle MadrigalError | ||||
| import madrigal.admin | ||||
| except ImportError: | ||||
| # Fatal error - madpy library not found | ||||
| print("Unable to import the madrigal python library - please alert the sys admin!") | ||||
| sys.exit(0) | ||||
| # try to run script, and report all errors to Madrigal sys admin | ||||
| try: | ||||
| import madrigal.metadata | ||||
| # create MadrigalDB obj | ||||
| madDBObj = madrigal.metadata.MadrigalDB() | ||||
| # if madroot not set, set it now | ||||
| if os.environ.get('MAD' + 'ROOT') == None: | ||||
| os.environ['MAD' + 'ROOT'] = madDBObj.getMadroot() | ||||
| # set default values of arguments | ||||
| start = None | ||||
| end = None | ||||
| excludePrivData = 0 | ||||
| ignoreDirCon = 1 | ||||
| includeNonDefData = 0 | ||||
| onlyData = 0 | ||||
| quiet = 0 | ||||
| filetype = 0 | ||||
| tarfile = '' | ||||
| # check number, type of arguments | ||||
| if len(sys.argv) < 2: | ||||
| printUsage() | ||||
| sys.exit(0) | ||||
| # loop through arguments | ||||
| count = 0 | ||||
| while count < len(sys.argv): | ||||
| if sys.argv[count] == '-f': | ||||
| if len(sys.argv) <= count + 1: | ||||
| print('A tar file name is required after -f') | ||||
| printUsage() | ||||
| sys.exit() | ||||
| else: | ||||
| tarfile = sys.argv[count + 1] | ||||
| count = count + 2 | ||||
| continue | ||||
| elif sys.argv[count] == '-start': | ||||
| if len(sys.argv) <= count + 1: | ||||
| print('A date in form YYYYMMDD is required after -start') | ||||
| sys.exit() | ||||
| else: | ||||
| start = sys.argv[count + 1] | ||||
| if len(start) != 8: | ||||
| print('A date in form YYYYMMDD is required after -start') | ||||
| sys.exit() | ||||
| try: | ||||
| int(start) | ||||
| except: | ||||
| print('A date in form YYYYMMDD is required after -start') | ||||
| sys.exit() | ||||
| if int(start[4:6]) > 12: | ||||
| print('YYYYMMDD after -start has illegal month') | ||||
| sys.exit() | ||||
| if int(start[6:8]) > 31: | ||||
| print('YYYYMMDD after -start has illegal day') | ||||
| sys.exit() | ||||
| count = count + 2 | ||||
| continue | ||||
| elif sys.argv[count] == '-end': | ||||
| if len(sys.argv) <= count + 1: | ||||
| print('A date in form YYYYMMDD is required after -end') | ||||
| sys.exit() | ||||
| else: | ||||
| end = sys.argv[count + 1] | ||||
| if len(end) != 8: | ||||
| print('A date in form YYYYMMDD is required after -end') | ||||
| sys.exit() | ||||
| try: | ||||
| int(end) | ||||
| except: | ||||
| print('A date in form YYYYMMDD is required after -end') | ||||
| sys.exit() | ||||
| if int(end[4:6]) > 12: | ||||
| print('YYYYMMDD after -end has illegal month') | ||||
| sys.exit() | ||||
| if int(end[6:8]) > 31: | ||||
| print('YYYYMMDD after -end has illegal day') | ||||
| sys.exit() | ||||
| count = count + 2 | ||||
| continue | ||||
| elif sys.argv[count] == '-excludePrivData': | ||||
| excludePrivData = 1 | ||||
| count = count + 1 | ||||
| elif sys.argv[count] == '-ignoreDirCon': | ||||
| ignoreDirCon = 1 | ||||
| count = count + 1 | ||||
| elif sys.argv[count] == '-includeNonDefData': | ||||
| includeNonDefData = 1 | ||||
| count = count + 1 | ||||
| elif sys.argv[count] == '-onlyData': | ||||
| onlyData = 1 | ||||
| count = count + 1 | ||||
| elif sys.argv[count] == '-quiet': | ||||
| quiet = 1 | ||||
| count = count + 1 | ||||
| elif sys.argv[count] == '-filetype': | ||||
| if len(sys.argv) <= count + 1: | ||||
| print('A filetype between 0 and 5 must be given after -filetype') | ||||
| sys.exit() | ||||
| else: | ||||
| try: | ||||
| filetype = int(sys.argv[count + 1]) | ||||
| except: | ||||
| print('A filetype between 0 and 5 must be given after -filetype') | ||||
| sys.exit() | ||||
| if filetype < 0 or filetype > 5: | ||||
| print('A filetype between 0 and 5 must be given after -filetype') | ||||
| sys.exit() | ||||
| count = count + 2 | ||||
| continue | ||||
| elif sys.argv[count][0] == '-': | ||||
| print('Unknown argument: ' + sys.argv[count]) | ||||
| print() | ||||
| printUsage() | ||||
| sys.exit() | ||||
| else: | ||||
| # ignore argument | ||||
| count = count + 1 | ||||
| # check tarfile at least is set | ||||
| if len(tarfile) == 0: | ||||
| print('Required argument -f <tarfile> missing') | ||||
| printUsage() | ||||
| sys.exit() | ||||
| # check that tarfile does not bein with a - | ||||
| if tarfile[0] == '-': | ||||
| print('Name of tarfile must follow -f option') | ||||
| printUsage() | ||||
| sys.exit() | ||||
| # check tarfile does not have .gz extension | ||||
| if len(tarfile) > 1: | ||||
| if tarfile[-3:] == '.gz': | ||||
| tarfile = tarfile[:-3] | ||||
| # now check tarfile has .tar extention | ||||
| if len(tarfile) > 3: | ||||
| if tarfile[-4:] != '.tar': | ||||
| tarfile = tarfile + '.tar' | ||||
| # create start and end times if needed | ||||
| if start != None: | ||||
| start = (int(start[0:4]), | ||||
| int(start[4:6]), | ||||
| int(start[6:8]), | ||||
| 0, 0, 0, 0, 0, 0) | ||||
| if end != None: | ||||
| end = (int(end[0:4]), | ||||
| int(end[4:6]), | ||||
| int(end[6:8]), | ||||
| 0, 0, 0, 0, 0, 0) | ||||
| if quiet == 1: | ||||
| verbose = 0 | ||||
| else: | ||||
| verbose = 1 | ||||
| # create tar file | ||||
| madDBObj.tarExperiments(tarfile, | ||||
| start, | ||||
| end, | ||||
| excludePrivData, | ||||
| ignoreDirCon, | ||||
| includeNonDefData, | ||||
| onlyData, | ||||
| filetype, | ||||
| verbose) | ||||
| if verbose == 1: | ||||
| print('Running gzip on tarfile...') | ||||
| # gzip tar file | ||||
| os.system('gzip ' + tarfile) | ||||
| except madrigal.admin.MadrigalError as e: | ||||
| # handle a MadrigalError | ||||
| errStr = 'Error occurred in tarExperiments.py\n' | ||||
| errStr = errStr + e.getExceptionHtml() | ||||
| err = traceback.format_exception(sys.exc_info()[0], | ||||
| sys.exc_info()[1], | ||||
| sys.exc_info()[2]) | ||||
| for errItem in err: | ||||
| errStr = errStr + '\n' + str(errItem) | ||||
| print(errStr) | ||||
| except: | ||||
| # handle a normal error | ||||
| # if a normal SystemExit, simply terminate | ||||
| if str(sys.exc_info()[0]).find('exceptions.SystemExit') != -1: | ||||
| sys.exit(0) | ||||
| errStr = 'Error occurred in running tarExperiments.py.\n' | ||||
| err = traceback.format_exception(sys.exc_info()[0], | ||||
| sys.exc_info()[1], | ||||
| sys.exc_info()[2]) | ||||
| for errItem in err: | ||||
| errStr = errStr + '\n' + str(errItem) | ||||
| print(errStr) | ||||
| # end script | ||||
