##// END OF EJS Templates
schain-jc
RSS
Bug fixed in jroIO_base: date and time filters were not working when walk=0 and there were many days inside one directory.
Miguel Valdez -
r640:4d492e909122
parent child
Show More
Show file before | Show file after | Hide comments
@@ -42,7 +42,7 def isNumber(cad):
42 except:
42 except:
43 return False
43 return False
44
44
45 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 """
46 """
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48
48
@@ -83,7 +83,7 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
83
83
84 return 1
84 return 1
85
85
86 def isFileinThisTime(filename, startTime, endTime):
86 def isFileInTimeRange(filename, startTime, endTime):
87 """
87 """
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89
89
@@ -127,6 +127,86 def isFileinThisTime(filename, startTime, endTime):
127
127
128 return thisDatetime
128 return thisDatetime
129
129
130 def isFolderInDateRange(folder, startDate=None, endDate=None):
131 """
132 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
133
134 Inputs:
135 folder : nombre completo del directorio.
136 Su formato deberia ser "/path_root/?YYYYDDD"
137
138 siendo:
139 YYYY : Anio (ejemplo 2015)
140 DDD : Dia del anio (ejemplo 305)
141
142 startDate : fecha inicial del rango seleccionado en formato datetime.date
143
144 endDate : fecha final del rango seleccionado en formato datetime.date
145
146 Return:
147 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
148 fecha especificado, de lo contrario retorna False.
149 Excepciones:
150 Si el directorio no tiene el formato adecuado
151 """
152
153 basename = os.path.basename(folder)
154
155 if not isRadarFolder(basename):
156 raise IOError, "The folder %s has not the rigth format" %folder
157
158 if startDate and endDate:
159 thisDate = getDateFromRadarFolder(basename)
160
161 if thisDate < startDate:
162 return 0
163
164 if thisDate > endDate:
165 return 0
166
167 return 1
168
169 def isFileInDateRange(filename, startDate=None, endDate=None):
170 """
171 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
172
173 Inputs:
174 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
175
176 Su formato deberia ser "?YYYYDDDsss"
177
178 siendo:
179 YYYY : Anio (ejemplo 2015)
180 DDD : Dia del anio (ejemplo 305)
181 sss : set
182
183 startDate : fecha inicial del rango seleccionado en formato datetime.date
184
185 endDate : fecha final del rango seleccionado en formato datetime.date
186
187 Return:
188 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
189 fecha especificado, de lo contrario retorna False.
190 Excepciones:
191 Si el archivo no tiene el formato adecuado
192 """
193
194 basename = os.path.basename(filename)
195
196 if not isRadarFile(basename):
197 raise IOError, "The filename %s has not the rigth format" %filename
198
199 if startDate and endDate:
200 thisDate = getDateFromRadarFile(basename)
201
202 if thisDate < startDate:
203 return 0
204
205 if thisDate > endDate:
206 return 0
207
208 return 1
209
130 def getFileFromSet(path, ext, set):
210 def getFileFromSet(path, ext, set):
131 validFilelist = []
211 validFilelist = []
132 fileList = os.listdir(path)
212 fileList = os.listdir(path)
@@ -282,16 +362,26 def isRadarFile(file):
282 return 1
362 return 1
283
363
284 def getDateFromRadarFile(file):
364 def getDateFromRadarFile(file):
285 try:
365 try:
286 year = int(file[1:5])
366 year = int(file[1:5])
287 doy = int(file[5:8])
367 doy = int(file[5:8])
288 set = int(file[8:11])
368 set = int(file[8:11])
289 except:
369 except:
290 return None
370 return None
291
371
292 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
372 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
293 return thisDate
373 return thisDate
294
374
375 def getDateFromRadarFolder(folder):
376 try:
377 year = int(folder[1:5])
378 doy = int(folder[5:8])
379 except:
380 return None
381
382 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
383 return thisDate
384
295 class JRODataIO:
385 class JRODataIO:
296
386
297 c = 3E8
387 c = 3E8
@@ -436,82 +526,87 class JRODataReader(JRODataIO):
436
526
437 pathList = []
527 pathList = []
438
528
439 if not walk:
529 # if not walk:
440 #pathList.append(path)
530 # #pathList.append(path)
441 multi_path = path.split(',')
531 # multi_path = path.split(',')
442 for single_path in multi_path:
532 # for single_path in multi_path:
443
533 #
444 if not os.path.isdir(single_path):
534 # if not os.path.isdir(single_path):
445 continue
535 # continue
446
536 #
447 pathList.append(single_path)
537 # pathList.append(single_path)
448
538 #
449 else:
539 # else:
450 #dirList = []
540 # #dirList = []
451 multi_path = path.split(',')
541 # multi_path = path.split(',')
452 for single_path in multi_path:
542 # for single_path in multi_path:
453
543 #
454 if not os.path.isdir(single_path):
544 # if not os.path.isdir(single_path):
455 continue
545 # continue
456
546 #
457 dirList = []
547 # dirList = []
458 for thisPath in os.listdir(single_path):
548 # for thisPath in os.listdir(single_path):
459 if not os.path.isdir(os.path.join(single_path,thisPath)):
549 # if not os.path.isdir(os.path.join(single_path,thisPath)):
460 continue
550 # continue
461 if not isRadarFolder(thisPath):
551 # if not isRadarFolder(thisPath):
462 continue
552 # continue
463
553 #
464 dirList.append(thisPath)
554 # dirList.append(thisPath)
465
555 #
466 if not(dirList):
556 # if not(dirList):
467 return None, None
557 # return None, None
468
558 #
469 if startDate and endDate:
559 # if startDate and endDate:
470 thisDate = startDate
560 # thisDate = startDate
471
561 #
472 while(thisDate <= endDate):
562 # while(thisDate <= endDate):
473 year = thisDate.timetuple().tm_year
563 # year = thisDate.timetuple().tm_year
474 doy = thisDate.timetuple().tm_yday
564 # doy = thisDate.timetuple().tm_yday
475
565 #
476 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
566 # matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
477 if len(matchlist) == 0:
567 # if len(matchlist) == 0:
478 thisDate += datetime.timedelta(1)
568 # thisDate += datetime.timedelta(1)
479 continue
569 # continue
480 for match in matchlist:
570 # for match in matchlist:
481 pathList.append(os.path.join(single_path,match,expLabel))
571 # pathList.append(os.path.join(single_path,match,expLabel))
482
572 #
483 thisDate += datetime.timedelta(1)
573 # thisDate += datetime.timedelta(1)
484 else:
574 # else:
485 for thiDir in dirList:
575 # for thiDir in dirList:
486 pathList.append(os.path.join(single_path,thiDir,expLabel))
576 # pathList.append(os.path.join(single_path,thiDir,expLabel))
487
577
488 if pathList == []:
578 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
579
580 if dateList == []:
489 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
581 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
490 return None, None
582 return None, None
491
583
492 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
584 if len(dateList) > 1:
493
585 print "%d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
586 else:
587 print "data was found for the date %s" %(dateList[0])
588
494 filenameList = []
589 filenameList = []
495 datetimeList = []
590 datetimeList = []
496 pathDict = {}
591 # pathDict = {}
497 filenameList_to_sort = []
592 # filenameList_to_sort = []
498
593 #
499 for i in range(len(pathList)):
594 # for i in range(len(pathList)):
500
595 #
501 thisPath = pathList[i]
596 # thisPath = pathList[i]
502
597 #
503 fileList = glob.glob1(thisPath, "*%s" %ext)
598 # fileList = glob.glob1(thisPath, "*%s" %ext)
504 if len(fileList) < 1:
599 # if len(fileList) < 1:
505 continue
600 # continue
506 fileList.sort()
601 # fileList.sort()
507 pathDict.setdefault(fileList[0])
602 # pathDict.setdefault(fileList[0])
508 pathDict[fileList[0]] = i
603 # pathDict[fileList[0]] = i
509 filenameList_to_sort.append(fileList[0])
604 # filenameList_to_sort.append(fileList[0])
510
605 #
511 filenameList_to_sort.sort()
606 # filenameList_to_sort.sort()
512
607
513 for file in filenameList_to_sort:
608 for thisPath in pathList:
514 thisPath = pathList[pathDict[file]]
609 # thisPath = pathList[pathDict[file]]
515
610
516 fileList = glob.glob1(thisPath, "*%s" %ext)
611 fileList = glob.glob1(thisPath, "*%s" %ext)
517 fileList.sort()
612 fileList.sort()
@@ -519,7 +614,11 class JRODataReader(JRODataIO):
519 for file in fileList:
614 for file in fileList:
520
615
521 filename = os.path.join(thisPath,file)
616 filename = os.path.join(thisPath,file)
522 thisDatetime = isFileinThisTime(filename, startTime, endTime)
617
618 if not isFileInDateRange(filename, startDate, endDate):
619 continue
620
621 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
523
622
524 if not(thisDatetime):
623 if not(thisDatetime):
525 continue
624 continue
@@ -976,20 +1075,20 class JRODataReader(JRODataIO):
976
1075
977 return True
1076 return True
978
1077
979 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
1078 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
980
1079
981 dateList = []
1080 dateList = []
982 pathList = []
1081 pathList = []
983
1082
1083 multi_path = path.split(',')
1084
984 if not walk:
1085 if not walk:
985 #pathList.append(path)
1086
986 multi_path = path.split(',')
987 for single_path in multi_path:
1087 for single_path in multi_path:
988
1088
989 if not os.path.isdir(single_path):
1089 if not os.path.isdir(single_path):
990 continue
1090 continue
991
1091
992 ok = False
993 fileList = glob.glob1(single_path, "*"+ext)
1092 fileList = glob.glob1(single_path, "*"+ext)
994
1093
995 for thisFile in fileList:
1094 for thisFile in fileList:
@@ -1000,63 +1099,41 class JRODataReader(JRODataIO):
1000 if not isRadarFile(thisFile):
1099 if not isRadarFile(thisFile):
1001 continue
1100 continue
1002
1101
1003 ok = True
1102 if not isFileInDateRange(thisFile, startDate, endDate):
1103 continue
1104
1004 thisDate = getDateFromRadarFile(thisFile)
1105 thisDate = getDateFromRadarFile(thisFile)
1005
1106
1006 if thisDate not in dateList:
1107 if thisDate in dateList:
1007 dateList.append(thisDate)
1108 continue
1008
1109
1009 if ok:
1110 dateList.append(thisDate)
1010 pathList.append(single_path)
1111 pathList.append(single_path)
1011
1012 return dateList
1013
1112
1014 multi_path = path.split(',')
1113 else:
1015 for single_path in multi_path:
1114 for single_path in multi_path:
1016
1017 if not os.path.isdir(single_path):
1018 continue
1019
1020 dirList = []
1021
1022 for thisPath in os.listdir(single_path):
1023
1115
1024 if not os.path.isdir(os.path.join(single_path,thisPath)):
1116 if not os.path.isdir(single_path):
1025 continue
1117 continue
1118
1119 dirList = []
1026
1120
1027 if not isRadarFolder(thisPath):
1121 for thisPath in os.listdir(single_path):
1028 continue
1029
1030 dirList.append(thisPath)
1031
1032 if not dirList:
1033 return dateList
1034
1035 if startDate and endDate:
1036 thisDate = startDate
1037
1038 while(thisDate <= endDate):
1039 year = thisDate.timetuple().tm_year
1040 doy = thisDate.timetuple().tm_yday
1041
1122
1042 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1123 if not os.path.isdir(os.path.join(single_path,thisPath)):
1043 if len(matchlist) == 0:
1044 thisDate += datetime.timedelta(1)
1045 continue
1124 continue
1046
1125
1047 for match in matchlist:
1126 if not isRadarFolder(thisPath):
1048
1127 continue
1049 datapath = os.path.join(single_path, match, expLabel)
1050 fileList = glob.glob1(datapath, "*"+ext)
1051
1052 if len(fileList) < 1:
1053 continue
1054
1055 pathList.append(datapath)
1056 dateList.append(thisDate)
1057
1128
1058 thisDate += datetime.timedelta(1)
1129 if not isFolderInDateRange(thisPath, startDate, endDate):
1059 else:
1130 continue
1131
1132 dirList.append(thisPath)
1133
1134 if not dirList:
1135 continue
1136
1060 for thisDir in dirList:
1137 for thisDir in dirList:
1061
1138
1062 datapath = os.path.join(single_path, thisDir, expLabel)
1139 datapath = os.path.join(single_path, thisDir, expLabel)
@@ -1065,15 +1142,17 class JRODataReader(JRODataIO):
1065 if len(fileList) < 1:
1142 if len(fileList) < 1:
1066 continue
1143 continue
1067
1144
1068 year = int(thisDir[1:5])
1145 thisDate = getDateFromRadarFolder(thisDir)
1069 doy = int(thisDir[5:8])
1070 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1071
1146
1072 pathList.append(datapath)
1147 pathList.append(datapath)
1073 dateList.append(thisDate)
1148 dateList.append(thisDate)
1074
1075 return dateList
1076
1149
1150 dateList.sort()
1151
1152 if include_path:
1153 return dateList, pathList
1154
1155 return dateList
1077
1156
1078 def setup(self,
1157 def setup(self,
1079 path=None,
1158 path=None,
General Comments 0
You need to be logged in to leave comments. Login now