##// END OF EJS Templates
schain-jc
RSS
Bug fixed in jroIO_base: date and time filters were not working when walk=0 and there were many days inside one directory.
Miguel Valdez -
r640:4d492e909122
parent child
Show More
Show file before | Show file after | Hide comments
@@ -42,7 +42,7 def isNumber(cad):
42 42 except:
43 43 return False
44 44
45 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
@@ -83,7 +83,7 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
83 83
84 84 return 1
85 85
86 def isFileinThisTime(filename, startTime, endTime):
86 def isFileInTimeRange(filename, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
@@ -127,6 +127,86 def isFileinThisTime(filename, startTime, endTime):
127 127
128 128 return thisDatetime
129 129
130 def isFolderInDateRange(folder, startDate=None, endDate=None):
131 """
132 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
133
134 Inputs:
135 folder : nombre completo del directorio.
136 Su formato deberia ser "/path_root/?YYYYDDD"
137
138 siendo:
139 YYYY : Anio (ejemplo 2015)
140 DDD : Dia del anio (ejemplo 305)
141
142 startDate : fecha inicial del rango seleccionado en formato datetime.date
143
144 endDate : fecha final del rango seleccionado en formato datetime.date
145
146 Return:
147 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
148 fecha especificado, de lo contrario retorna False.
149 Excepciones:
150 Si el directorio no tiene el formato adecuado
151 """
152
153 basename = os.path.basename(folder)
154
155 if not isRadarFolder(basename):
156 raise IOError, "The folder %s has not the rigth format" %folder
157
158 if startDate and endDate:
159 thisDate = getDateFromRadarFolder(basename)
160
161 if thisDate < startDate:
162 return 0
163
164 if thisDate > endDate:
165 return 0
166
167 return 1
168
169 def isFileInDateRange(filename, startDate=None, endDate=None):
170 """
171 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
172
173 Inputs:
174 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
175
176 Su formato deberia ser "?YYYYDDDsss"
177
178 siendo:
179 YYYY : Anio (ejemplo 2015)
180 DDD : Dia del anio (ejemplo 305)
181 sss : set
182
183 startDate : fecha inicial del rango seleccionado en formato datetime.date
184
185 endDate : fecha final del rango seleccionado en formato datetime.date
186
187 Return:
188 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
189 fecha especificado, de lo contrario retorna False.
190 Excepciones:
191 Si el archivo no tiene el formato adecuado
192 """
193
194 basename = os.path.basename(filename)
195
196 if not isRadarFile(basename):
197 raise IOError, "The filename %s has not the rigth format" %filename
198
199 if startDate and endDate:
200 thisDate = getDateFromRadarFile(basename)
201
202 if thisDate < startDate:
203 return 0
204
205 if thisDate > endDate:
206 return 0
207
208 return 1
209
130 210 def getFileFromSet(path, ext, set):
131 211 validFilelist = []
132 212 fileList = os.listdir(path)
@@ -282,16 +362,26 def isRadarFile(file):
282 362 return 1
283 363
284 364 def getDateFromRadarFile(file):
285 try:
286 year = int(file[1:5])
287 doy = int(file[5:8])
288 set = int(file[8:11])
289 except:
290 return None
291
292 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
293 return thisDate
294
365 try:
366 year = int(file[1:5])
367 doy = int(file[5:8])
368 set = int(file[8:11])
369 except:
370 return None
371
372 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
373 return thisDate
374
375 def getDateFromRadarFolder(folder):
376 try:
377 year = int(folder[1:5])
378 doy = int(folder[5:8])
379 except:
380 return None
381
382 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
383 return thisDate
384
295 385 class JRODataIO:
296 386
297 387 c = 3E8
@@ -436,82 +526,87 class JRODataReader(JRODataIO):
436 526
437 527 pathList = []
438 528
439 if not walk:
440 #pathList.append(path)
441 multi_path = path.split(',')
442 for single_path in multi_path:
443
444 if not os.path.isdir(single_path):
445 continue
446
447 pathList.append(single_path)
448
449 else:
450 #dirList = []
451 multi_path = path.split(',')
452 for single_path in multi_path:
453
454 if not os.path.isdir(single_path):
455 continue
456
457 dirList = []
458 for thisPath in os.listdir(single_path):
459 if not os.path.isdir(os.path.join(single_path,thisPath)):
460 continue
461 if not isRadarFolder(thisPath):
462 continue
463
464 dirList.append(thisPath)
465
466 if not(dirList):
467 return None, None
468
469 if startDate and endDate:
470 thisDate = startDate
471
472 while(thisDate <= endDate):
473 year = thisDate.timetuple().tm_year
474 doy = thisDate.timetuple().tm_yday
475
476 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
477 if len(matchlist) == 0:
478 thisDate += datetime.timedelta(1)
479 continue
480 for match in matchlist:
481 pathList.append(os.path.join(single_path,match,expLabel))
482
483 thisDate += datetime.timedelta(1)
484 else:
485 for thiDir in dirList:
486 pathList.append(os.path.join(single_path,thiDir,expLabel))
487
488 if pathList == []:
529 # if not walk:
530 # #pathList.append(path)
531 # multi_path = path.split(',')
532 # for single_path in multi_path:
533 #
534 # if not os.path.isdir(single_path):
535 # continue
536 #
537 # pathList.append(single_path)
538 #
539 # else:
540 # #dirList = []
541 # multi_path = path.split(',')
542 # for single_path in multi_path:
543 #
544 # if not os.path.isdir(single_path):
545 # continue
546 #
547 # dirList = []
548 # for thisPath in os.listdir(single_path):
549 # if not os.path.isdir(os.path.join(single_path,thisPath)):
550 # continue
551 # if not isRadarFolder(thisPath):
552 # continue
553 #
554 # dirList.append(thisPath)
555 #
556 # if not(dirList):
557 # return None, None
558 #
559 # if startDate and endDate:
560 # thisDate = startDate
561 #
562 # while(thisDate <= endDate):
563 # year = thisDate.timetuple().tm_year
564 # doy = thisDate.timetuple().tm_yday
565 #
566 # matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
567 # if len(matchlist) == 0:
568 # thisDate += datetime.timedelta(1)
569 # continue
570 # for match in matchlist:
571 # pathList.append(os.path.join(single_path,match,expLabel))
572 #
573 # thisDate += datetime.timedelta(1)
574 # else:
575 # for thiDir in dirList:
576 # pathList.append(os.path.join(single_path,thiDir,expLabel))
577
578 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
579
580 if dateList == []:
489 581 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
490 582 return None, None
491 583
492 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
493
584 if len(dateList) > 1:
585 print "%d dates with data were found for the date range: %s - %s" %(len(dateList), startDate, endDate)
586 else:
587 print "data was found for the date %s" %(dateList[0])
588
494 589 filenameList = []
495 590 datetimeList = []
496 pathDict = {}
497 filenameList_to_sort = []
498
499 for i in range(len(pathList)):
500
501 thisPath = pathList[i]
502
503 fileList = glob.glob1(thisPath, "*%s" %ext)
504 if len(fileList) < 1:
505 continue
506 fileList.sort()
507 pathDict.setdefault(fileList[0])
508 pathDict[fileList[0]] = i
509 filenameList_to_sort.append(fileList[0])
510
511 filenameList_to_sort.sort()
512
513 for file in filenameList_to_sort:
514 thisPath = pathList[pathDict[file]]
591 # pathDict = {}
592 # filenameList_to_sort = []
593 #
594 # for i in range(len(pathList)):
595 #
596 # thisPath = pathList[i]
597 #
598 # fileList = glob.glob1(thisPath, "*%s" %ext)
599 # if len(fileList) < 1:
600 # continue
601 # fileList.sort()
602 # pathDict.setdefault(fileList[0])
603 # pathDict[fileList[0]] = i
604 # filenameList_to_sort.append(fileList[0])
605 #
606 # filenameList_to_sort.sort()
607
608 for thisPath in pathList:
609 # thisPath = pathList[pathDict[file]]
515 610
516 611 fileList = glob.glob1(thisPath, "*%s" %ext)
517 612 fileList.sort()
@@ -519,7 +614,11 class JRODataReader(JRODataIO):
519 614 for file in fileList:
520 615
521 616 filename = os.path.join(thisPath,file)
522 thisDatetime = isFileinThisTime(filename, startTime, endTime)
617
618 if not isFileInDateRange(filename, startDate, endDate):
619 continue
620
621 thisDatetime = isFileInTimeRange(filename, startTime, endTime)
523 622
524 623 if not(thisDatetime):
525 624 continue
@@ -976,20 +1075,20 class JRODataReader(JRODataIO):
976 1075
977 1076 return True
978 1077
979 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
1078 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
980 1079
981 1080 dateList = []
982 1081 pathList = []
983 1082
1083 multi_path = path.split(',')
1084
984 1085 if not walk:
985 #pathList.append(path)
986 multi_path = path.split(',')
1086
987 1087 for single_path in multi_path:
988 1088
989 1089 if not os.path.isdir(single_path):
990 1090 continue
991 1091
992 ok = False
993 1092 fileList = glob.glob1(single_path, "*"+ext)
994 1093
995 1094 for thisFile in fileList:
@@ -1000,63 +1099,41 class JRODataReader(JRODataIO):
1000 1099 if not isRadarFile(thisFile):
1001 1100 continue
1002 1101
1003 ok = True
1102 if not isFileInDateRange(thisFile, startDate, endDate):
1103 continue
1104
1004 1105 thisDate = getDateFromRadarFile(thisFile)
1005 1106
1006 if thisDate not in dateList:
1007 dateList.append(thisDate)
1008
1009 if ok:
1107 if thisDate in dateList:
1108 continue
1109
1110 dateList.append(thisDate)
1010 1111 pathList.append(single_path)
1011
1012 return dateList
1013 1112
1014 multi_path = path.split(',')
1015 for single_path in multi_path:
1016
1017 if not os.path.isdir(single_path):
1018 continue
1019
1020 dirList = []
1021
1022 for thisPath in os.listdir(single_path):
1113 else:
1114 for single_path in multi_path:
1023 1115
1024 if not os.path.isdir(os.path.join(single_path,thisPath)):
1116 if not os.path.isdir(single_path):
1025 1117 continue
1118
1119 dirList = []
1026 1120
1027 if not isRadarFolder(thisPath):
1028 continue
1029
1030 dirList.append(thisPath)
1031
1032 if not dirList:
1033 return dateList
1034
1035 if startDate and endDate:
1036 thisDate = startDate
1037
1038 while(thisDate <= endDate):
1039 year = thisDate.timetuple().tm_year
1040 doy = thisDate.timetuple().tm_yday
1121 for thisPath in os.listdir(single_path):
1041 1122
1042 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1043 if len(matchlist) == 0:
1044 thisDate += datetime.timedelta(1)
1123 if not os.path.isdir(os.path.join(single_path,thisPath)):
1045 1124 continue
1046 1125
1047 for match in matchlist:
1048
1049 datapath = os.path.join(single_path, match, expLabel)
1050 fileList = glob.glob1(datapath, "*"+ext)
1051
1052 if len(fileList) < 1:
1053 continue
1054
1055 pathList.append(datapath)
1056 dateList.append(thisDate)
1126 if not isRadarFolder(thisPath):
1127 continue
1057 1128
1058 thisDate += datetime.timedelta(1)
1059 else:
1129 if not isFolderInDateRange(thisPath, startDate, endDate):
1130 continue
1131
1132 dirList.append(thisPath)
1133
1134 if not dirList:
1135 continue
1136
1060 1137 for thisDir in dirList:
1061 1138
1062 1139 datapath = os.path.join(single_path, thisDir, expLabel)
@@ -1065,15 +1142,17 class JRODataReader(JRODataIO):
1065 1142 if len(fileList) < 1:
1066 1143 continue
1067 1144
1068 year = int(thisDir[1:5])
1069 doy = int(thisDir[5:8])
1070 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1145 thisDate = getDateFromRadarFolder(thisDir)
1071 1146
1072 1147 pathList.append(datapath)
1073 1148 dateList.append(thisDate)
1074
1075 return dateList
1076 1149
1150 dateList.sort()
1151
1152 if include_path:
1153 return dateList, pathList
1154
1155 return dateList
1077 1156
1078 1157 def setup(self,
1079 1158 path=None,
General Comments 0
You need to be logged in to leave comments. Login now