| @@ -1,1544 +1,1543 | |||||
| 1 | import numpy |
|
1 | import numpy | |
| 2 | import time |
|
2 | import time | |
| 3 | import os |
|
3 | import os | |
| 4 | import h5py |
|
4 | import h5py | |
| 5 | import re |
|
5 | import re | |
| 6 | import datetime |
|
6 | import datetime | |
| 7 |
|
7 | |||
| 8 | from schainpy.model.data.jrodata import * |
|
8 | from schainpy.model.data.jrodata import * | |
| 9 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator |
|
9 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator | |
| 10 | from schainpy.model.io.jroIO_base import * |
|
10 | from schainpy.model.io.jroIO_base import * | |
| 11 | from schainpy.utils import log |
|
11 | from schainpy.utils import log | |
| 12 |
|
12 | |||
| 13 | @MPDecorator |
|
13 | @MPDecorator | |
| 14 | class ParamReader(JRODataReader,ProcessingUnit): |
|
14 | class ParamReader(JRODataReader,ProcessingUnit): | |
| 15 | ''' |
|
15 | ''' | |
| 16 | Reads HDF5 format files |
|
16 | Reads HDF5 format files | |
| 17 | path |
|
17 | path | |
| 18 | startDate |
|
18 | startDate | |
| 19 | endDate |
|
19 | endDate | |
| 20 | startTime |
|
20 | startTime | |
| 21 | endTime |
|
21 | endTime | |
| 22 | ''' |
|
22 | ''' | |
| 23 |
|
23 | |||
| 24 | ext = ".hdf5" |
|
24 | ext = ".hdf5" | |
| 25 | optchar = "D" |
|
25 | optchar = "D" | |
| 26 | timezone = None |
|
26 | timezone = None | |
| 27 | startTime = None |
|
27 | startTime = None | |
| 28 | endTime = None |
|
28 | endTime = None | |
| 29 | fileIndex = None |
|
29 | fileIndex = None | |
| 30 | utcList = None #To select data in the utctime list |
|
30 | utcList = None #To select data in the utctime list | |
| 31 | blockList = None #List to blocks to be read from the file |
|
31 | blockList = None #List to blocks to be read from the file | |
| 32 | blocksPerFile = None #Number of blocks to be read |
|
32 | blocksPerFile = None #Number of blocks to be read | |
| 33 | blockIndex = None |
|
33 | blockIndex = None | |
| 34 | path = None |
|
34 | path = None | |
| 35 | #List of Files |
|
35 | #List of Files | |
| 36 | filenameList = None |
|
36 | filenameList = None | |
| 37 | datetimeList = None |
|
37 | datetimeList = None | |
| 38 | #Hdf5 File |
|
38 | #Hdf5 File | |
| 39 | listMetaname = None |
|
39 | listMetaname = None | |
| 40 | listMeta = None |
|
40 | listMeta = None | |
| 41 | listDataname = None |
|
41 | listDataname = None | |
| 42 | listData = None |
|
42 | listData = None | |
| 43 | listShapes = None |
|
43 | listShapes = None | |
| 44 | fp = None |
|
44 | fp = None | |
| 45 | #dataOut reconstruction |
|
45 | #dataOut reconstruction | |
| 46 | dataOut = None |
|
46 | dataOut = None | |
| 47 |
|
47 | |||
| 48 | def __init__(self):#, **kwargs): |
|
48 | def __init__(self):#, **kwargs): | |
| 49 | ProcessingUnit.__init__(self) #, **kwargs) |
|
49 | ProcessingUnit.__init__(self) #, **kwargs) | |
| 50 | self.dataOut = Parameters() |
|
50 | self.dataOut = Parameters() | |
| 51 | return |
|
51 | return | |
| 52 |
|
52 | |||
| 53 | def setup(self, **kwargs): |
|
53 | def setup(self, **kwargs): | |
| 54 |
|
54 | |||
| 55 | path = kwargs['path'] |
|
55 | path = kwargs['path'] | |
| 56 | startDate = kwargs['startDate'] |
|
56 | startDate = kwargs['startDate'] | |
| 57 | endDate = kwargs['endDate'] |
|
57 | endDate = kwargs['endDate'] | |
| 58 | startTime = kwargs['startTime'] |
|
58 | startTime = kwargs['startTime'] | |
| 59 | endTime = kwargs['endTime'] |
|
59 | endTime = kwargs['endTime'] | |
| 60 | walk = kwargs['walk'] |
|
60 | walk = kwargs['walk'] | |
| 61 | if 'ext' in kwargs: |
|
61 | if 'ext' in kwargs: | |
| 62 | ext = kwargs['ext'] |
|
62 | ext = kwargs['ext'] | |
| 63 | else: |
|
63 | else: | |
| 64 | ext = '.hdf5' |
|
64 | ext = '.hdf5' | |
| 65 | if 'timezone' in kwargs: |
|
65 | if 'timezone' in kwargs: | |
| 66 | self.timezone = kwargs['timezone'] |
|
66 | self.timezone = kwargs['timezone'] | |
| 67 | else: |
|
67 | else: | |
| 68 | self.timezone = 'lt' |
|
68 | self.timezone = 'lt' | |
| 69 |
|
69 | |||
| 70 | print("[Reading] Searching files in offline mode ...") |
|
70 | print("[Reading] Searching files in offline mode ...") | |
| 71 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, |
|
71 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, | |
| 72 | startTime=startTime, endTime=endTime, |
|
72 | startTime=startTime, endTime=endTime, | |
| 73 | ext=ext, walk=walk) |
|
73 | ext=ext, walk=walk) | |
| 74 |
|
74 | |||
| 75 | if not(filenameList): |
|
75 | if not(filenameList): | |
| 76 | print("There is no files into the folder: %s"%(path)) |
|
76 | print("There is no files into the folder: %s"%(path)) | |
| 77 | sys.exit(-1) |
|
77 | sys.exit(-1) | |
| 78 |
|
78 | |||
| 79 | self.fileIndex = -1 |
|
79 | self.fileIndex = -1 | |
| 80 | self.startTime = startTime |
|
80 | self.startTime = startTime | |
| 81 | self.endTime = endTime |
|
81 | self.endTime = endTime | |
| 82 |
|
82 | |||
| 83 | self.__readMetadata() |
|
83 | self.__readMetadata() | |
| 84 |
|
84 | |||
| 85 | self.__setNextFileOffline() |
|
85 | self.__setNextFileOffline() | |
| 86 |
|
86 | |||
| 87 | return |
|
87 | return | |
| 88 |
|
88 | |||
| 89 | def searchFilesOffLine(self, |
|
89 | def searchFilesOffLine(self, | |
| 90 | path, |
|
90 | path, | |
| 91 | startDate=None, |
|
91 | startDate=None, | |
| 92 | endDate=None, |
|
92 | endDate=None, | |
| 93 | startTime=datetime.time(0,0,0), |
|
93 | startTime=datetime.time(0,0,0), | |
| 94 | endTime=datetime.time(23,59,59), |
|
94 | endTime=datetime.time(23,59,59), | |
| 95 | ext='.hdf5', |
|
95 | ext='.hdf5', | |
| 96 | walk=True): |
|
96 | walk=True): | |
| 97 |
|
97 | |||
| 98 | expLabel = '' |
|
98 | expLabel = '' | |
| 99 | self.filenameList = [] |
|
99 | self.filenameList = [] | |
| 100 | self.datetimeList = [] |
|
100 | self.datetimeList = [] | |
| 101 |
|
101 | |||
| 102 | pathList = [] |
|
102 | pathList = [] | |
| 103 |
|
103 | |||
| 104 | JRODataObj = JRODataReader() |
|
104 | JRODataObj = JRODataReader() | |
| 105 | dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) |
|
105 | dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) | |
| 106 |
|
106 | |||
| 107 | if dateList == []: |
|
107 | if dateList == []: | |
| 108 | print("[Reading] No *%s files in %s from %s to %s)"%(ext, path, |
|
108 | print("[Reading] No *%s files in %s from %s to %s)"%(ext, path, | |
| 109 | datetime.datetime.combine(startDate,startTime).ctime(), |
|
109 | datetime.datetime.combine(startDate,startTime).ctime(), | |
| 110 | datetime.datetime.combine(endDate,endTime).ctime())) |
|
110 | datetime.datetime.combine(endDate,endTime).ctime())) | |
| 111 |
|
111 | |||
| 112 | return None, None |
|
112 | return None, None | |
| 113 |
|
113 | |||
| 114 | if len(dateList) > 1: |
|
114 | if len(dateList) > 1: | |
| 115 | print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)) |
|
115 | print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)) | |
| 116 | else: |
|
116 | else: | |
| 117 | print("[Reading] data was found for the date %s" %(dateList[0])) |
|
117 | print("[Reading] data was found for the date %s" %(dateList[0])) | |
| 118 |
|
118 | |||
| 119 | filenameList = [] |
|
119 | filenameList = [] | |
| 120 | datetimeList = [] |
|
120 | datetimeList = [] | |
| 121 |
|
121 | |||
| 122 | #---------------------------------------------------------------------------------- |
|
122 | #---------------------------------------------------------------------------------- | |
| 123 |
|
123 | |||
| 124 | for thisPath in pathList: |
|
124 | for thisPath in pathList: | |
| 125 |
|
125 | |||
| 126 | fileList = glob.glob1(thisPath, "*%s" %ext) |
|
126 | fileList = glob.glob1(thisPath, "*%s" %ext) | |
| 127 | fileList.sort() |
|
127 | fileList.sort() | |
| 128 |
|
128 | |||
| 129 | for file in fileList: |
|
129 | for file in fileList: | |
| 130 |
|
130 | |||
| 131 | filename = os.path.join(thisPath,file) |
|
131 | filename = os.path.join(thisPath,file) | |
| 132 |
|
132 | |||
| 133 | if not isFileInDateRange(filename, startDate, endDate): |
|
133 | if not isFileInDateRange(filename, startDate, endDate): | |
| 134 | continue |
|
134 | continue | |
| 135 |
|
135 | |||
| 136 | thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime) |
|
136 | thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime) | |
| 137 |
|
137 | |||
| 138 | if not(thisDatetime): |
|
138 | if not(thisDatetime): | |
| 139 | continue |
|
139 | continue | |
| 140 |
|
140 | |||
| 141 | filenameList.append(filename) |
|
141 | filenameList.append(filename) | |
| 142 | datetimeList.append(thisDatetime) |
|
142 | datetimeList.append(thisDatetime) | |
| 143 |
|
143 | |||
| 144 | if not(filenameList): |
|
144 | if not(filenameList): | |
| 145 | print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())) |
|
145 | print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())) | |
| 146 | return None, None |
|
146 | return None, None | |
| 147 |
|
147 | |||
| 148 | print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)) |
|
148 | print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)) | |
| 149 | print() |
|
149 | print() | |
| 150 |
|
150 | |||
| 151 | self.filenameList = filenameList |
|
151 | self.filenameList = filenameList | |
| 152 | self.datetimeList = datetimeList |
|
152 | self.datetimeList = datetimeList | |
| 153 |
|
153 | |||
| 154 | return pathList, filenameList |
|
154 | return pathList, filenameList | |
| 155 |
|
155 | |||
| 156 | def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime): |
|
156 | def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime): | |
| 157 |
|
157 | |||
| 158 | """ |
|
158 | """ | |
| 159 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. |
|
159 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |
| 160 |
|
160 | |||
| 161 | Inputs: |
|
161 | Inputs: | |
| 162 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) |
|
162 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |
| 163 | startDate : fecha inicial del rango seleccionado en formato datetime.date |
|
163 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| 164 | endDate : fecha final del rango seleccionado en formato datetime.date |
|
164 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| 165 | startTime : tiempo inicial del rango seleccionado en formato datetime.time |
|
165 | startTime : tiempo inicial del rango seleccionado en formato datetime.time | |
| 166 | endTime : tiempo final del rango seleccionado en formato datetime.time |
|
166 | endTime : tiempo final del rango seleccionado en formato datetime.time | |
| 167 |
|
167 | |||
| 168 | Return: |
|
168 | Return: | |
| 169 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de |
|
169 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |
| 170 | fecha especificado, de lo contrario retorna False. |
|
170 | fecha especificado, de lo contrario retorna False. | |
| 171 |
|
171 | |||
| 172 | Excepciones: |
|
172 | Excepciones: | |
| 173 | Si el archivo no existe o no puede ser abierto |
|
173 | Si el archivo no existe o no puede ser abierto | |
| 174 | Si la cabecera no puede ser leida. |
|
174 | Si la cabecera no puede ser leida. | |
| 175 |
|
175 | |||
| 176 | """ |
|
176 | """ | |
| 177 |
|
177 | |||
| 178 | try: |
|
178 | try: | |
| 179 | fp = h5py.File(filename,'r') |
|
179 | fp = h5py.File(filename,'r') | |
| 180 | grp1 = fp['Data'] |
|
180 | grp1 = fp['Data'] | |
| 181 |
|
181 | |||
| 182 | except IOError: |
|
182 | except IOError: | |
| 183 | traceback.print_exc() |
|
183 | traceback.print_exc() | |
| 184 | raise IOError("The file %s can't be opened" %(filename)) |
|
184 | raise IOError("The file %s can't be opened" %(filename)) | |
| 185 |
|
185 | |||
| 186 | #In case has utctime attribute |
|
186 | #In case has utctime attribute | |
| 187 | grp2 = grp1['utctime'] |
|
187 | grp2 = grp1['utctime'] | |
| 188 | # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time |
|
188 | # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time | |
| 189 | thisUtcTime = grp2.value[0] |
|
189 | thisUtcTime = grp2.value[0] | |
| 190 |
|
190 | |||
| 191 | fp.close() |
|
191 | fp.close() | |
| 192 |
|
192 | |||
| 193 | if self.timezone == 'lt': |
|
193 | if self.timezone == 'lt': | |
| 194 | thisUtcTime -= 5*3600 |
|
194 | thisUtcTime -= 5*3600 | |
| 195 |
|
195 | |||
| 196 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) |
|
196 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) | |
| 197 | thisDate = thisDatetime.date() |
|
197 | thisDate = thisDatetime.date() | |
| 198 | thisTime = thisDatetime.time() |
|
198 | thisTime = thisDatetime.time() | |
| 199 |
|
199 | |||
| 200 | startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds() |
|
200 | startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds() | |
| 201 | endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds() |
|
201 | endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds() | |
| 202 |
|
202 | |||
| 203 | #General case |
|
203 | #General case | |
| 204 | # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o |
|
204 | # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o | |
| 205 | #-----------o----------------------------o----------- |
|
205 | #-----------o----------------------------o----------- | |
| 206 | # startTime endTime |
|
206 | # startTime endTime | |
| 207 |
|
207 | |||
| 208 | if endTime >= startTime: |
|
208 | if endTime >= startTime: | |
| 209 | thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime) |
|
209 | thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime) | |
| 210 | if numpy.any(thisUtcLog): #If there is one block between the hours mentioned |
|
210 | if numpy.any(thisUtcLog): #If there is one block between the hours mentioned | |
| 211 | return thisDatetime |
|
211 | return thisDatetime | |
| 212 | return None |
|
212 | return None | |
| 213 |
|
213 | |||
| 214 | #If endTime < startTime then endTime belongs to the next day |
|
214 | #If endTime < startTime then endTime belongs to the next day | |
| 215 | #<<<<<<<<<<<o o>>>>>>>>>>> |
|
215 | #<<<<<<<<<<<o o>>>>>>>>>>> | |
| 216 | #-----------o----------------------------o----------- |
|
216 | #-----------o----------------------------o----------- | |
| 217 | # endTime startTime |
|
217 | # endTime startTime | |
| 218 |
|
218 | |||
| 219 | if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime): |
|
219 | if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime): | |
| 220 | return None |
|
220 | return None | |
| 221 |
|
221 | |||
| 222 | if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime): |
|
222 | if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime): | |
| 223 | return None |
|
223 | return None | |
| 224 |
|
224 | |||
| 225 | if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime): |
|
225 | if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime): | |
| 226 | return None |
|
226 | return None | |
| 227 |
|
227 | |||
| 228 | return thisDatetime |
|
228 | return thisDatetime | |
| 229 |
|
229 | |||
| 230 | def __setNextFileOffline(self): |
|
230 | def __setNextFileOffline(self): | |
| 231 |
|
231 | |||
| 232 | self.fileIndex += 1 |
|
232 | self.fileIndex += 1 | |
| 233 | idFile = self.fileIndex |
|
233 | idFile = self.fileIndex | |
| 234 |
|
234 | |||
| 235 | if not(idFile < len(self.filenameList)): |
|
235 | if not(idFile < len(self.filenameList)): | |
| 236 | self.dataOut.error = "No more Files" |
|
236 | self.dataOut.error = "No more Files" | |
| 237 | return 0 |
|
237 | return 0 | |
| 238 |
|
238 | |||
| 239 | filename = self.filenameList[idFile] |
|
239 | filename = self.filenameList[idFile] | |
| 240 | filePointer = h5py.File(filename,'r') |
|
240 | filePointer = h5py.File(filename,'r') | |
| 241 | self.filename = filename |
|
241 | self.filename = filename | |
| 242 | self.fp = filePointer |
|
242 | self.fp = filePointer | |
| 243 |
|
243 | |||
| 244 | print("Setting the file: %s"%self.filename) |
|
244 | print("Setting the file: %s"%self.filename) | |
| 245 |
|
245 | |||
| 246 | self.__setBlockList() |
|
246 | self.__setBlockList() | |
| 247 | self.__readData() |
|
247 | self.__readData() | |
| 248 | self.blockIndex = 0 |
|
248 | self.blockIndex = 0 | |
| 249 | return 1 |
|
249 | return 1 | |
| 250 |
|
250 | |||
| 251 | def __setBlockList(self): |
|
251 | def __setBlockList(self): | |
| 252 | ''' |
|
252 | ''' | |
| 253 | Selects the data within the times defined |
|
253 | Selects the data within the times defined | |
| 254 |
|
254 | |||
| 255 | self.fp |
|
255 | self.fp | |
| 256 | self.startTime |
|
256 | self.startTime | |
| 257 | self.endTime |
|
257 | self.endTime | |
| 258 |
|
258 | |||
| 259 | self.blockList |
|
259 | self.blockList | |
| 260 | self.blocksPerFile |
|
260 | self.blocksPerFile | |
| 261 |
|
261 | |||
| 262 | ''' |
|
262 | ''' | |
| 263 | fp = self.fp |
|
263 | fp = self.fp | |
| 264 | startTime = self.startTime |
|
264 | startTime = self.startTime | |
| 265 | endTime = self.endTime |
|
265 | endTime = self.endTime | |
| 266 |
|
266 | |||
| 267 | grp = fp['Data'] |
|
267 | grp = fp['Data'] | |
| 268 | thisUtcTime = grp['utctime'].value.astype(numpy.float)[0] |
|
268 | thisUtcTime = grp['utctime'].value.astype(numpy.float)[0] | |
| 269 |
|
269 | |||
| 270 | #ERROOOOR |
|
270 | #ERROOOOR | |
| 271 | if self.timezone == 'lt': |
|
271 | if self.timezone == 'lt': | |
| 272 | thisUtcTime -= 5*3600 |
|
272 | thisUtcTime -= 5*3600 | |
| 273 |
|
273 | |||
| 274 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) |
|
274 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) | |
| 275 |
|
275 | |||
| 276 | thisDate = thisDatetime.date() |
|
276 | thisDate = thisDatetime.date() | |
| 277 | thisTime = thisDatetime.time() |
|
277 | thisTime = thisDatetime.time() | |
| 278 |
|
278 | |||
| 279 | startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds() |
|
279 | startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds() | |
| 280 | endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds() |
|
280 | endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds() | |
| 281 |
|
281 | |||
| 282 | ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] |
|
282 | ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] | |
| 283 |
|
283 | |||
| 284 | self.blockList = ind |
|
284 | self.blockList = ind | |
| 285 | self.blocksPerFile = len(ind) |
|
285 | self.blocksPerFile = len(ind) | |
| 286 |
|
286 | |||
| 287 | return |
|
287 | return | |
| 288 |
|
288 | |||
| 289 | def __readMetadata(self): |
|
289 | def __readMetadata(self): | |
| 290 | ''' |
|
290 | ''' | |
| 291 | Reads Metadata |
|
291 | Reads Metadata | |
| 292 |
|
292 | |||
| 293 | self.pathMeta |
|
293 | self.pathMeta | |
| 294 | self.listShapes |
|
294 | self.listShapes | |
| 295 | self.listMetaname |
|
295 | self.listMetaname | |
| 296 | self.listMeta |
|
296 | self.listMeta | |
| 297 |
|
297 | |||
| 298 | ''' |
|
298 | ''' | |
| 299 |
|
299 | |||
| 300 | filename = self.filenameList[0] |
|
300 | filename = self.filenameList[0] | |
| 301 | fp = h5py.File(filename,'r') |
|
301 | fp = h5py.File(filename,'r') | |
| 302 | gp = fp['Metadata'] |
|
302 | gp = fp['Metadata'] | |
| 303 |
|
303 | |||
| 304 | listMetaname = [] |
|
304 | listMetaname = [] | |
| 305 | listMetadata = [] |
|
305 | listMetadata = [] | |
| 306 | for item in list(gp.items()): |
|
306 | for item in list(gp.items()): | |
| 307 | name = item[0] |
|
307 | name = item[0] | |
| 308 |
|
308 | |||
| 309 | if name=='array dimensions': |
|
309 | if name=='array dimensions': | |
| 310 | table = gp[name][:] |
|
310 | table = gp[name][:] | |
| 311 | listShapes = {} |
|
311 | listShapes = {} | |
| 312 | for shapes in table: |
|
312 | for shapes in table: | |
| 313 | listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]]) |
|
313 | listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]]) | |
| 314 | else: |
|
314 | else: | |
| 315 | data = gp[name].value |
|
315 | data = gp[name].value | |
| 316 | listMetaname.append(name) |
|
316 | listMetaname.append(name) | |
| 317 | listMetadata.append(data) |
|
317 | listMetadata.append(data) | |
| 318 |
|
318 | |||
| 319 | self.listShapes = listShapes |
|
319 | self.listShapes = listShapes | |
| 320 | self.listMetaname = listMetaname |
|
320 | self.listMetaname = listMetaname | |
| 321 | self.listMeta = listMetadata |
|
321 | self.listMeta = listMetadata | |
| 322 |
|
322 | |||
| 323 | fp.close() |
|
323 | fp.close() | |
| 324 | return |
|
324 | return | |
| 325 |
|
325 | |||
| 326 | def __readData(self): |
|
326 | def __readData(self): | |
| 327 | grp = self.fp['Data'] |
|
327 | grp = self.fp['Data'] | |
| 328 | listdataname = [] |
|
328 | listdataname = [] | |
| 329 | listdata = [] |
|
329 | listdata = [] | |
| 330 |
|
330 | |||
| 331 | for item in list(grp.items()): |
|
331 | for item in list(grp.items()): | |
| 332 | name = item[0] |
|
332 | name = item[0] | |
| 333 | listdataname.append(name) |
|
333 | listdataname.append(name) | |
| 334 |
|
334 | |||
| 335 | array = self.__setDataArray(grp[name],self.listShapes[name]) |
|
335 | array = self.__setDataArray(grp[name],self.listShapes[name]) | |
| 336 | listdata.append(array) |
|
336 | listdata.append(array) | |
| 337 |
|
337 | |||
| 338 | self.listDataname = listdataname |
|
338 | self.listDataname = listdataname | |
| 339 | self.listData = listdata |
|
339 | self.listData = listdata | |
| 340 | return |
|
340 | return | |
| 341 |
|
341 | |||
| 342 | def __setDataArray(self, dataset, shapes): |
|
342 | def __setDataArray(self, dataset, shapes): | |
| 343 |
|
343 | |||
| 344 | nDims = shapes[0] |
|
344 | nDims = shapes[0] | |
| 345 | nDim2 = shapes[1] #Dimension 0 |
|
345 | nDim2 = shapes[1] #Dimension 0 | |
| 346 | nDim1 = shapes[2] #Dimension 1, number of Points or Parameters |
|
346 | nDim1 = shapes[2] #Dimension 1, number of Points or Parameters | |
| 347 | nDim0 = shapes[3] #Dimension 2, number of samples or ranges |
|
347 | nDim0 = shapes[3] #Dimension 2, number of samples or ranges | |
| 348 | mode = shapes[4] #Mode of storing |
|
348 | mode = shapes[4] #Mode of storing | |
| 349 | blockList = self.blockList |
|
349 | blockList = self.blockList | |
| 350 | blocksPerFile = self.blocksPerFile |
|
350 | blocksPerFile = self.blocksPerFile | |
| 351 |
|
351 | |||
| 352 | #Depending on what mode the data was stored |
|
352 | #Depending on what mode the data was stored | |
| 353 | if mode == 0: #Divided in channels |
|
353 | if mode == 0: #Divided in channels | |
| 354 | arrayData = dataset.value.astype(numpy.float)[0][blockList] |
|
354 | arrayData = dataset.value.astype(numpy.float)[0][blockList] | |
| 355 | if mode == 1: #Divided in parameter |
|
355 | if mode == 1: #Divided in parameter | |
| 356 | strds = 'table' |
|
356 | strds = 'table' | |
| 357 | nDatas = nDim1 |
|
357 | nDatas = nDim1 | |
| 358 | newShapes = (blocksPerFile,nDim2,nDim0) |
|
358 | newShapes = (blocksPerFile,nDim2,nDim0) | |
| 359 | elif mode==2: #Concatenated in a table |
|
359 | elif mode==2: #Concatenated in a table | |
| 360 | strds = 'table0' |
|
360 | strds = 'table0' | |
| 361 | arrayData = dataset[strds].value |
|
361 | arrayData = dataset[strds].value | |
| 362 | #Selecting part of the dataset |
|
362 | #Selecting part of the dataset | |
| 363 | utctime = arrayData[:,0] |
|
363 | utctime = arrayData[:,0] | |
| 364 | u, indices = numpy.unique(utctime, return_index=True) |
|
364 | u, indices = numpy.unique(utctime, return_index=True) | |
| 365 |
|
365 | |||
| 366 | if blockList.size != indices.size: |
|
366 | if blockList.size != indices.size: | |
| 367 | indMin = indices[blockList[0]] |
|
367 | indMin = indices[blockList[0]] | |
| 368 | if blockList[1] + 1 >= indices.size: |
|
368 | if blockList[1] + 1 >= indices.size: | |
| 369 | arrayData = arrayData[indMin:,:] |
|
369 | arrayData = arrayData[indMin:,:] | |
| 370 | else: |
|
370 | else: | |
| 371 | indMax = indices[blockList[1] + 1] |
|
371 | indMax = indices[blockList[1] + 1] | |
| 372 | arrayData = arrayData[indMin:indMax,:] |
|
372 | arrayData = arrayData[indMin:indMax,:] | |
| 373 | return arrayData |
|
373 | return arrayData | |
| 374 |
|
374 | |||
| 375 | # One dimension |
|
375 | # One dimension | |
| 376 | if nDims == 0: |
|
376 | if nDims == 0: | |
| 377 | arrayData = dataset.value.astype(numpy.float)[0][blockList] |
|
377 | arrayData = dataset.value.astype(numpy.float)[0][blockList] | |
| 378 |
|
378 | |||
| 379 | # Two dimensions |
|
379 | # Two dimensions | |
| 380 | elif nDims == 2: |
|
380 | elif nDims == 2: | |
| 381 | arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0)) |
|
381 | arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0)) | |
| 382 | newShapes = (blocksPerFile,nDim0) |
|
382 | newShapes = (blocksPerFile,nDim0) | |
| 383 | nDatas = nDim1 |
|
383 | nDatas = nDim1 | |
| 384 |
|
384 | |||
| 385 | for i in range(nDatas): |
|
385 | for i in range(nDatas): | |
| 386 | data = dataset[strds + str(i)].value |
|
386 | data = dataset[strds + str(i)].value | |
| 387 | arrayData[:,i,:] = data[blockList,:] |
|
387 | arrayData[:,i,:] = data[blockList,:] | |
| 388 |
|
388 | |||
| 389 | # Three dimensions |
|
389 | # Three dimensions | |
| 390 | else: |
|
390 | else: | |
| 391 | arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0)) |
|
391 | arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0)) | |
| 392 | for i in range(nDatas): |
|
392 | for i in range(nDatas): | |
| 393 |
|
393 | |||
| 394 | data = dataset[strds + str(i)].value |
|
394 | data = dataset[strds + str(i)].value | |
| 395 |
|
395 | |||
| 396 | for b in range(blockList.size): |
|
396 | for b in range(blockList.size): | |
| 397 | arrayData[b,:,i,:] = data[:,:,blockList[b]] |
|
397 | arrayData[b,:,i,:] = data[:,:,blockList[b]] | |
| 398 |
|
398 | |||
| 399 | return arrayData |
|
399 | return arrayData | |
| 400 |
|
400 | |||
| 401 | def __setDataOut(self): |
|
401 | def __setDataOut(self): | |
| 402 | listMeta = self.listMeta |
|
402 | listMeta = self.listMeta | |
| 403 | listMetaname = self.listMetaname |
|
403 | listMetaname = self.listMetaname | |
| 404 | listDataname = self.listDataname |
|
404 | listDataname = self.listDataname | |
| 405 | listData = self.listData |
|
405 | listData = self.listData | |
| 406 | listShapes = self.listShapes |
|
406 | listShapes = self.listShapes | |
| 407 |
|
407 | |||
| 408 | blockIndex = self.blockIndex |
|
408 | blockIndex = self.blockIndex | |
| 409 | # blockList = self.blockList |
|
409 | # blockList = self.blockList | |
| 410 |
|
410 | |||
| 411 | for i in range(len(listMeta)): |
|
411 | for i in range(len(listMeta)): | |
| 412 | setattr(self.dataOut,listMetaname[i],listMeta[i]) |
|
412 | setattr(self.dataOut,listMetaname[i],listMeta[i]) | |
| 413 |
|
413 | |||
| 414 | for j in range(len(listData)): |
|
414 | for j in range(len(listData)): | |
| 415 | nShapes = listShapes[listDataname[j]][0] |
|
415 | nShapes = listShapes[listDataname[j]][0] | |
| 416 | mode = listShapes[listDataname[j]][4] |
|
416 | mode = listShapes[listDataname[j]][4] | |
| 417 | if nShapes == 1: |
|
417 | if nShapes == 1: | |
| 418 | setattr(self.dataOut,listDataname[j],listData[j][blockIndex]) |
|
418 | setattr(self.dataOut,listDataname[j],listData[j][blockIndex]) | |
| 419 | elif nShapes > 1: |
|
419 | elif nShapes > 1: | |
| 420 | setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:]) |
|
420 | setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:]) | |
| 421 | elif mode==0: |
|
421 | elif mode==0: | |
| 422 | setattr(self.dataOut,listDataname[j],listData[j][blockIndex]) |
|
422 | setattr(self.dataOut,listDataname[j],listData[j][blockIndex]) | |
| 423 | #Mode Meteors |
|
423 | #Mode Meteors | |
| 424 | elif mode ==2: |
|
424 | elif mode ==2: | |
| 425 | selectedData = self.__selectDataMode2(listData[j], blockIndex) |
|
425 | selectedData = self.__selectDataMode2(listData[j], blockIndex) | |
| 426 | setattr(self.dataOut, listDataname[j], selectedData) |
|
426 | setattr(self.dataOut, listDataname[j], selectedData) | |
| 427 | return |
|
427 | return | |
| 428 |
|
428 | |||
| 429 | def __selectDataMode2(self, data, blockIndex): |
|
429 | def __selectDataMode2(self, data, blockIndex): | |
| 430 | utctime = data[:,0] |
|
430 | utctime = data[:,0] | |
| 431 | aux, indices = numpy.unique(utctime, return_inverse=True) |
|
431 | aux, indices = numpy.unique(utctime, return_inverse=True) | |
| 432 | selInd = numpy.where(indices == blockIndex)[0] |
|
432 | selInd = numpy.where(indices == blockIndex)[0] | |
| 433 | selData = data[selInd,:] |
|
433 | selData = data[selInd,:] | |
| 434 |
|
434 | |||
| 435 | return selData |
|
435 | return selData | |
| 436 |
|
436 | |||
| 437 | def getData(self): |
|
437 | def getData(self): | |
| 438 |
|
438 | |||
| 439 | if self.blockIndex==self.blocksPerFile: |
|
439 | if self.blockIndex==self.blocksPerFile: | |
| 440 | if not( self.__setNextFileOffline() ): |
|
440 | if not( self.__setNextFileOffline() ): | |
| 441 | self.dataOut.flagNoData = True |
|
441 | self.dataOut.flagNoData = True | |
| 442 | return 0 |
|
442 | return 0 | |
| 443 |
|
443 | |||
| 444 | self.__setDataOut() |
|
444 | self.__setDataOut() | |
| 445 | self.dataOut.flagNoData = False |
|
445 | self.dataOut.flagNoData = False | |
| 446 |
|
446 | |||
| 447 | self.blockIndex += 1 |
|
447 | self.blockIndex += 1 | |
| 448 |
|
448 | |||
| 449 | return |
|
449 | return | |
| 450 |
|
450 | |||
| 451 | def run(self, **kwargs): |
|
451 | def run(self, **kwargs): | |
| 452 |
|
452 | |||
| 453 | if not(self.isConfig): |
|
453 | if not(self.isConfig): | |
| 454 | self.setup(**kwargs) |
|
454 | self.setup(**kwargs) | |
| 455 | self.isConfig = True |
|
455 | self.isConfig = True | |
| 456 |
|
456 | |||
| 457 | self.getData() |
|
457 | self.getData() | |
| 458 |
|
458 | |||
| 459 | return |
|
459 | return | |
| 460 |
|
460 | |||
| 461 | @MPDecorator |
|
461 | @MPDecorator | |
| 462 | class ParamWriter(Operation): |
|
462 | class ParamWriter(Operation): | |
| 463 | ''' |
|
463 | ''' | |
| 464 | HDF5 Writer, stores parameters data in HDF5 format files |
|
464 | HDF5 Writer, stores parameters data in HDF5 format files | |
| 465 |
|
465 | |||
| 466 | path: path where the files will be stored |
|
466 | path: path where the files will be stored | |
| 467 | blocksPerFile: number of blocks that will be saved in per HDF5 format file |
|
467 | blocksPerFile: number of blocks that will be saved in per HDF5 format file | |
| 468 | mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) |
|
468 | mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) | |
| 469 | metadataList: list of attributes that will be stored as metadata |
|
469 | metadataList: list of attributes that will be stored as metadata | |
| 470 | dataList: list of attributes that will be stores as data |
|
470 | dataList: list of attributes that will be stores as data | |
| 471 | ''' |
|
471 | ''' | |
| 472 |
|
472 | |||
| 473 | ext = ".hdf5" |
|
473 | ext = ".hdf5" | |
| 474 | optchar = "D" |
|
474 | optchar = "D" | |
| 475 | metaoptchar = "M" |
|
475 | metaoptchar = "M" | |
| 476 | metaFile = None |
|
476 | metaFile = None | |
| 477 | filename = None |
|
477 | filename = None | |
| 478 | path = None |
|
478 | path = None | |
| 479 | setFile = None |
|
479 | setFile = None | |
| 480 | fp = None |
|
480 | fp = None | |
| 481 | grp = None |
|
481 | grp = None | |
| 482 | ds = None |
|
482 | ds = None | |
| 483 | firsttime = True |
|
483 | firsttime = True | |
| 484 | #Configurations |
|
484 | #Configurations | |
| 485 | blocksPerFile = None |
|
485 | blocksPerFile = None | |
| 486 | blockIndex = None |
|
486 | blockIndex = None | |
| 487 | dataOut = None |
|
487 | dataOut = None | |
| 488 | #Data Arrays |
|
488 | #Data Arrays | |
| 489 | dataList = None |
|
489 | dataList = None | |
| 490 | metadataList = None |
|
490 | metadataList = None | |
| 491 | dsList = None #List of dictionaries with dataset properties |
|
491 | dsList = None #List of dictionaries with dataset properties | |
| 492 | tableDim = None |
|
492 | tableDim = None | |
| 493 | dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')] |
|
493 | dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')] | |
| 494 | currentDay = None |
|
494 | currentDay = None | |
| 495 | lastTime = None |
|
495 | lastTime = None | |
| 496 | setType = None |
|
496 | setType = None | |
| 497 |
|
497 | |||
| 498 | def __init__(self): |
|
498 | def __init__(self): | |
| 499 |
|
499 | |||
| 500 | Operation.__init__(self) |
|
500 | Operation.__init__(self) | |
| 501 | return |
|
501 | return | |
| 502 |
|
502 | |||
| 503 | def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None): |
|
503 | def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None): | |
| 504 | self.path = path |
|
504 | self.path = path | |
| 505 | self.blocksPerFile = blocksPerFile |
|
505 | self.blocksPerFile = blocksPerFile | |
| 506 | self.metadataList = metadataList |
|
506 | self.metadataList = metadataList | |
| 507 | self.dataList = dataList |
|
507 | self.dataList = dataList | |
| 508 | self.dataOut = dataOut |
|
508 | self.dataOut = dataOut | |
| 509 | self.mode = mode |
|
509 | self.mode = mode | |
| 510 | if self.mode is not None: |
|
510 | if self.mode is not None: | |
| 511 | self.mode = numpy.zeros(len(self.dataList)) + mode |
|
511 | self.mode = numpy.zeros(len(self.dataList)) + mode | |
| 512 | else: |
|
512 | else: | |
| 513 | self.mode = numpy.ones(len(self.dataList)) |
|
513 | self.mode = numpy.ones(len(self.dataList)) | |
| 514 |
|
514 | |||
| 515 | self.setType = setType |
|
515 | self.setType = setType | |
| 516 |
|
516 | |||
| 517 | arrayDim = numpy.zeros((len(self.dataList),5)) |
|
517 | arrayDim = numpy.zeros((len(self.dataList),5)) | |
| 518 |
|
518 | |||
| 519 | #Table dimensions |
|
519 | #Table dimensions | |
| 520 | dtype0 = self.dtype |
|
520 | dtype0 = self.dtype | |
| 521 | tableList = [] |
|
521 | tableList = [] | |
| 522 |
|
522 | |||
| 523 | #Dictionary and list of tables |
|
523 | #Dictionary and list of tables | |
| 524 | dsList = [] |
|
524 | dsList = [] | |
| 525 |
|
525 | |||
| 526 | for i in range(len(self.dataList)): |
|
526 | for i in range(len(self.dataList)): | |
| 527 | dsDict = {} |
|
527 | dsDict = {} | |
| 528 | dataAux = getattr(self.dataOut, self.dataList[i]) |
|
528 | dataAux = getattr(self.dataOut, self.dataList[i]) | |
| 529 | dsDict['variable'] = self.dataList[i] |
|
529 | dsDict['variable'] = self.dataList[i] | |
| 530 | #--------------------- Conditionals ------------------------ |
|
530 | #--------------------- Conditionals ------------------------ | |
| 531 | #There is no data |
|
531 | #There is no data | |
| 532 |
|
532 | |||
| 533 | if dataAux is None: |
|
533 | if dataAux is None: | |
| 534 |
|
534 | |||
| 535 | return 0 |
|
535 | return 0 | |
| 536 |
|
536 | |||
| 537 | if isinstance(dataAux, (int, float, numpy.integer, numpy.float)): |
|
537 | if isinstance(dataAux, (int, float, numpy.integer, numpy.float)): | |
| 538 | dsDict['mode'] = 0 |
|
538 | dsDict['mode'] = 0 | |
| 539 | dsDict['nDim'] = 0 |
|
539 | dsDict['nDim'] = 0 | |
| 540 | arrayDim[i,0] = 0 |
|
540 | arrayDim[i,0] = 0 | |
| 541 | dsList.append(dsDict) |
|
541 | dsList.append(dsDict) | |
| 542 |
|
542 | |||
| 543 | #Mode 2: meteors |
|
543 | #Mode 2: meteors | |
| 544 | elif self.mode[i] == 2: |
|
544 | elif self.mode[i] == 2: | |
| 545 | dsDict['dsName'] = 'table0' |
|
545 | dsDict['dsName'] = 'table0' | |
| 546 | dsDict['mode'] = 2 # Mode meteors |
|
546 | dsDict['mode'] = 2 # Mode meteors | |
| 547 | dsDict['shape'] = dataAux.shape[-1] |
|
547 | dsDict['shape'] = dataAux.shape[-1] | |
| 548 | dsDict['nDim'] = 0 |
|
548 | dsDict['nDim'] = 0 | |
| 549 | dsDict['dsNumber'] = 1 |
|
549 | dsDict['dsNumber'] = 1 | |
| 550 | arrayDim[i,3] = dataAux.shape[-1] |
|
550 | arrayDim[i,3] = dataAux.shape[-1] | |
| 551 | arrayDim[i,4] = self.mode[i] #Mode the data was stored |
|
551 | arrayDim[i,4] = self.mode[i] #Mode the data was stored | |
| 552 | dsList.append(dsDict) |
|
552 | dsList.append(dsDict) | |
| 553 |
|
553 | |||
| 554 | #Mode 1 |
|
554 | #Mode 1 | |
| 555 | else: |
|
555 | else: | |
| 556 | arrayDim0 = dataAux.shape #Data dimensions |
|
556 | arrayDim0 = dataAux.shape #Data dimensions | |
| 557 | arrayDim[i,0] = len(arrayDim0) #Number of array dimensions |
|
557 | arrayDim[i,0] = len(arrayDim0) #Number of array dimensions | |
| 558 | arrayDim[i,4] = self.mode[i] #Mode the data was stored |
|
558 | arrayDim[i,4] = self.mode[i] #Mode the data was stored | |
| 559 | strtable = 'table' |
|
559 | strtable = 'table' | |
| 560 | dsDict['mode'] = 1 # Mode parameters |
|
560 | dsDict['mode'] = 1 # Mode parameters | |
| 561 |
|
561 | |||
| 562 | # Three-dimension arrays |
|
562 | # Three-dimension arrays | |
| 563 | if len(arrayDim0) == 3: |
|
563 | if len(arrayDim0) == 3: | |
| 564 | arrayDim[i,1:-1] = numpy.array(arrayDim0) |
|
564 | arrayDim[i,1:-1] = numpy.array(arrayDim0) | |
| 565 | nTables = int(arrayDim[i,2]) |
|
565 | nTables = int(arrayDim[i,2]) | |
| 566 | dsDict['dsNumber'] = nTables |
|
566 | dsDict['dsNumber'] = nTables | |
| 567 | dsDict['shape'] = arrayDim[i,2:4] |
|
567 | dsDict['shape'] = arrayDim[i,2:4] | |
| 568 | dsDict['nDim'] = 3 |
|
568 | dsDict['nDim'] = 3 | |
| 569 |
|
569 | |||
| 570 | for j in range(nTables): |
|
570 | for j in range(nTables): | |
| 571 | dsDict = dsDict.copy() |
|
571 | dsDict = dsDict.copy() | |
| 572 | dsDict['dsName'] = strtable + str(j) |
|
572 | dsDict['dsName'] = strtable + str(j) | |
| 573 | dsList.append(dsDict) |
|
573 | dsList.append(dsDict) | |
| 574 |
|
574 | |||
| 575 | # Two-dimension arrays |
|
575 | # Two-dimension arrays | |
| 576 | elif len(arrayDim0) == 2: |
|
576 | elif len(arrayDim0) == 2: | |
| 577 | arrayDim[i,2:-1] = numpy.array(arrayDim0) |
|
577 | arrayDim[i,2:-1] = numpy.array(arrayDim0) | |
| 578 | nTables = int(arrayDim[i,2]) |
|
578 | nTables = int(arrayDim[i,2]) | |
| 579 | dsDict['dsNumber'] = nTables |
|
579 | dsDict['dsNumber'] = nTables | |
| 580 | dsDict['shape'] = arrayDim[i,3] |
|
580 | dsDict['shape'] = arrayDim[i,3] | |
| 581 | dsDict['nDim'] = 2 |
|
581 | dsDict['nDim'] = 2 | |
| 582 |
|
582 | |||
| 583 | for j in range(nTables): |
|
583 | for j in range(nTables): | |
| 584 | dsDict = dsDict.copy() |
|
584 | dsDict = dsDict.copy() | |
| 585 | dsDict['dsName'] = strtable + str(j) |
|
585 | dsDict['dsName'] = strtable + str(j) | |
| 586 | dsList.append(dsDict) |
|
586 | dsList.append(dsDict) | |
| 587 |
|
587 | |||
| 588 | # One-dimension arrays |
|
588 | # One-dimension arrays | |
| 589 | elif len(arrayDim0) == 1: |
|
589 | elif len(arrayDim0) == 1: | |
| 590 | arrayDim[i,3] = arrayDim0[0] |
|
590 | arrayDim[i,3] = arrayDim0[0] | |
| 591 | dsDict['shape'] = arrayDim0[0] |
|
591 | dsDict['shape'] = arrayDim0[0] | |
| 592 | dsDict['dsNumber'] = 1 |
|
592 | dsDict['dsNumber'] = 1 | |
| 593 | dsDict['dsName'] = strtable + str(0) |
|
593 | dsDict['dsName'] = strtable + str(0) | |
| 594 | dsDict['nDim'] = 1 |
|
594 | dsDict['nDim'] = 1 | |
| 595 | dsList.append(dsDict) |
|
595 | dsList.append(dsDict) | |
| 596 |
|
596 | |||
| 597 | table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0) |
|
597 | table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0) | |
| 598 | tableList.append(table) |
|
598 | tableList.append(table) | |
| 599 |
|
599 | |||
| 600 | self.dsList = dsList |
|
600 | self.dsList = dsList | |
| 601 | self.tableDim = numpy.array(tableList, dtype = dtype0) |
|
601 | self.tableDim = numpy.array(tableList, dtype = dtype0) | |
| 602 | self.blockIndex = 0 |
|
602 | self.blockIndex = 0 | |
| 603 | timeTuple = time.localtime(dataOut.utctime) |
|
603 | timeTuple = time.localtime(dataOut.utctime) | |
| 604 | self.currentDay = timeTuple.tm_yday |
|
604 | self.currentDay = timeTuple.tm_yday | |
| 605 |
|
605 | |||
| 606 | def putMetadata(self): |
|
606 | def putMetadata(self): | |
| 607 |
|
607 | |||
| 608 | fp = self.createMetadataFile() |
|
608 | fp = self.createMetadataFile() | |
| 609 | self.writeMetadata(fp) |
|
609 | self.writeMetadata(fp) | |
| 610 | fp.close() |
|
610 | fp.close() | |
| 611 | return |
|
611 | return | |
| 612 |
|
612 | |||
| 613 | def createMetadataFile(self): |
|
613 | def createMetadataFile(self): | |
| 614 | ext = self.ext |
|
614 | ext = self.ext | |
| 615 | path = self.path |
|
615 | path = self.path | |
| 616 | setFile = self.setFile |
|
616 | setFile = self.setFile | |
| 617 |
|
617 | |||
| 618 | timeTuple = time.localtime(self.dataOut.utctime) |
|
618 | timeTuple = time.localtime(self.dataOut.utctime) | |
| 619 |
|
619 | |||
| 620 | subfolder = '' |
|
620 | subfolder = '' | |
| 621 | fullpath = os.path.join( path, subfolder ) |
|
621 | fullpath = os.path.join( path, subfolder ) | |
| 622 |
|
622 | |||
| 623 | if not( os.path.exists(fullpath) ): |
|
623 | if not( os.path.exists(fullpath) ): | |
| 624 | os.mkdir(fullpath) |
|
624 | os.mkdir(fullpath) | |
| 625 | setFile = -1 #inicializo mi contador de seteo |
|
625 | setFile = -1 #inicializo mi contador de seteo | |
| 626 |
|
626 | |||
| 627 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) |
|
627 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |
| 628 | fullpath = os.path.join( path, subfolder ) |
|
628 | fullpath = os.path.join( path, subfolder ) | |
| 629 |
|
629 | |||
| 630 | if not( os.path.exists(fullpath) ): |
|
630 | if not( os.path.exists(fullpath) ): | |
| 631 | os.mkdir(fullpath) |
|
631 | os.mkdir(fullpath) | |
| 632 | setFile = -1 #inicializo mi contador de seteo |
|
632 | setFile = -1 #inicializo mi contador de seteo | |
| 633 |
|
633 | |||
| 634 | else: |
|
634 | else: | |
| 635 | filesList = os.listdir( fullpath ) |
|
635 | filesList = os.listdir( fullpath ) | |
| 636 | filesList = sorted( filesList, key=str.lower ) |
|
636 | filesList = sorted( filesList, key=str.lower ) | |
| 637 | if len( filesList ) > 0: |
|
637 | if len( filesList ) > 0: | |
| 638 | filesList = [k for k in filesList if k.startswith(self.metaoptchar)] |
|
638 | filesList = [k for k in filesList if k.startswith(self.metaoptchar)] | |
| 639 | filen = filesList[-1] |
|
639 | filen = filesList[-1] | |
| 640 | # el filename debera tener el siguiente formato |
|
640 | # el filename debera tener el siguiente formato | |
| 641 | # 0 1234 567 89A BCDE (hex) |
|
641 | # 0 1234 567 89A BCDE (hex) | |
| 642 | # x YYYY DDD SSS .ext |
|
642 | # x YYYY DDD SSS .ext | |
| 643 | if isNumber( filen[8:11] ): |
|
643 | if isNumber( filen[8:11] ): | |
| 644 | setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file |
|
644 | setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | |
| 645 | else: |
|
645 | else: | |
| 646 | setFile = -1 |
|
646 | setFile = -1 | |
| 647 | else: |
|
647 | else: | |
| 648 | setFile = -1 #inicializo mi contador de seteo |
|
648 | setFile = -1 #inicializo mi contador de seteo | |
| 649 |
|
649 | |||
| 650 | if self.setType is None: |
|
650 | if self.setType is None: | |
| 651 | setFile += 1 |
|
651 | setFile += 1 | |
| 652 | file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar, |
|
652 | file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar, | |
| 653 | timeTuple.tm_year, |
|
653 | timeTuple.tm_year, | |
| 654 | timeTuple.tm_yday, |
|
654 | timeTuple.tm_yday, | |
| 655 | setFile, |
|
655 | setFile, | |
| 656 | ext ) |
|
656 | ext ) | |
| 657 | else: |
|
657 | else: | |
| 658 | setFile = timeTuple.tm_hour*60+timeTuple.tm_min |
|
658 | setFile = timeTuple.tm_hour*60+timeTuple.tm_min | |
| 659 | file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar, |
|
659 | file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar, | |
| 660 | timeTuple.tm_year, |
|
660 | timeTuple.tm_year, | |
| 661 | timeTuple.tm_yday, |
|
661 | timeTuple.tm_yday, | |
| 662 | setFile, |
|
662 | setFile, | |
| 663 | ext ) |
|
663 | ext ) | |
| 664 |
|
664 | |||
| 665 | filename = os.path.join( path, subfolder, file ) |
|
665 | filename = os.path.join( path, subfolder, file ) | |
| 666 | self.metaFile = file |
|
666 | self.metaFile = file | |
| 667 | #Setting HDF5 File |
|
667 | #Setting HDF5 File | |
| 668 | fp = h5py.File(filename,'w') |
|
668 | fp = h5py.File(filename,'w') | |
| 669 |
|
669 | |||
| 670 | return fp |
|
670 | return fp | |
| 671 |
|
671 | |||
| 672 | def writeMetadata(self, fp): |
|
672 | def writeMetadata(self, fp): | |
| 673 |
|
673 | |||
| 674 | grp = fp.create_group("Metadata") |
|
674 | grp = fp.create_group("Metadata") | |
| 675 | grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype) |
|
675 | grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype) | |
| 676 |
|
676 | |||
| 677 | for i in range(len(self.metadataList)): |
|
677 | for i in range(len(self.metadataList)): | |
| 678 | grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i])) |
|
678 | grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i])) | |
| 679 | return |
|
679 | return | |
| 680 |
|
680 | |||
| 681 | def timeFlag(self): |
|
681 | def timeFlag(self): | |
| 682 | currentTime = self.dataOut.utctime |
|
682 | currentTime = self.dataOut.utctime | |
| 683 |
|
683 | |||
| 684 | if self.lastTime is None: |
|
684 | if self.lastTime is None: | |
| 685 | self.lastTime = currentTime |
|
685 | self.lastTime = currentTime | |
| 686 |
|
686 | |||
| 687 | #Day |
|
687 | #Day | |
| 688 | timeTuple = time.localtime(currentTime) |
|
688 | timeTuple = time.localtime(currentTime) | |
| 689 | dataDay = timeTuple.tm_yday |
|
689 | dataDay = timeTuple.tm_yday | |
| 690 |
|
690 | |||
| 691 | #Time |
|
691 | #Time | |
| 692 | timeDiff = currentTime - self.lastTime |
|
692 | timeDiff = currentTime - self.lastTime | |
| 693 |
|
693 | |||
| 694 | #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora |
|
694 | #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora | |
| 695 | if dataDay != self.currentDay: |
|
695 | if dataDay != self.currentDay: | |
| 696 | self.currentDay = dataDay |
|
696 | self.currentDay = dataDay | |
| 697 | return True |
|
697 | return True | |
| 698 | elif timeDiff > 3*60*60: |
|
698 | elif timeDiff > 3*60*60: | |
| 699 | self.lastTime = currentTime |
|
699 | self.lastTime = currentTime | |
| 700 | return True |
|
700 | return True | |
| 701 | else: |
|
701 | else: | |
| 702 | self.lastTime = currentTime |
|
702 | self.lastTime = currentTime | |
| 703 | return False |
|
703 | return False | |
| 704 |
|
704 | |||
| 705 | def setNextFile(self): |
|
705 | def setNextFile(self): | |
| 706 |
|
706 | |||
| 707 | ext = self.ext |
|
707 | ext = self.ext | |
| 708 | path = self.path |
|
708 | path = self.path | |
| 709 | setFile = self.setFile |
|
709 | setFile = self.setFile | |
| 710 | mode = self.mode |
|
710 | mode = self.mode | |
| 711 |
|
711 | |||
| 712 | timeTuple = time.localtime(self.dataOut.utctime) |
|
712 | timeTuple = time.localtime(self.dataOut.utctime) | |
| 713 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) |
|
713 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |
| 714 |
|
714 | |||
| 715 | fullpath = os.path.join( path, subfolder ) |
|
715 | fullpath = os.path.join( path, subfolder ) | |
| 716 |
|
716 | |||
| 717 | if os.path.exists(fullpath): |
|
717 | if os.path.exists(fullpath): | |
| 718 | filesList = os.listdir( fullpath ) |
|
718 | filesList = os.listdir( fullpath ) | |
| 719 | filesList = [k for k in filesList if 'M' in k] |
|
719 | filesList = [k for k in filesList if 'M' in k] | |
| 720 | if len( filesList ) > 0: |
|
720 | if len( filesList ) > 0: | |
| 721 | filesList = sorted( filesList, key=str.lower ) |
|
721 | filesList = sorted( filesList, key=str.lower ) | |
| 722 | filen = filesList[-1] |
|
722 | filen = filesList[-1] | |
| 723 | # el filename debera tener el siguiente formato |
|
723 | # el filename debera tener el siguiente formato | |
| 724 | # 0 1234 567 89A BCDE (hex) |
|
724 | # 0 1234 567 89A BCDE (hex) | |
| 725 | # x YYYY DDD SSS .ext |
|
725 | # x YYYY DDD SSS .ext | |
| 726 | if isNumber( filen[8:11] ): |
|
726 | if isNumber( filen[8:11] ): | |
| 727 | setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file |
|
727 | setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | |
| 728 | else: |
|
728 | else: | |
| 729 | setFile = -1 |
|
729 | setFile = -1 | |
| 730 | else: |
|
730 | else: | |
| 731 | setFile = -1 #inicializo mi contador de seteo |
|
731 | setFile = -1 #inicializo mi contador de seteo | |
| 732 | else: |
|
732 | else: | |
| 733 | os.makedirs(fullpath) |
|
733 | os.makedirs(fullpath) | |
| 734 | setFile = -1 #inicializo mi contador de seteo |
|
734 | setFile = -1 #inicializo mi contador de seteo | |
| 735 |
|
735 | |||
| 736 | if self.setType is None: |
|
736 | if self.setType is None: | |
| 737 | setFile += 1 |
|
737 | setFile += 1 | |
| 738 | file = '%s%4.4d%3.3d%03d%s' % (self.optchar, |
|
738 | file = '%s%4.4d%3.3d%03d%s' % (self.optchar, | |
| 739 | timeTuple.tm_year, |
|
739 | timeTuple.tm_year, | |
| 740 | timeTuple.tm_yday, |
|
740 | timeTuple.tm_yday, | |
| 741 | setFile, |
|
741 | setFile, | |
| 742 | ext ) |
|
742 | ext ) | |
| 743 | else: |
|
743 | else: | |
| 744 | setFile = timeTuple.tm_hour*60+timeTuple.tm_min |
|
744 | setFile = timeTuple.tm_hour*60+timeTuple.tm_min | |
| 745 | file = '%s%4.4d%3.3d%04d%s' % (self.optchar, |
|
745 | file = '%s%4.4d%3.3d%04d%s' % (self.optchar, | |
| 746 | timeTuple.tm_year, |
|
746 | timeTuple.tm_year, | |
| 747 | timeTuple.tm_yday, |
|
747 | timeTuple.tm_yday, | |
| 748 | setFile, |
|
748 | setFile, | |
| 749 | ext ) |
|
749 | ext ) | |
| 750 |
|
750 | |||
| 751 | filename = os.path.join( path, subfolder, file ) |
|
751 | filename = os.path.join( path, subfolder, file ) | |
| 752 |
|
752 | |||
| 753 | #Setting HDF5 File |
|
753 | #Setting HDF5 File | |
| 754 | fp = h5py.File(filename,'w') |
|
754 | fp = h5py.File(filename,'w') | |
| 755 | #write metadata |
|
755 | #write metadata | |
| 756 | self.writeMetadata(fp) |
|
756 | self.writeMetadata(fp) | |
| 757 | #Write data |
|
757 | #Write data | |
| 758 | grp = fp.create_group("Data") |
|
758 | grp = fp.create_group("Data") | |
| 759 | ds = [] |
|
759 | ds = [] | |
| 760 | data = [] |
|
760 | data = [] | |
| 761 | dsList = self.dsList |
|
761 | dsList = self.dsList | |
| 762 | i = 0 |
|
762 | i = 0 | |
| 763 | while i < len(dsList): |
|
763 | while i < len(dsList): | |
| 764 | dsInfo = dsList[i] |
|
764 | dsInfo = dsList[i] | |
| 765 | #One-dimension data |
|
765 | #One-dimension data | |
| 766 | if dsInfo['mode'] == 0: |
|
766 | if dsInfo['mode'] == 0: | |
| 767 | ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64) |
|
767 | ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64) | |
| 768 | ds.append(ds0) |
|
768 | ds.append(ds0) | |
| 769 | data.append([]) |
|
769 | data.append([]) | |
| 770 | i += 1 |
|
770 | i += 1 | |
| 771 | continue |
|
771 | continue | |
| 772 |
|
772 | |||
| 773 | elif dsInfo['mode'] == 2: |
|
773 | elif dsInfo['mode'] == 2: | |
| 774 | grp0 = grp.create_group(dsInfo['variable']) |
|
774 | grp0 = grp.create_group(dsInfo['variable']) | |
| 775 | ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True) |
|
775 | ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True) | |
| 776 | ds.append(ds0) |
|
776 | ds.append(ds0) | |
| 777 | data.append([]) |
|
777 | data.append([]) | |
| 778 | i += 1 |
|
778 | i += 1 | |
| 779 | continue |
|
779 | continue | |
| 780 |
|
780 | |||
| 781 | elif dsInfo['mode'] == 1: |
|
781 | elif dsInfo['mode'] == 1: | |
| 782 | grp0 = grp.create_group(dsInfo['variable']) |
|
782 | grp0 = grp.create_group(dsInfo['variable']) | |
| 783 |
|
783 | |||
| 784 | for j in range(dsInfo['dsNumber']): |
|
784 | for j in range(dsInfo['dsNumber']): | |
| 785 | dsInfo = dsList[i] |
|
785 | dsInfo = dsList[i] | |
| 786 | tableName = dsInfo['dsName'] |
|
786 | tableName = dsInfo['dsName'] | |
| 787 |
|
787 | |||
| 788 |
|
788 | |||
| 789 | if dsInfo['nDim'] == 3: |
|
789 | if dsInfo['nDim'] == 3: | |
| 790 | shape = dsInfo['shape'].astype(int) |
|
790 | shape = dsInfo['shape'].astype(int) | |
| 791 | ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True) |
|
791 | ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True) | |
| 792 | else: |
|
792 | else: | |
| 793 | shape = int(dsInfo['shape']) |
|
793 | shape = int(dsInfo['shape']) | |
| 794 | ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True) |
|
794 | ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True) | |
| 795 |
|
795 | |||
| 796 | ds.append(ds0) |
|
796 | ds.append(ds0) | |
| 797 | data.append([]) |
|
797 | data.append([]) | |
| 798 | i += 1 |
|
798 | i += 1 | |
| 799 |
|
799 | |||
| 800 | fp.flush() |
|
800 | fp.flush() | |
| 801 | fp.close() |
|
801 | fp.close() | |
| 802 |
|
802 | |||
| 803 | log.log('creating file: {}'.format(filename), 'Writing') |
|
803 | log.log('creating file: {}'.format(filename), 'Writing') | |
| 804 | self.filename = filename |
|
804 | self.filename = filename | |
| 805 | self.ds = ds |
|
805 | self.ds = ds | |
| 806 | self.data = data |
|
806 | self.data = data | |
| 807 | self.firsttime = True |
|
807 | self.firsttime = True | |
| 808 | self.blockIndex = 0 |
|
808 | self.blockIndex = 0 | |
| 809 | return |
|
809 | return | |
| 810 |
|
810 | |||
| 811 | def putData(self): |
|
811 | def putData(self): | |
| 812 |
|
812 | |||
| 813 | if self.blockIndex == self.blocksPerFile or self.timeFlag(): |
|
813 | if self.blockIndex == self.blocksPerFile or self.timeFlag(): | |
| 814 | self.setNextFile() |
|
814 | self.setNextFile() | |
| 815 |
|
815 | |||
| 816 | self.readBlock() |
|
816 | self.readBlock() | |
| 817 | self.setBlock() #Prepare data to be written |
|
817 | self.setBlock() #Prepare data to be written | |
| 818 | self.writeBlock() #Write data |
|
818 | self.writeBlock() #Write data | |
| 819 |
|
819 | |||
| 820 | return |
|
820 | return | |
| 821 |
|
821 | |||
| 822 | def readBlock(self): |
|
822 | def readBlock(self): | |
| 823 |
|
823 | |||
| 824 | ''' |
|
824 | ''' | |
| 825 | data Array configured |
|
825 | data Array configured | |
| 826 |
|
826 | |||
| 827 |
|
827 | |||
| 828 | self.data |
|
828 | self.data | |
| 829 | ''' |
|
829 | ''' | |
| 830 | dsList = self.dsList |
|
830 | dsList = self.dsList | |
| 831 | ds = self.ds |
|
831 | ds = self.ds | |
| 832 | #Setting HDF5 File |
|
832 | #Setting HDF5 File | |
| 833 | fp = h5py.File(self.filename,'r+') |
|
833 | fp = h5py.File(self.filename,'r+') | |
| 834 | grp = fp["Data"] |
|
834 | grp = fp["Data"] | |
| 835 | ind = 0 |
|
835 | ind = 0 | |
| 836 |
|
836 | |||
| 837 | while ind < len(dsList): |
|
837 | while ind < len(dsList): | |
| 838 | dsInfo = dsList[ind] |
|
838 | dsInfo = dsList[ind] | |
| 839 |
|
839 | |||
| 840 | if dsInfo['mode'] == 0: |
|
840 | if dsInfo['mode'] == 0: | |
| 841 | ds0 = grp[dsInfo['variable']] |
|
841 | ds0 = grp[dsInfo['variable']] | |
| 842 | ds[ind] = ds0 |
|
842 | ds[ind] = ds0 | |
| 843 | ind += 1 |
|
843 | ind += 1 | |
| 844 | else: |
|
844 | else: | |
| 845 |
|
845 | |||
| 846 | grp0 = grp[dsInfo['variable']] |
|
846 | grp0 = grp[dsInfo['variable']] | |
| 847 |
|
847 | |||
| 848 | for j in range(dsInfo['dsNumber']): |
|
848 | for j in range(dsInfo['dsNumber']): | |
| 849 | dsInfo = dsList[ind] |
|
849 | dsInfo = dsList[ind] | |
| 850 | ds0 = grp0[dsInfo['dsName']] |
|
850 | ds0 = grp0[dsInfo['dsName']] | |
| 851 | ds[ind] = ds0 |
|
851 | ds[ind] = ds0 | |
| 852 | ind += 1 |
|
852 | ind += 1 | |
| 853 |
|
853 | |||
| 854 | self.fp = fp |
|
854 | self.fp = fp | |
| 855 | self.grp = grp |
|
855 | self.grp = grp | |
| 856 | self.ds = ds |
|
856 | self.ds = ds | |
| 857 |
|
857 | |||
| 858 | return |
|
858 | return | |
| 859 |
|
859 | |||
| 860 | def setBlock(self): |
|
860 | def setBlock(self): | |
| 861 | ''' |
|
861 | ''' | |
| 862 | data Array configured |
|
862 | data Array configured | |
| 863 |
|
863 | |||
| 864 |
|
864 | |||
| 865 | self.data |
|
865 | self.data | |
| 866 | ''' |
|
866 | ''' | |
| 867 | #Creating Arrays |
|
867 | #Creating Arrays | |
| 868 | dsList = self.dsList |
|
868 | dsList = self.dsList | |
| 869 | data = self.data |
|
869 | data = self.data | |
| 870 | ind = 0 |
|
870 | ind = 0 | |
| 871 |
|
871 | |||
| 872 | while ind < len(dsList): |
|
872 | while ind < len(dsList): | |
| 873 | dsInfo = dsList[ind] |
|
873 | dsInfo = dsList[ind] | |
| 874 | dataAux = getattr(self.dataOut, dsInfo['variable']) |
|
874 | dataAux = getattr(self.dataOut, dsInfo['variable']) | |
| 875 |
|
875 | |||
| 876 | mode = dsInfo['mode'] |
|
876 | mode = dsInfo['mode'] | |
| 877 | nDim = dsInfo['nDim'] |
|
877 | nDim = dsInfo['nDim'] | |
| 878 |
|
878 | |||
| 879 | if mode == 0 or mode == 2 or nDim == 1: |
|
879 | if mode == 0 or mode == 2 or nDim == 1: | |
| 880 | data[ind] = dataAux |
|
880 | data[ind] = dataAux | |
| 881 | ind += 1 |
|
881 | ind += 1 | |
| 882 | # elif nDim == 1: |
|
882 | # elif nDim == 1: | |
| 883 | # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1)) |
|
883 | # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1)) | |
| 884 | # ind += 1 |
|
884 | # ind += 1 | |
| 885 | elif nDim == 2: |
|
885 | elif nDim == 2: | |
| 886 | for j in range(dsInfo['dsNumber']): |
|
886 | for j in range(dsInfo['dsNumber']): | |
| 887 | data[ind] = dataAux[j,:] |
|
887 | data[ind] = dataAux[j,:] | |
| 888 | ind += 1 |
|
888 | ind += 1 | |
| 889 | elif nDim == 3: |
|
889 | elif nDim == 3: | |
| 890 | for j in range(dsInfo['dsNumber']): |
|
890 | for j in range(dsInfo['dsNumber']): | |
| 891 | data[ind] = dataAux[:,j,:] |
|
891 | data[ind] = dataAux[:,j,:] | |
| 892 | ind += 1 |
|
892 | ind += 1 | |
| 893 |
|
893 | |||
| 894 | self.data = data |
|
894 | self.data = data | |
| 895 | return |
|
895 | return | |
| 896 |
|
896 | |||
| 897 | def writeBlock(self): |
|
897 | def writeBlock(self): | |
| 898 | ''' |
|
898 | ''' | |
| 899 | Saves the block in the HDF5 file |
|
899 | Saves the block in the HDF5 file | |
| 900 | ''' |
|
900 | ''' | |
| 901 | dsList = self.dsList |
|
901 | dsList = self.dsList | |
| 902 |
|
902 | |||
| 903 | for i in range(len(self.ds)): |
|
903 | for i in range(len(self.ds)): | |
| 904 | dsInfo = dsList[i] |
|
904 | dsInfo = dsList[i] | |
| 905 | nDim = dsInfo['nDim'] |
|
905 | nDim = dsInfo['nDim'] | |
| 906 | mode = dsInfo['mode'] |
|
906 | mode = dsInfo['mode'] | |
| 907 |
|
907 | |||
| 908 | # First time |
|
908 | # First time | |
| 909 | if self.firsttime: |
|
909 | if self.firsttime: | |
| 910 | if type(self.data[i]) == numpy.ndarray: |
|
910 | if type(self.data[i]) == numpy.ndarray: | |
| 911 |
|
911 | |||
| 912 | if nDim == 3: |
|
912 | if nDim == 3: | |
| 913 | self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1)) |
|
913 | self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1)) | |
| 914 | self.ds[i].resize(self.data[i].shape) |
|
914 | self.ds[i].resize(self.data[i].shape) | |
| 915 | if mode == 2: |
|
915 | if mode == 2: | |
| 916 | self.ds[i].resize(self.data[i].shape) |
|
916 | self.ds[i].resize(self.data[i].shape) | |
| 917 | self.ds[i][:] = self.data[i] |
|
917 | self.ds[i][:] = self.data[i] | |
| 918 | else: |
|
918 | else: | |
| 919 |
|
919 | |||
| 920 | # From second time |
|
920 | # From second time | |
| 921 | # Meteors! |
|
921 | # Meteors! | |
| 922 | if mode == 2: |
|
922 | if mode == 2: | |
| 923 | dataShape = self.data[i].shape |
|
923 | dataShape = self.data[i].shape | |
| 924 | dsShape = self.ds[i].shape |
|
924 | dsShape = self.ds[i].shape | |
| 925 | self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1])) |
|
925 | self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1])) | |
| 926 | self.ds[i][dsShape[0]:,:] = self.data[i] |
|
926 | self.ds[i][dsShape[0]:,:] = self.data[i] | |
| 927 | # No dimension |
|
927 | # No dimension | |
| 928 | elif mode == 0: |
|
928 | elif mode == 0: | |
| 929 | self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1)) |
|
929 | self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1)) | |
| 930 | self.ds[i][0,-1] = self.data[i] |
|
930 | self.ds[i][0,-1] = self.data[i] | |
| 931 | # One dimension |
|
931 | # One dimension | |
| 932 | elif nDim == 1: |
|
932 | elif nDim == 1: | |
| 933 | self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1])) |
|
933 | self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1])) | |
| 934 | self.ds[i][-1,:] = self.data[i] |
|
934 | self.ds[i][-1,:] = self.data[i] | |
| 935 | # Two dimension |
|
935 | # Two dimension | |
| 936 | elif nDim == 2: |
|
936 | elif nDim == 2: | |
| 937 | self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1])) |
|
937 | self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1])) | |
| 938 | self.ds[i][self.blockIndex,:] = self.data[i] |
|
938 | self.ds[i][self.blockIndex,:] = self.data[i] | |
| 939 | # Three dimensions |
|
939 | # Three dimensions | |
| 940 | elif nDim == 3: |
|
940 | elif nDim == 3: | |
| 941 | self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1)) |
|
941 | self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1)) | |
| 942 | self.ds[i][:,:,-1] = self.data[i] |
|
942 | self.ds[i][:,:,-1] = self.data[i] | |
| 943 |
|
943 | |||
| 944 | self.firsttime = False |
|
944 | self.firsttime = False | |
| 945 | self.blockIndex += 1 |
|
945 | self.blockIndex += 1 | |
| 946 |
|
946 | |||
| 947 | #Close to save changes |
|
947 | #Close to save changes | |
| 948 | self.fp.flush() |
|
948 | self.fp.flush() | |
| 949 | self.fp.close() |
|
949 | self.fp.close() | |
| 950 | return |
|
950 | return | |
| 951 |
|
951 | |||
| 952 | def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None): |
|
952 | def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None): | |
| 953 |
|
953 | |||
| 954 | self.dataOut = dataOut |
|
954 | self.dataOut = dataOut | |
| 955 | if not(self.isConfig): |
|
955 | if not(self.isConfig): | |
| 956 | self.setup(dataOut, path=path, blocksPerFile=blocksPerFile, |
|
956 | self.setup(dataOut, path=path, blocksPerFile=blocksPerFile, | |
| 957 | metadataList=metadataList, dataList=dataList, mode=mode, |
|
957 | metadataList=metadataList, dataList=dataList, mode=mode, | |
| 958 | setType=setType) |
|
958 | setType=setType) | |
| 959 |
|
959 | |||
| 960 | self.isConfig = True |
|
960 | self.isConfig = True | |
| 961 | self.setNextFile() |
|
961 | self.setNextFile() | |
| 962 |
|
962 | |||
| 963 | self.putData() |
|
963 | self.putData() | |
| 964 | return |
|
964 | return | |
| 965 |
|
965 | |||
| 966 |
|
966 | |||
| 967 | @MPDecorator |
|
967 | @MPDecorator | |
| 968 | class ParameterReader(JRODataReader,ProcessingUnit): |
|
968 | class ParameterReader(JRODataReader,ProcessingUnit): | |
| 969 | ''' |
|
969 | ''' | |
| 970 | Reads HDF5 format files |
|
970 | Reads HDF5 format files | |
| 971 | ''' |
|
971 | ''' | |
| 972 |
|
972 | |||
| 973 | ext = ".hdf5" |
|
973 | ext = ".hdf5" | |
| 974 | optchar = "D" |
|
974 | optchar = "D" | |
| 975 | timezone = None |
|
975 | timezone = None | |
| 976 | startTime = None |
|
976 | startTime = None | |
| 977 | endTime = None |
|
977 | endTime = None | |
| 978 | fileIndex = None |
|
978 | fileIndex = None | |
| 979 | blockList = None #List to blocks to be read from the file |
|
979 | blockList = None #List to blocks to be read from the file | |
| 980 | blocksPerFile = None #Number of blocks to be read |
|
980 | blocksPerFile = None #Number of blocks to be read | |
| 981 | blockIndex = None |
|
981 | blockIndex = None | |
| 982 | path = None |
|
982 | path = None | |
| 983 | #List of Files |
|
983 | #List of Files | |
| 984 | filenameList = None |
|
984 | filenameList = None | |
| 985 | datetimeList = None |
|
985 | datetimeList = None | |
| 986 | #Hdf5 File |
|
986 | #Hdf5 File | |
| 987 | listMetaname = None |
|
987 | listMetaname = None | |
| 988 | listMeta = None |
|
988 | listMeta = None | |
| 989 | listDataname = None |
|
989 | listDataname = None | |
| 990 | listData = None |
|
990 | listData = None | |
| 991 | listShapes = None |
|
991 | listShapes = None | |
| 992 | fp = None |
|
992 | fp = None | |
| 993 | #dataOut reconstruction |
|
993 | #dataOut reconstruction | |
| 994 | dataOut = None |
|
994 | dataOut = None | |
| 995 |
|
995 | |||
| 996 | def __init__(self): |
|
996 | def __init__(self): | |
| 997 | ProcessingUnit.__init__(self) |
|
997 | ProcessingUnit.__init__(self) | |
| 998 | self.dataOut = Parameters() |
|
998 | self.dataOut = Parameters() | |
| 999 | return |
|
999 | return | |
| 1000 |
|
1000 | |||
| 1001 | def setup(self, **kwargs): |
|
1001 | def setup(self, **kwargs): | |
| 1002 |
|
1002 | |||
| 1003 | path = kwargs['path'] |
|
1003 | path = kwargs['path'] | |
| 1004 | startDate = kwargs['startDate'] |
|
1004 | startDate = kwargs['startDate'] | |
| 1005 | endDate = kwargs['endDate'] |
|
1005 | endDate = kwargs['endDate'] | |
| 1006 | startTime = kwargs['startTime'] |
|
1006 | startTime = kwargs['startTime'] | |
| 1007 | endTime = kwargs['endTime'] |
|
1007 | endTime = kwargs['endTime'] | |
| 1008 | walk = kwargs['walk'] |
|
1008 | walk = kwargs['walk'] | |
| 1009 | if 'ext' in kwargs: |
|
1009 | if 'ext' in kwargs: | |
| 1010 | ext = kwargs['ext'] |
|
1010 | ext = kwargs['ext'] | |
| 1011 | else: |
|
1011 | else: | |
| 1012 | ext = '.hdf5' |
|
1012 | ext = '.hdf5' | |
| 1013 | if 'timezone' in kwargs: |
|
1013 | if 'timezone' in kwargs: | |
| 1014 | self.timezone = kwargs['timezone'] |
|
1014 | self.timezone = kwargs['timezone'] | |
| 1015 | else: |
|
1015 | else: | |
| 1016 | self.timezone = 'lt' |
|
1016 | self.timezone = 'lt' | |
| 1017 |
|
1017 | |||
| 1018 | print("[Reading] Searching files in offline mode ...") |
|
1018 | print("[Reading] Searching files in offline mode ...") | |
| 1019 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, |
|
1019 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, | |
| 1020 | startTime=startTime, endTime=endTime, |
|
1020 | startTime=startTime, endTime=endTime, | |
| 1021 | ext=ext, walk=walk) |
|
1021 | ext=ext, walk=walk) | |
| 1022 |
|
1022 | |||
| 1023 | if not(filenameList): |
|
1023 | if not(filenameList): | |
| 1024 | print("There is no files into the folder: %s"%(path)) |
|
1024 | print("There is no files into the folder: %s"%(path)) | |
| 1025 | sys.exit(-1) |
|
1025 | sys.exit(-1) | |
| 1026 |
|
1026 | |||
| 1027 | self.fileIndex = -1 |
|
1027 | self.fileIndex = -1 | |
| 1028 | self.startTime = startTime |
|
1028 | self.startTime = startTime | |
| 1029 | self.endTime = endTime |
|
1029 | self.endTime = endTime | |
| 1030 | self.__readMetadata() |
|
1030 | self.__readMetadata() | |
| 1031 | self.__setNextFileOffline() |
|
1031 | self.__setNextFileOffline() | |
| 1032 |
|
1032 | |||
| 1033 | return |
|
1033 | return | |
| 1034 |
|
1034 | |||
| 1035 | def searchFilesOffLine(self, path, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), ext='.hdf5', walk=True): |
|
1035 | def searchFilesOffLine(self, path, startDate=None, endDate=None, startTime=datetime.time(0,0,0), endTime=datetime.time(23,59,59), ext='.hdf5', walk=True): | |
| 1036 |
|
1036 | |||
| 1037 | expLabel = '' |
|
1037 | expLabel = '' | |
| 1038 | self.filenameList = [] |
|
1038 | self.filenameList = [] | |
| 1039 | self.datetimeList = [] |
|
1039 | self.datetimeList = [] | |
| 1040 | pathList = [] |
|
1040 | pathList = [] | |
| 1041 | dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) |
|
1041 | dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) | |
| 1042 |
|
1042 | |||
| 1043 | if dateList == []: |
|
1043 | if dateList == []: | |
| 1044 | print("[Reading] No *%s files in %s from %s to %s)"%(ext, path, |
|
1044 | print("[Reading] No *%s files in %s from %s to %s)"%(ext, path, | |
| 1045 | datetime.datetime.combine(startDate,startTime).ctime(), |
|
1045 | datetime.datetime.combine(startDate,startTime).ctime(), | |
| 1046 | datetime.datetime.combine(endDate,endTime).ctime())) |
|
1046 | datetime.datetime.combine(endDate,endTime).ctime())) | |
| 1047 |
|
1047 | |||
| 1048 | return None, None |
|
1048 | return None, None | |
| 1049 |
|
1049 | |||
| 1050 | if len(dateList) > 1: |
|
1050 | if len(dateList) > 1: | |
| 1051 | print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)) |
|
1051 | print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)) | |
| 1052 | else: |
|
1052 | else: | |
| 1053 | print("[Reading] data was found for the date %s" %(dateList[0])) |
|
1053 | print("[Reading] data was found for the date %s" %(dateList[0])) | |
| 1054 |
|
1054 | |||
| 1055 | filenameList = [] |
|
1055 | filenameList = [] | |
| 1056 | datetimeList = [] |
|
1056 | datetimeList = [] | |
| 1057 |
|
1057 | |||
| 1058 | for thisPath in pathList: |
|
1058 | for thisPath in pathList: | |
| 1059 |
|
1059 | |||
| 1060 | fileList = glob.glob1(thisPath, "*%s" %ext) |
|
1060 | fileList = glob.glob1(thisPath, "*%s" %ext) | |
| 1061 | fileList.sort() |
|
1061 | fileList.sort() | |
| 1062 |
|
1062 | |||
| 1063 | for file in fileList: |
|
1063 | for file in fileList: | |
| 1064 |
|
1064 | |||
| 1065 | filename = os.path.join(thisPath,file) |
|
1065 | filename = os.path.join(thisPath,file) | |
| 1066 |
|
1066 | |||
| 1067 | if not isFileInDateRange(filename, startDate, endDate): |
|
1067 | if not isFileInDateRange(filename, startDate, endDate): | |
| 1068 | continue |
|
1068 | continue | |
| 1069 |
|
1069 | |||
| 1070 | thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime) |
|
1070 | thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime) | |
| 1071 |
|
1071 | |||
| 1072 | if not(thisDatetime): |
|
1072 | if not(thisDatetime): | |
| 1073 | continue |
|
1073 | continue | |
| 1074 |
|
1074 | |||
| 1075 | filenameList.append(filename) |
|
1075 | filenameList.append(filename) | |
| 1076 | datetimeList.append(thisDatetime) |
|
1076 | datetimeList.append(thisDatetime) | |
| 1077 |
|
1077 | |||
| 1078 | if not(filenameList): |
|
1078 | if not(filenameList): | |
| 1079 | print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())) |
|
1079 | print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())) | |
| 1080 | return None, None |
|
1080 | return None, None | |
| 1081 |
|
1081 | |||
| 1082 | print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)) |
|
1082 | print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)) | |
| 1083 | print() |
|
1083 | print() | |
| 1084 |
|
1084 | |||
| 1085 | self.filenameList = filenameList |
|
1085 | self.filenameList = filenameList | |
| 1086 | self.datetimeList = datetimeList |
|
1086 | self.datetimeList = datetimeList | |
| 1087 |
|
1087 | |||
| 1088 | return pathList, filenameList |
|
1088 | return pathList, filenameList | |
| 1089 |
|
1089 | |||
| 1090 | def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime): |
|
1090 | def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime): | |
| 1091 |
|
1091 | |||
| 1092 | """ |
|
1092 | """ | |
| 1093 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. |
|
1093 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |
| 1094 |
|
1094 | |||
| 1095 | Inputs: |
|
1095 | Inputs: | |
| 1096 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) |
|
1096 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |
| 1097 | startDate : fecha inicial del rango seleccionado en formato datetime.date |
|
1097 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| 1098 | endDate : fecha final del rango seleccionado en formato datetime.date |
|
1098 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| 1099 | startTime : tiempo inicial del rango seleccionado en formato datetime.time |
|
1099 | startTime : tiempo inicial del rango seleccionado en formato datetime.time | |
| 1100 | endTime : tiempo final del rango seleccionado en formato datetime.time |
|
1100 | endTime : tiempo final del rango seleccionado en formato datetime.time | |
| 1101 |
|
1101 | |||
| 1102 | Return: |
|
1102 | Return: | |
| 1103 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de |
|
1103 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |
| 1104 | fecha especificado, de lo contrario retorna False. |
|
1104 | fecha especificado, de lo contrario retorna False. | |
| 1105 |
|
1105 | |||
| 1106 | Excepciones: |
|
1106 | Excepciones: | |
| 1107 | Si el archivo no existe o no puede ser abierto |
|
1107 | Si el archivo no existe o no puede ser abierto | |
| 1108 | Si la cabecera no puede ser leida. |
|
1108 | Si la cabecera no puede ser leida. | |
| 1109 |
|
1109 | |||
| 1110 | """ |
|
1110 | """ | |
| 1111 |
|
1111 | |||
| 1112 | try: |
|
1112 | try: | |
| 1113 | fp = h5py.File(filename, 'r') |
|
1113 | fp = h5py.File(filename, 'r') | |
| 1114 | grp1 = fp['Data'] |
|
1114 | grp1 = fp['Data'] | |
| 1115 |
|
1115 | |||
| 1116 | except IOError: |
|
1116 | except IOError: | |
| 1117 | traceback.print_exc() |
|
1117 | traceback.print_exc() | |
| 1118 | raise IOError("The file %s can't be opened" %(filename)) |
|
1118 | raise IOError("The file %s can't be opened" %(filename)) | |
| 1119 | #In case has utctime attribute |
|
1119 | #In case has utctime attribute | |
| 1120 | grp2 = grp1['utctime'] |
|
1120 | grp2 = grp1['utctime'] | |
| 1121 | thisUtcTime = grp2.value[0] |
|
1121 | thisUtcTime = grp2.value[0] | |
| 1122 |
|
1122 | |||
| 1123 | fp.close() |
|
1123 | fp.close() | |
| 1124 |
|
1124 | |||
| 1125 | if self.timezone == 'lt': |
|
1125 | if self.timezone == 'lt': | |
| 1126 | thisUtcTime -= 5*3600 |
|
1126 | thisUtcTime -= 5*3600 | |
| 1127 |
|
1127 | |||
| 1128 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600) |
|
1128 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600) | |
| 1129 | thisDate = thisDatetime.date() |
|
1129 | thisDate = thisDatetime.date() | |
| 1130 | thisTime = thisDatetime.time() |
|
1130 | thisTime = thisDatetime.time() | |
| 1131 |
|
1131 | |||
| 1132 | startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds() |
|
1132 | startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds() | |
| 1133 | endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds() |
|
1133 | endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds() | |
| 1134 |
|
1134 | |||
| 1135 | #General case |
|
1135 | #General case | |
| 1136 | # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o |
|
1136 | # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o | |
| 1137 | #-----------o----------------------------o----------- |
|
1137 | #-----------o----------------------------o----------- | |
| 1138 | # startTime endTime |
|
1138 | # startTime endTime | |
| 1139 |
|
1139 | |||
| 1140 | if endTime >= startTime: |
|
1140 | if endTime >= startTime: | |
| 1141 | thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime) |
|
1141 | thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime) | |
| 1142 | if numpy.any(thisUtcLog): #If there is one block between the hours mentioned |
|
1142 | if numpy.any(thisUtcLog): #If there is one block between the hours mentioned | |
| 1143 | return thisDatetime |
|
1143 | return thisDatetime | |
| 1144 | return None |
|
1144 | return None | |
| 1145 |
|
1145 | |||
| 1146 | #If endTime < startTime then endTime belongs to the next day |
|
1146 | #If endTime < startTime then endTime belongs to the next day | |
| 1147 | #<<<<<<<<<<<o o>>>>>>>>>>> |
|
1147 | #<<<<<<<<<<<o o>>>>>>>>>>> | |
| 1148 | #-----------o----------------------------o----------- |
|
1148 | #-----------o----------------------------o----------- | |
| 1149 | # endTime startTime |
|
1149 | # endTime startTime | |
| 1150 |
|
1150 | |||
| 1151 | if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime): |
|
1151 | if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime): | |
| 1152 | return None |
|
1152 | return None | |
| 1153 |
|
1153 | |||
| 1154 | if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime): |
|
1154 | if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime): | |
| 1155 | return None |
|
1155 | return None | |
| 1156 |
|
1156 | |||
| 1157 | if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime): |
|
1157 | if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime): | |
| 1158 | return None |
|
1158 | return None | |
| 1159 |
|
1159 | |||
| 1160 | return thisDatetime |
|
1160 | return thisDatetime | |
| 1161 |
|
1161 | |||
| 1162 | def __setNextFileOffline(self): |
|
1162 | def __setNextFileOffline(self): | |
| 1163 |
|
1163 | |||
| 1164 | self.fileIndex += 1 |
|
1164 | self.fileIndex += 1 | |
| 1165 | idFile = self.fileIndex |
|
1165 | idFile = self.fileIndex | |
| 1166 |
|
1166 | |||
| 1167 | if not(idFile < len(self.filenameList)): |
|
1167 | if not(idFile < len(self.filenameList)): | |
| 1168 | self.dataOut.error = 'No more files' |
|
1168 | self.dataOut.error = 'No more files' | |
| 1169 | return 0 |
|
1169 | return 0 | |
| 1170 |
|
1170 | |||
| 1171 | filename = self.filenameList[idFile] |
|
1171 | filename = self.filenameList[idFile] | |
| 1172 | self.fp = h5py.File(filename, 'r') |
|
1172 | self.fp = h5py.File(filename, 'r') | |
| 1173 | self.filename = filename |
|
1173 | self.filename = filename | |
| 1174 |
|
1174 | |||
| 1175 | print("Setting the file: %s"%self.filename) |
|
1175 | print("Setting the file: %s"%self.filename) | |
| 1176 |
|
1176 | |||
| 1177 | self.__setBlockList() |
|
1177 | self.__setBlockList() | |
| 1178 | self.__readData() |
|
1178 | self.__readData() | |
| 1179 | self.blockIndex = 0 |
|
1179 | self.blockIndex = 0 | |
| 1180 | return 1 |
|
1180 | return 1 | |
| 1181 |
|
1181 | |||
| 1182 | def __setBlockList(self): |
|
1182 | def __setBlockList(self): | |
| 1183 | ''' |
|
1183 | ''' | |
| 1184 | Selects the data within the times defined |
|
1184 | Selects the data within the times defined | |
| 1185 |
|
1185 | |||
| 1186 | self.fp |
|
1186 | self.fp | |
| 1187 | self.startTime |
|
1187 | self.startTime | |
| 1188 | self.endTime |
|
1188 | self.endTime | |
| 1189 | self.blockList |
|
1189 | self.blockList | |
| 1190 | self.blocksPerFile |
|
1190 | self.blocksPerFile | |
| 1191 |
|
1191 | |||
| 1192 | ''' |
|
1192 | ''' | |
| 1193 | fp = self.fp |
|
1193 | fp = self.fp | |
| 1194 | startTime = self.startTime |
|
1194 | startTime = self.startTime | |
| 1195 | endTime = self.endTime |
|
1195 | endTime = self.endTime | |
| 1196 |
|
1196 | |||
| 1197 | grp = fp['Data'] |
|
1197 | grp = fp['Data'] | |
| 1198 |
thisUtcTime = grp['utctime'].value |
|
1198 | thisUtcTime = grp['utctime'].value | |
| 1199 |
|
1199 | |||
| 1200 | if self.timezone == 'lt': |
|
1200 | if self.timezone == 'lt': | |
| 1201 | thisUtcTime -= 5*3600 |
|
1201 | thisUtcTime -= 5*3600 | |
| 1202 |
|
1202 | |||
| 1203 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime + 5*3600) |
|
1203 | thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600) | |
| 1204 |
|
1204 | |||
| 1205 | thisDate = thisDatetime.date() |
|
1205 | thisDate = thisDatetime.date() | |
| 1206 | thisTime = thisDatetime.time() |
|
1206 | thisTime = thisDatetime.time() | |
| 1207 |
|
1207 | |||
| 1208 | startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds() |
|
1208 | startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds() | |
| 1209 | endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds() |
|
1209 | endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds() | |
| 1210 |
|
1210 | |||
| 1211 | ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] |
|
1211 | ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0] | |
| 1212 |
|
1212 | |||
| 1213 | self.blockList = ind |
|
1213 | self.blockList = ind | |
| 1214 | self.blocksPerFile = len(ind) |
|
1214 | self.blocksPerFile = len(ind) | |
| 1215 |
|
||||
| 1216 | return |
|
1215 | return | |
| 1217 |
|
1216 | |||
| 1218 | def __readMetadata(self): |
|
1217 | def __readMetadata(self): | |
| 1219 | ''' |
|
1218 | ''' | |
| 1220 | Reads Metadata |
|
1219 | Reads Metadata | |
| 1221 | ''' |
|
1220 | ''' | |
| 1222 |
|
1221 | |||
| 1223 | filename = self.filenameList[0] |
|
1222 | filename = self.filenameList[0] | |
| 1224 | fp = h5py.File(filename, 'r') |
|
1223 | fp = h5py.File(filename, 'r') | |
| 1225 | gp = fp['Metadata'] |
|
1224 | gp = fp['Metadata'] | |
| 1226 | listMetaname = [] |
|
1225 | listMetaname = [] | |
| 1227 | listMetadata = [] |
|
1226 | listMetadata = [] | |
| 1228 |
|
1227 | |||
| 1229 | for item in list(gp.items()): |
|
1228 | for item in list(gp.items()): | |
| 1230 | name = item[0] |
|
1229 | name = item[0] | |
| 1231 |
|
1230 | |||
| 1232 | if name=='variables': |
|
1231 | if name=='variables': | |
| 1233 | table = gp[name][:] |
|
1232 | table = gp[name][:] | |
| 1234 | listShapes = {} |
|
1233 | listShapes = {} | |
| 1235 | for shapes in table: |
|
1234 | for shapes in table: | |
| 1236 | listShapes[shapes[0].decode()] = numpy.array([shapes[1]]) |
|
1235 | listShapes[shapes[0].decode()] = numpy.array([shapes[1]]) | |
| 1237 | else: |
|
1236 | else: | |
| 1238 | data = gp[name].value |
|
1237 | data = gp[name].value | |
| 1239 | listMetaname.append(name) |
|
1238 | listMetaname.append(name) | |
| 1240 | listMetadata.append(data) |
|
1239 | listMetadata.append(data) | |
| 1241 |
|
1240 | |||
| 1242 | self.listShapes = listShapes |
|
1241 | self.listShapes = listShapes | |
| 1243 | self.listMetaname = listMetaname |
|
1242 | self.listMetaname = listMetaname | |
| 1244 | self.listMeta = listMetadata |
|
1243 | self.listMeta = listMetadata | |
| 1245 |
|
1244 | |||
| 1246 | fp.close() |
|
1245 | fp.close() | |
| 1247 | return |
|
1246 | return | |
| 1248 |
|
1247 | |||
| 1249 | def __readData(self): |
|
1248 | def __readData(self): | |
| 1250 |
|
1249 | |||
| 1251 | grp = self.fp['Data'] |
|
1250 | grp = self.fp['Data'] | |
| 1252 | listdataname = [] |
|
1251 | listdataname = [] | |
| 1253 | listdata = [] |
|
1252 | listdata = [] | |
| 1254 |
|
1253 | |||
| 1255 | for item in list(grp.items()): |
|
1254 | for item in list(grp.items()): | |
| 1256 | name = item[0] |
|
1255 | name = item[0] | |
| 1257 | listdataname.append(name) |
|
1256 | listdataname.append(name) | |
| 1258 | dim = self.listShapes[name][0] |
|
1257 | dim = self.listShapes[name][0] | |
| 1259 | if dim == 0: |
|
1258 | if dim == 0: | |
| 1260 | array = grp[name].value |
|
1259 | array = grp[name].value | |
| 1261 | else: |
|
1260 | else: | |
| 1262 | array = [] |
|
1261 | array = [] | |
| 1263 | for i in range(dim): |
|
1262 | for i in range(dim): | |
| 1264 | array.append(grp[name]['table{:02d}'.format(i)].value) |
|
1263 | array.append(grp[name]['table{:02d}'.format(i)].value) | |
| 1265 | array = numpy.array(array) |
|
1264 | array = numpy.array(array) | |
| 1266 |
|
1265 | |||
| 1267 | listdata.append(array) |
|
1266 | listdata.append(array) | |
| 1268 |
|
1267 | |||
| 1269 | self.listDataname = listdataname |
|
1268 | self.listDataname = listdataname | |
| 1270 | self.listData = listdata |
|
1269 | self.listData = listdata | |
| 1271 | return |
|
1270 | return | |
| 1272 |
|
1271 | |||
| 1273 | def getData(self): |
|
1272 | def getData(self): | |
| 1274 |
|
1273 | |||
| 1275 | for i in range(len(self.listMeta)): |
|
1274 | for i in range(len(self.listMeta)): | |
| 1276 | setattr(self.dataOut, self.listMetaname[i], self.listMeta[i]) |
|
1275 | setattr(self.dataOut, self.listMetaname[i], self.listMeta[i]) | |
| 1277 |
|
1276 | |||
| 1278 | for j in range(len(self.listData)): |
|
1277 | for j in range(len(self.listData)): | |
| 1279 | dim = self.listShapes[self.listDataname[j]][0] |
|
1278 | dim = self.listShapes[self.listDataname[j]][0] | |
| 1280 | if dim == 0: |
|
1279 | if dim == 0: | |
| 1281 | setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex]) |
|
1280 | setattr(self.dataOut, self.listDataname[j], self.listData[j][self.blockIndex]) | |
| 1282 | else: |
|
1281 | else: | |
| 1283 | setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex]) |
|
1282 | setattr(self.dataOut, self.listDataname[j], self.listData[j][:,self.blockIndex]) | |
| 1284 |
|
1283 | |||
| 1285 | self.dataOut.flagNoData = False |
|
1284 | self.dataOut.flagNoData = False | |
| 1286 | self.blockIndex += 1 |
|
1285 | self.blockIndex += 1 | |
| 1287 |
|
1286 | |||
| 1288 | return |
|
1287 | return | |
| 1289 |
|
1288 | |||
| 1290 | def run(self, **kwargs): |
|
1289 | def run(self, **kwargs): | |
| 1291 |
|
1290 | |||
| 1292 | if not(self.isConfig): |
|
1291 | if not(self.isConfig): | |
| 1293 | self.setup(**kwargs) |
|
1292 | self.setup(**kwargs) | |
| 1294 | self.isConfig = True |
|
1293 | self.isConfig = True | |
| 1295 |
|
1294 | |||
| 1296 | if self.blockIndex == self.blocksPerFile: |
|
1295 | if self.blockIndex == self.blocksPerFile: | |
| 1297 |
|
|
1296 | if not(self.__setNextFileOffline()): | |
| 1298 | self.dataOut.flagNoData = True |
|
1297 | self.dataOut.flagNoData = True | |
| 1299 | return 0 |
|
1298 | return 0 | |
| 1300 |
|
1299 | |||
| 1301 | self.getData() |
|
1300 | self.getData() | |
| 1302 |
|
1301 | |||
| 1303 | return |
|
1302 | return | |
| 1304 |
|
1303 | |||
| 1305 | @MPDecorator |
|
1304 | @MPDecorator | |
| 1306 | class ParameterWriter(Operation): |
|
1305 | class ParameterWriter(Operation): | |
| 1307 | ''' |
|
1306 | ''' | |
| 1308 | HDF5 Writer, stores parameters data in HDF5 format files |
|
1307 | HDF5 Writer, stores parameters data in HDF5 format files | |
| 1309 |
|
1308 | |||
| 1310 | path: path where the files will be stored |
|
1309 | path: path where the files will be stored | |
| 1311 | blocksPerFile: number of blocks that will be saved in per HDF5 format file |
|
1310 | blocksPerFile: number of blocks that will be saved in per HDF5 format file | |
| 1312 | mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) |
|
1311 | mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors) | |
| 1313 | metadataList: list of attributes that will be stored as metadata |
|
1312 | metadataList: list of attributes that will be stored as metadata | |
| 1314 | dataList: list of attributes that will be stores as data |
|
1313 | dataList: list of attributes that will be stores as data | |
| 1315 | ''' |
|
1314 | ''' | |
| 1316 |
|
1315 | |||
| 1317 |
|
1316 | |||
| 1318 | ext = ".hdf5" |
|
1317 | ext = ".hdf5" | |
| 1319 | optchar = "D" |
|
1318 | optchar = "D" | |
| 1320 | metaoptchar = "M" |
|
1319 | metaoptchar = "M" | |
| 1321 | metaFile = None |
|
1320 | metaFile = None | |
| 1322 | filename = None |
|
1321 | filename = None | |
| 1323 | path = None |
|
1322 | path = None | |
| 1324 | setFile = None |
|
1323 | setFile = None | |
| 1325 | fp = None |
|
1324 | fp = None | |
| 1326 | grp = None |
|
1325 | grp = None | |
| 1327 | ds = None |
|
1326 | ds = None | |
| 1328 | firsttime = True |
|
1327 | firsttime = True | |
| 1329 | #Configurations |
|
1328 | #Configurations | |
| 1330 | blocksPerFile = None |
|
1329 | blocksPerFile = None | |
| 1331 | blockIndex = None |
|
1330 | blockIndex = None | |
| 1332 | dataOut = None |
|
1331 | dataOut = None | |
| 1333 | #Data Arrays |
|
1332 | #Data Arrays | |
| 1334 | dataList = None |
|
1333 | dataList = None | |
| 1335 | metadataList = None |
|
1334 | metadataList = None | |
| 1336 | dsList = None #List of dictionaries with dataset properties |
|
1335 | dsList = None #List of dictionaries with dataset properties | |
| 1337 | tableDim = None |
|
1336 | tableDim = None | |
| 1338 | dtype = [('name', 'S20'),('nDim', 'i')] |
|
1337 | dtype = [('name', 'S20'),('nDim', 'i')] | |
| 1339 | currentDay = None |
|
1338 | currentDay = None | |
| 1340 | lastTime = None |
|
1339 | lastTime = None | |
| 1341 |
|
1340 | |||
| 1342 | def __init__(self): |
|
1341 | def __init__(self): | |
| 1343 |
|
1342 | |||
| 1344 | Operation.__init__(self) |
|
1343 | Operation.__init__(self) | |
| 1345 | return |
|
1344 | return | |
| 1346 |
|
1345 | |||
| 1347 | def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None): |
|
1346 | def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None): | |
| 1348 | self.path = path |
|
1347 | self.path = path | |
| 1349 | self.blocksPerFile = blocksPerFile |
|
1348 | self.blocksPerFile = blocksPerFile | |
| 1350 | self.metadataList = metadataList |
|
1349 | self.metadataList = metadataList | |
| 1351 | self.dataList = dataList |
|
1350 | self.dataList = dataList | |
| 1352 | self.setType = setType |
|
1351 | self.setType = setType | |
| 1353 |
|
1352 | |||
| 1354 | tableList = [] |
|
1353 | tableList = [] | |
| 1355 | dsList = [] |
|
1354 | dsList = [] | |
| 1356 |
|
1355 | |||
| 1357 | for i in range(len(self.dataList)): |
|
1356 | for i in range(len(self.dataList)): | |
| 1358 | dsDict = {} |
|
1357 | dsDict = {} | |
| 1359 | dataAux = getattr(self.dataOut, self.dataList[i]) |
|
1358 | dataAux = getattr(self.dataOut, self.dataList[i]) | |
| 1360 | dsDict['variable'] = self.dataList[i] |
|
1359 | dsDict['variable'] = self.dataList[i] | |
| 1361 |
|
1360 | |||
| 1362 | if dataAux is None: |
|
1361 | if dataAux is None: | |
| 1363 | continue |
|
1362 | continue | |
| 1364 | elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)): |
|
1363 | elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)): | |
| 1365 | dsDict['nDim'] = 0 |
|
1364 | dsDict['nDim'] = 0 | |
| 1366 | else: |
|
1365 | else: | |
| 1367 | dsDict['nDim'] = len(dataAux.shape) |
|
1366 | dsDict['nDim'] = len(dataAux.shape) | |
| 1368 | dsDict['shape'] = dataAux.shape |
|
1367 | dsDict['shape'] = dataAux.shape | |
| 1369 | dsDict['dsNumber'] = dataAux.shape[0] |
|
1368 | dsDict['dsNumber'] = dataAux.shape[0] | |
| 1370 |
|
1369 | |||
| 1371 | dsList.append(dsDict) |
|
1370 | dsList.append(dsDict) | |
| 1372 | tableList.append((self.dataList[i], dsDict['nDim'])) |
|
1371 | tableList.append((self.dataList[i], dsDict['nDim'])) | |
| 1373 |
|
1372 | |||
| 1374 | self.dsList = dsList |
|
1373 | self.dsList = dsList | |
| 1375 | self.tableDim = numpy.array(tableList, dtype=self.dtype) |
|
1374 | self.tableDim = numpy.array(tableList, dtype=self.dtype) | |
| 1376 | self.currentDay = self.dataOut.datatime.date() |
|
1375 | self.currentDay = self.dataOut.datatime.date() | |
| 1377 |
|
1376 | |||
| 1378 | def timeFlag(self): |
|
1377 | def timeFlag(self): | |
| 1379 | currentTime = self.dataOut.utctime |
|
1378 | currentTime = self.dataOut.utctime | |
| 1380 | timeTuple = time.localtime(currentTime) |
|
1379 | timeTuple = time.localtime(currentTime) | |
| 1381 | dataDay = timeTuple.tm_yday |
|
1380 | dataDay = timeTuple.tm_yday | |
| 1382 |
|
1381 | |||
| 1383 | if self.lastTime is None: |
|
1382 | if self.lastTime is None: | |
| 1384 | self.lastTime = currentTime |
|
1383 | self.lastTime = currentTime | |
| 1385 | self.currentDay = dataDay |
|
1384 | self.currentDay = dataDay | |
| 1386 | return False |
|
1385 | return False | |
| 1387 |
|
1386 | |||
| 1388 | timeDiff = currentTime - self.lastTime |
|
1387 | timeDiff = currentTime - self.lastTime | |
| 1389 |
|
1388 | |||
| 1390 | #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora |
|
1389 | #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora | |
| 1391 | if dataDay != self.currentDay: |
|
1390 | if dataDay != self.currentDay: | |
| 1392 | self.currentDay = dataDay |
|
1391 | self.currentDay = dataDay | |
| 1393 | return True |
|
1392 | return True | |
| 1394 | elif timeDiff > 3*60*60: |
|
1393 | elif timeDiff > 3*60*60: | |
| 1395 | self.lastTime = currentTime |
|
1394 | self.lastTime = currentTime | |
| 1396 | return True |
|
1395 | return True | |
| 1397 | else: |
|
1396 | else: | |
| 1398 | self.lastTime = currentTime |
|
1397 | self.lastTime = currentTime | |
| 1399 | return False |
|
1398 | return False | |
| 1400 |
|
1399 | |||
| 1401 | def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None): |
|
1400 | def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, setType=None): | |
| 1402 |
|
1401 | |||
| 1403 | self.dataOut = dataOut |
|
1402 | self.dataOut = dataOut | |
| 1404 | if not(self.isConfig): |
|
1403 | if not(self.isConfig): | |
| 1405 | self.setup(path=path, blocksPerFile=blocksPerFile, |
|
1404 | self.setup(path=path, blocksPerFile=blocksPerFile, | |
| 1406 | metadataList=metadataList, dataList=dataList, |
|
1405 | metadataList=metadataList, dataList=dataList, | |
| 1407 | setType=setType) |
|
1406 | setType=setType) | |
| 1408 |
|
1407 | |||
| 1409 | self.isConfig = True |
|
1408 | self.isConfig = True | |
| 1410 | self.setNextFile() |
|
1409 | self.setNextFile() | |
| 1411 |
|
1410 | |||
| 1412 | self.putData() |
|
1411 | self.putData() | |
| 1413 | return |
|
1412 | return | |
| 1414 |
|
1413 | |||
| 1415 | def setNextFile(self): |
|
1414 | def setNextFile(self): | |
| 1416 |
|
1415 | |||
| 1417 | ext = self.ext |
|
1416 | ext = self.ext | |
| 1418 | path = self.path |
|
1417 | path = self.path | |
| 1419 | setFile = self.setFile |
|
1418 | setFile = self.setFile | |
| 1420 |
|
1419 | |||
| 1421 | timeTuple = time.localtime(self.dataOut.utctime) |
|
1420 | timeTuple = time.localtime(self.dataOut.utctime) | |
| 1422 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) |
|
1421 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |
| 1423 | fullpath = os.path.join(path, subfolder) |
|
1422 | fullpath = os.path.join(path, subfolder) | |
| 1424 |
|
1423 | |||
| 1425 | if os.path.exists(fullpath): |
|
1424 | if os.path.exists(fullpath): | |
| 1426 | filesList = os.listdir(fullpath) |
|
1425 | filesList = os.listdir(fullpath) | |
| 1427 | filesList = [k for k in filesList if k.startswith(self.optchar)] |
|
1426 | filesList = [k for k in filesList if k.startswith(self.optchar)] | |
| 1428 | if len( filesList ) > 0: |
|
1427 | if len( filesList ) > 0: | |
| 1429 | filesList = sorted(filesList, key=str.lower) |
|
1428 | filesList = sorted(filesList, key=str.lower) | |
| 1430 | filen = filesList[-1] |
|
1429 | filen = filesList[-1] | |
| 1431 | # el filename debera tener el siguiente formato |
|
1430 | # el filename debera tener el siguiente formato | |
| 1432 | # 0 1234 567 89A BCDE (hex) |
|
1431 | # 0 1234 567 89A BCDE (hex) | |
| 1433 | # x YYYY DDD SSS .ext |
|
1432 | # x YYYY DDD SSS .ext | |
| 1434 | if isNumber(filen[8:11]): |
|
1433 | if isNumber(filen[8:11]): | |
| 1435 | setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file |
|
1434 | setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file | |
| 1436 | else: |
|
1435 | else: | |
| 1437 | setFile = -1 |
|
1436 | setFile = -1 | |
| 1438 | else: |
|
1437 | else: | |
| 1439 | setFile = -1 #inicializo mi contador de seteo |
|
1438 | setFile = -1 #inicializo mi contador de seteo | |
| 1440 | else: |
|
1439 | else: | |
| 1441 | os.makedirs(fullpath) |
|
1440 | os.makedirs(fullpath) | |
| 1442 | setFile = -1 #inicializo mi contador de seteo |
|
1441 | setFile = -1 #inicializo mi contador de seteo | |
| 1443 |
|
1442 | |||
| 1444 | if self.setType is None: |
|
1443 | if self.setType is None: | |
| 1445 | setFile += 1 |
|
1444 | setFile += 1 | |
| 1446 | file = '%s%4.4d%3.3d%03d%s' % (self.optchar, |
|
1445 | file = '%s%4.4d%3.3d%03d%s' % (self.optchar, | |
| 1447 | timeTuple.tm_year, |
|
1446 | timeTuple.tm_year, | |
| 1448 | timeTuple.tm_yday, |
|
1447 | timeTuple.tm_yday, | |
| 1449 | setFile, |
|
1448 | setFile, | |
| 1450 | ext ) |
|
1449 | ext ) | |
| 1451 | else: |
|
1450 | else: | |
| 1452 | setFile = timeTuple.tm_hour*60+timeTuple.tm_min |
|
1451 | setFile = timeTuple.tm_hour*60+timeTuple.tm_min | |
| 1453 | file = '%s%4.4d%3.3d%04d%s' % (self.optchar, |
|
1452 | file = '%s%4.4d%3.3d%04d%s' % (self.optchar, | |
| 1454 | timeTuple.tm_year, |
|
1453 | timeTuple.tm_year, | |
| 1455 | timeTuple.tm_yday, |
|
1454 | timeTuple.tm_yday, | |
| 1456 | setFile, |
|
1455 | setFile, | |
| 1457 | ext ) |
|
1456 | ext ) | |
| 1458 |
|
1457 | |||
| 1459 | self.filename = os.path.join( path, subfolder, file ) |
|
1458 | self.filename = os.path.join( path, subfolder, file ) | |
| 1460 |
|
1459 | |||
| 1461 | #Setting HDF5 File |
|
1460 | #Setting HDF5 File | |
| 1462 | self.fp = h5py.File(self.filename, 'w') |
|
1461 | self.fp = h5py.File(self.filename, 'w') | |
| 1463 | #write metadata |
|
1462 | #write metadata | |
| 1464 | self.writeMetadata(self.fp) |
|
1463 | self.writeMetadata(self.fp) | |
| 1465 | #Write data |
|
1464 | #Write data | |
| 1466 | self.writeData(self.fp) |
|
1465 | self.writeData(self.fp) | |
| 1467 |
|
1466 | |||
| 1468 | def writeMetadata(self, fp): |
|
1467 | def writeMetadata(self, fp): | |
| 1469 |
|
1468 | |||
| 1470 | grp = fp.create_group("Metadata") |
|
1469 | grp = fp.create_group("Metadata") | |
| 1471 | grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype) |
|
1470 | grp.create_dataset('variables', data=self.tableDim, dtype=self.dtype) | |
| 1472 |
|
1471 | |||
| 1473 | for i in range(len(self.metadataList)): |
|
1472 | for i in range(len(self.metadataList)): | |
| 1474 | if not hasattr(self.dataOut, self.metadataList[i]): |
|
1473 | if not hasattr(self.dataOut, self.metadataList[i]): | |
| 1475 | log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name) |
|
1474 | log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name) | |
| 1476 | continue |
|
1475 | continue | |
| 1477 | value = getattr(self.dataOut, self.metadataList[i]) |
|
1476 | value = getattr(self.dataOut, self.metadataList[i]) | |
| 1478 | grp.create_dataset(self.metadataList[i], data=value) |
|
1477 | grp.create_dataset(self.metadataList[i], data=value) | |
| 1479 | return |
|
1478 | return | |
| 1480 |
|
1479 | |||
| 1481 | def writeData(self, fp): |
|
1480 | def writeData(self, fp): | |
| 1482 |
|
1481 | |||
| 1483 | grp = fp.create_group("Data") |
|
1482 | grp = fp.create_group("Data") | |
| 1484 | dtsets = [] |
|
1483 | dtsets = [] | |
| 1485 | data = [] |
|
1484 | data = [] | |
| 1486 |
|
1485 | |||
| 1487 | for dsInfo in self.dsList: |
|
1486 | for dsInfo in self.dsList: | |
| 1488 | if dsInfo['nDim'] == 0: |
|
1487 | if dsInfo['nDim'] == 0: | |
| 1489 | ds = grp.create_dataset( |
|
1488 | ds = grp.create_dataset( | |
| 1490 | dsInfo['variable'], |
|
1489 | dsInfo['variable'], | |
| 1491 | (self.blocksPerFile, ), |
|
1490 | (self.blocksPerFile, ), | |
| 1492 | chunks=True, |
|
1491 | chunks=True, | |
| 1493 | dtype=numpy.float64) |
|
1492 | dtype=numpy.float64) | |
| 1494 | dtsets.append(ds) |
|
1493 | dtsets.append(ds) | |
| 1495 | data.append((dsInfo['variable'], -1)) |
|
1494 | data.append((dsInfo['variable'], -1)) | |
| 1496 | else: |
|
1495 | else: | |
| 1497 | sgrp = grp.create_group(dsInfo['variable']) |
|
1496 | sgrp = grp.create_group(dsInfo['variable']) | |
| 1498 | for i in range(dsInfo['dsNumber']): |
|
1497 | for i in range(dsInfo['dsNumber']): | |
| 1499 | ds = sgrp.create_dataset( |
|
1498 | ds = sgrp.create_dataset( | |
| 1500 | 'table{:02d}'.format(i), |
|
1499 | 'table{:02d}'.format(i), | |
| 1501 | (self.blocksPerFile, ) + dsInfo['shape'][1:], |
|
1500 | (self.blocksPerFile, ) + dsInfo['shape'][1:], | |
| 1502 | chunks=True) |
|
1501 | chunks=True) | |
| 1503 | dtsets.append(ds) |
|
1502 | dtsets.append(ds) | |
| 1504 | data.append((dsInfo['variable'], i)) |
|
1503 | data.append((dsInfo['variable'], i)) | |
| 1505 | fp.flush() |
|
1504 | fp.flush() | |
| 1506 |
|
1505 | |||
| 1507 | log.log('creating file: {}'.format(fp.filename), 'Writing') |
|
1506 | log.log('creating file: {}'.format(fp.filename), 'Writing') | |
| 1508 |
|
1507 | |||
| 1509 | self.ds = dtsets |
|
1508 | self.ds = dtsets | |
| 1510 | self.data = data |
|
1509 | self.data = data | |
| 1511 | self.firsttime = True |
|
1510 | self.firsttime = True | |
| 1512 | self.blockIndex = 0 |
|
1511 | self.blockIndex = 0 | |
| 1513 | return |
|
1512 | return | |
| 1514 |
|
1513 | |||
| 1515 | def putData(self): |
|
1514 | def putData(self): | |
| 1516 |
|
1515 | |||
| 1517 | if (self.blockIndex == self.blocksPerFile) or self.timeFlag(): |
|
1516 | if (self.blockIndex == self.blocksPerFile) or self.timeFlag(): | |
| 1518 | self.closeFile() |
|
1517 | self.closeFile() | |
| 1519 | self.setNextFile() |
|
1518 | self.setNextFile() | |
| 1520 |
|
1519 | |||
| 1521 | for i, ds in enumerate(self.ds): |
|
1520 | for i, ds in enumerate(self.ds): | |
| 1522 | attr, ch = self.data[i] |
|
1521 | attr, ch = self.data[i] | |
| 1523 | if ch == -1: |
|
1522 | if ch == -1: | |
| 1524 | ds[self.blockIndex] = getattr(self.dataOut, attr) |
|
1523 | ds[self.blockIndex] = getattr(self.dataOut, attr) | |
| 1525 | else: |
|
1524 | else: | |
| 1526 | ds[self.blockIndex] = getattr(self.dataOut, attr)[ch] |
|
1525 | ds[self.blockIndex] = getattr(self.dataOut, attr)[ch] | |
| 1527 |
|
1526 | |||
| 1528 | self.fp.flush() |
|
1527 | self.fp.flush() | |
| 1529 | self.blockIndex += 1 |
|
1528 | self.blockIndex += 1 | |
| 1530 |
|
1529 | |||
| 1531 | return |
|
1530 | return | |
| 1532 |
|
1531 | |||
| 1533 | def closeFile(self): |
|
1532 | def closeFile(self): | |
| 1534 |
|
1533 | |||
| 1535 | if self.blockIndex != self.blocksPerFile: |
|
1534 | if self.blockIndex != self.blocksPerFile: | |
| 1536 | for ds in self.ds: |
|
1535 | for ds in self.ds: | |
| 1537 | ds.resize(self.blockIndex, axis=0) |
|
1536 | ds.resize(self.blockIndex, axis=0) | |
| 1538 |
|
1537 | |||
| 1539 | self.fp.flush() |
|
1538 | self.fp.flush() | |
| 1540 | self.fp.close() |
|
1539 | self.fp.close() | |
| 1541 |
|
1540 | |||
| 1542 | def close(self): |
|
1541 | def close(self): | |
| 1543 |
|
1542 | |||
| 1544 | self.closeFile() |
|
1543 | self.closeFile() | |
General Comments 0
You need to be logged in to leave comments.
Login now