##// END OF EJS Templates
schain
RSS
fix save data and test multiple plot
avaldezp -
r1464:3250059d6d95
parent child
Show More
Side by Side Unified
Context file:
r1464:3250059d6d95 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,735 +1,734
1 1 import os
2 2 import time
3 3 import datetime
4 4
5 5 import numpy
6 6 import h5py
7 7
8 8 import schainpy.admin
9 9 from schainpy.model.data.jrodata import *
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.io.jroIO_base import *
12 12 from schainpy.utils import log
13 13
14 14
15 15 class HDFReader(Reader, ProcessingUnit):
16 16 """Processing unit to read HDF5 format files
17 17
18 18 This unit reads HDF5 files created with `HDFWriter` operation contains
19 19 by default two groups Data and Metadata all variables would be saved as `dataOut`
20 20 attributes.
21 21 It is possible to read any HDF5 file by given the structure in the `description`
22 22 parameter, also you can add extra values to metadata with the parameter `extras`.
23 23
24 24 Parameters:
25 25 -----------
26 26 path : str
27 27 Path where files are located.
28 28 startDate : date
29 29 Start date of the files
30 30 endDate : list
31 31 End date of the files
32 32 startTime : time
33 33 Start time of the files
34 34 endTime : time
35 35 End time of the files
36 36 description : dict, optional
37 37 Dictionary with the description of the HDF5 file
38 38 extras : dict, optional
39 39 Dictionary with extra metadata to be be added to `dataOut`
40 40
41 41 Examples
42 42 --------
43 43
44 44 desc = {
45 45 'Data': {
46 46 'data_output': ['u', 'v', 'w'],
47 47 'utctime': 'timestamps',
48 48 } ,
49 49 'Metadata': {
50 50 'heightList': 'heights'
51 51 }
52 52 }
53 53
54 54 desc = {
55 55 'Data': {
56 56 'data_output': 'winds',
57 57 'utctime': 'timestamps'
58 58 },
59 59 'Metadata': {
60 60 'heightList': 'heights'
61 61 }
62 62 }
63 63
64 64 extras = {
65 65 'timeZone': 300
66 66 }
67 67
68 68 reader = project.addReadUnit(
69 69 name='HDFReader',
70 70 path='/path/to/files',
71 71 startDate='2019/01/01',
72 72 endDate='2019/01/31',
73 73 startTime='00:00:00',
74 74 endTime='23:59:59',
75 75 # description=json.dumps(desc),
76 76 # extras=json.dumps(extras),
77 77 )
78 78
79 79 """
80 80
81 81 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
82 82
83 83 def __init__(self):
84 84 ProcessingUnit.__init__(self)
85 85 self.dataOut = Parameters()
86 86 self.ext = ".hdf5"
87 87 self.optchar = "D"
88 88 self.meta = {}
89 89 self.data = {}
90 90 self.open_file = h5py.File
91 91 self.open_mode = 'r'
92 92 self.description = {}
93 93 self.extras = {}
94 94 self.filefmt = "*%Y%j***"
95 95 self.folderfmt = "*%Y%j"
96 96 self.utcoffset = 0
97 97
98 98 def setup(self, **kwargs):
99 99
100 100 self.set_kwargs(**kwargs)
101 101 if not self.ext.startswith('.'):
102 102 self.ext = '.{}'.format(self.ext)
103 103
104 104 if self.online:
105 105 log.log("Searching files in online mode...", self.name)
106 106
107 107 for nTries in range(self.nTries):
108 108 fullpath = self.searchFilesOnLine(self.path, self.startDate,
109 109 self.endDate, self.expLabel, self.ext, self.walk,
110 110 self.filefmt, self.folderfmt)
111 111 try:
112 112 fullpath = next(fullpath)
113 113 except:
114 114 fullpath = None
115 115
116 116 if fullpath:
117 117 break
118 118
119 119 log.warning(
120 120 'Waiting {} sec for a valid file in {}: try {} ...'.format(
121 121 self.delay, self.path, nTries + 1),
122 122 self.name)
123 123 time.sleep(self.delay)
124 124
125 125 if not(fullpath):
126 126 raise schainpy.admin.SchainError(
127 127 'There isn\'t any valid file in {}'.format(self.path))
128 128
129 129 pathname, filename = os.path.split(fullpath)
130 130 self.year = int(filename[1:5])
131 131 self.doy = int(filename[5:8])
132 132 self.set = int(filename[8:11]) - 1
133 133 else:
134 134 log.log("Searching files in {}".format(self.path), self.name)
135 135 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
136 136 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
137 137
138 138 self.setNextFile()
139 139
140 140 return
141 141
142 142 def readFirstHeader(self):
143 143 '''Read metadata and data'''
144 144
145 145 self.__readMetadata()
146 146 self.__readData()
147 147 self.__setBlockList()
148 148
149 149 if 'type' in self.meta:
150 150 self.dataOut = eval(self.meta['type'])()
151 151
152 152 for attr in self.meta:
153 153 setattr(self.dataOut, attr, self.meta[attr])
154 154
155 155 self.blockIndex = 0
156 156
157 157 return
158 158
159 159 def __setBlockList(self):
160 160 '''
161 161 Selects the data within the times defined
162 162
163 163 self.fp
164 164 self.startTime
165 165 self.endTime
166 166 self.blockList
167 167 self.blocksPerFile
168 168
169 169 '''
170 170
171 171 startTime = self.startTime
172 172 endTime = self.endTime
173 173 thisUtcTime = self.data['utctime'] + self.utcoffset
174 174 self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
175 175 thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
176 176
177 177 thisDate = thisDatetime.date()
178 178 thisTime = thisDatetime.time()
179 179
180 180 startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
181 181 endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
182 182
183 183 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
184 184
185 185 self.blockList = ind
186 186 self.blocksPerFile = len(ind)
187 187 return
188 188
189 189 def __readMetadata(self):
190 190 '''
191 191 Reads Metadata
192 192 '''
193 193
194 194 meta = {}
195 195
196 196 if self.description:
197 197 for key, value in self.description['Metadata'].items():
198 198 meta[key] = self.fp[value][()]
199 199 else:
200 200 grp = self.fp['Metadata']
201 201 for name in grp:
202 202 meta[name] = grp[name][()]
203 203
204 204 if self.extras:
205 205 for key, value in self.extras.items():
206 206 meta[key] = value
207 207 self.meta = meta
208 208
209 209 return
210 210
211 211 def __readData(self):
212 212
213 213 data = {}
214 214
215 215 if self.description:
216 216 for key, value in self.description['Data'].items():
217 217 if isinstance(value, str):
218 218 if isinstance(self.fp[value], h5py.Dataset):
219 219 data[key] = self.fp[value][()]
220 220 elif isinstance(self.fp[value], h5py.Group):
221 221 array = []
222 222 for ch in self.fp[value]:
223 223 array.append(self.fp[value][ch][()])
224 224 data[key] = numpy.array(array)
225 225 elif isinstance(value, list):
226 226 array = []
227 227 for ch in value:
228 228 array.append(self.fp[ch][()])
229 229 data[key] = numpy.array(array)
230 230 else:
231 231 grp = self.fp['Data']
232 232 for name in grp:
233 233 if isinstance(grp[name], h5py.Dataset):
234 234 array = grp[name][()]
235 235 elif isinstance(grp[name], h5py.Group):
236 236 array = []
237 237 for ch in grp[name]:
238 238 array.append(grp[name][ch][()])
239 239 array = numpy.array(array)
240 240 else:
241 241 log.warning('Unknown type: {}'.format(name))
242 242
243 243 if name in self.description:
244 244 key = self.description[name]
245 245 else:
246 246 key = name
247 247 data[key] = array
248 248
249 249 self.data = data
250 250 return
251 251
252 252 def getData(self):
253 253
254 254 for attr in self.data:
255 255 if self.data[attr].ndim == 1:
256 256 setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
257 257 else:
258 258 setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
259 259
260 260 self.dataOut.flagNoData = False
261 261 self.blockIndex += 1
262 262
263 263 log.log("Block No. {}/{} -> {}".format(
264 264 self.blockIndex,
265 265 self.blocksPerFile,
266 266 self.dataOut.datatime.ctime()), self.name)
267 267
268 268 return
269 269
270 270 def run(self, **kwargs):
271 271
272 272 if not(self.isConfig):
273 273 self.setup(**kwargs)
274 274 self.isConfig = True
275 275
276 276 if self.blockIndex == self.blocksPerFile:
277 277 self.setNextFile()
278 278
279 279 self.getData()
280 280
281 281 return
282 282
283 283 @MPDecorator
284 284 class HDFWriter(Operation):
285 285 """Operation to write HDF5 files.
286 286
287 287 The HDF5 file contains by default two groups Data and Metadata where
288 288 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
289 289 parameters, data attributes are normaly time dependent where the metadata
290 290 are not.
291 291 It is possible to customize the structure of the HDF5 file with the
292 292 optional description parameter see the examples.
293 293
294 294 Parameters:
295 295 -----------
296 296 path : str
297 297 Path where files will be saved.
298 298 blocksPerFile : int
299 299 Number of blocks per file
300 300 metadataList : list
301 301 List of the dataOut attributes that will be saved as metadata
302 302 dataList : int
303 303 List of the dataOut attributes that will be saved as data
304 304 setType : bool
305 305 If True the name of the files corresponds to the timestamp of the data
306 306 description : dict, optional
307 307 Dictionary with the desired description of the HDF5 file
308 308
309 309 Examples
310 310 --------
311 311
312 312 desc = {
313 313 'data_output': {'winds': ['z', 'w', 'v']},
314 314 'utctime': 'timestamps',
315 315 'heightList': 'heights'
316 316 }
317 317 desc = {
318 318 'data_output': ['z', 'w', 'v'],
319 319 'utctime': 'timestamps',
320 320 'heightList': 'heights'
321 321 }
322 322 desc = {
323 323 'Data': {
324 324 'data_output': 'winds',
325 325 'utctime': 'timestamps'
326 326 },
327 327 'Metadata': {
328 328 'heightList': 'heights'
329 329 }
330 330 }
331 331
332 332 writer = proc_unit.addOperation(name='HDFWriter')
333 333 writer.addParameter(name='path', value='/path/to/file')
334 334 writer.addParameter(name='blocksPerFile', value='32')
335 335 writer.addParameter(name='metadataList', value='heightList,timeZone')
336 336 writer.addParameter(name='dataList',value='data_output,utctime')
337 337 # writer.addParameter(name='description',value=json.dumps(desc))
338 338
339 339 """
340 340
341 341 ext = ".hdf5"
342 342 optchar = "D"
343 343 filename = None
344 344 path = None
345 345 setFile = None
346 346 fp = None
347 347 firsttime = True
348 348 #Configurations
349 349 blocksPerFile = None
350 350 blockIndex = None
351 351 dataOut = None
352 352 #Data Arrays
353 353 dataList = None
354 354 metadataList = None
355 355 currentDay = None
356 356 lastTime = None
357 357 last_Azipos = None
358 358 last_Elepos = None
359 359 mode = None
360 360 #-----------------------
361 361 Typename = None
362 362
363 363
364 364
365 365 def __init__(self):
366 366
367 367 Operation.__init__(self)
368 368 return
369 369
370 370
371 371 def set_kwargs(self, **kwargs):
372 372
373 373 for key, value in kwargs.items():
374 374 setattr(self, key, value)
375 375
376 376 def set_kwargs_obj(self,obj, **kwargs):
377 377
378 378 for key, value in kwargs.items():
379 379 setattr(obj, key, value)
380 380
381 381 def generalFlag(self):
382 382 ####rint("GENERALFLAG")
383 383 if self.mode== "weather":
384 384 if self.last_Azipos == None:
385 385 tmp = self.dataOut.azimuth
386 386 ####print("ang azimuth writer",tmp)
387 387 self.last_Azipos = tmp
388 388 flag = False
389 389 return flag
390 390 ####print("ang_azimuth writer",self.dataOut.azimuth)
391 391 result = self.dataOut.azimuth - self.last_Azipos
392 392 self.last_Azipos = self.dataOut.azimuth
393 393 if result<0:
394 394 flag = True
395 395 return flag
396 396
397 397 def generalFlag_vRF(self):
398 398 ####rint("GENERALFLAG")
399 399
400 400 try:
401 401 self.dataOut.flagBlock360Done
402 402 return self.dataOut.flagBlock360Done
403 403 except:
404 404 return 0
405 405
406 406
407 407 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None,type_data=None,**kwargs):
408 408 self.path = path
409 409 self.blocksPerFile = blocksPerFile
410 410 self.metadataList = metadataList
411 411 self.dataList = [s.strip() for s in dataList]
412 412 self.setType = setType
413 if self.mode == "weather":
414 self.setType = "weather"
413 if self.setType == "weather":
415 414 self.set_kwargs(**kwargs)
416 415 self.set_kwargs_obj(self.dataOut,**kwargs)
417 416
418 417
419 418 self.description = description
420 419 self.type_data=type_data
421 420
422 421 if self.metadataList is None:
423 422 self.metadataList = self.dataOut.metadata_list
424 423
425 424 tableList = []
426 425 dsList = []
427 426
428 427 for i in range(len(self.dataList)):
429 428 dsDict = {}
430 429 if hasattr(self.dataOut, self.dataList[i]):
431 430 dataAux = getattr(self.dataOut, self.dataList[i])
432 431 dsDict['variable'] = self.dataList[i]
433 432 else:
434 433 log.warning('Attribute {} not found in dataOut', self.name)
435 434 continue
436 435
437 436 if dataAux is None:
438 437 continue
439 438 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
440 439 dsDict['nDim'] = 0
441 440 else:
442 441 dsDict['nDim'] = len(dataAux.shape)
443 442 dsDict['shape'] = dataAux.shape
444 443 dsDict['dsNumber'] = dataAux.shape[0]
445 444 dsDict['dtype'] = dataAux.dtype
446 445 dsList.append(dsDict)
447 446
448 447 self.dsList = dsList
449 448 self.currentDay = self.dataOut.datatime.date()
450 449
451 450 def timeFlag(self):
452 451 currentTime = self.dataOut.utctime
453 452 timeTuple = time.localtime(currentTime)
454 453 dataDay = timeTuple.tm_yday
455 454
456 455 if self.lastTime is None:
457 456 self.lastTime = currentTime
458 457 self.currentDay = dataDay
459 458 return False
460 459
461 460 timeDiff = currentTime - self.lastTime
462 461
463 462 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
464 463 if dataDay != self.currentDay:
465 464 self.currentDay = dataDay
466 465 return True
467 466 elif timeDiff > 3*60*60:
468 467 self.lastTime = currentTime
469 468 return True
470 469 else:
471 470 self.lastTime = currentTime
472 471 return False
473 472
474 473 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
475 474 dataList=[], setType=None, description={},mode= None,type_data=None,Reset = False,**kwargs):
476 475
477 476 if Reset:
478 477 self.isConfig = False
479 478 self.closeFile()
480 479 self.lastTime = None
481 480 self.blockIndex = 0
482 481
483 482 self.dataOut = dataOut
484 483 self.mode = mode
485 484 self.var = dataList[0]
486 485
487 486 if not(self.isConfig):
488 487 self.setup(path=path, blocksPerFile=blocksPerFile,
489 488 metadataList=metadataList, dataList=dataList,
490 489 setType=setType, description=description,type_data=type_data,**kwargs)
491 490
492 491 self.isConfig = True
493 492 self.setNextFile()
494 493
495 494 self.putData()
496 495 return
497 496
498 497 def setNextFile(self):
499 498 ###print("HELLO WORLD--------------------------------")
500 499 ext = self.ext
501 500 path = self.path
502 501 setFile = self.setFile
503 502 type_data = self.type_data
504 503
505 504 timeTuple = time.localtime(self.dataOut.utctime)
506 505 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
507 506 fullpath = os.path.join(path, subfolder)
508 507
509 508 if os.path.exists(fullpath):
510 509 filesList = os.listdir(fullpath)
511 510 filesList = [k for k in filesList if k.startswith(self.optchar)]
512 511 if len( filesList ) > 0:
513 512 filesList = sorted(filesList, key=str.lower)
514 513 filen = filesList[-1]
515 514 # el filename debera tener el siguiente formato
516 515 # 0 1234 567 89A BCDE (hex)
517 516 # x YYYY DDD SSS .ext
518 517 if isNumber(filen[8:11]):
519 518 setFile = int(filen[8:11]) #inicializo mi contador de seteo al seteo del ultimo file
520 519 else:
521 520 setFile = -1
522 521 else:
523 522 setFile = -1 #inicializo mi contador de seteo
524 523 else:
525 524 os.makedirs(fullpath)
526 525 setFile = -1 #inicializo mi contador de seteo
527 526
528 527 ###print("**************************",self.setType)
529 528 if self.setType is None:
530 529 setFile += 1
531 530 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
532 531 timeTuple.tm_year,
533 532 timeTuple.tm_yday,
534 533 setFile,
535 534 ext )
536 535 elif self.setType == "weather":
537 536
538 537 if self.var.lower() == 'Zdb'.lower():
539 538 wr_type = 'Z'
540 539 elif self.var.lower() == 'Zdb_D'.lower():
541 540 wr_type = 'D'
542 541 elif self.var.lower() == 'PhiD_P'.lower():
543 542 wr_type = 'P'
544 543 elif self.var.lower() == 'RhoHV_R'.lower():
545 544 wr_type = 'R'
546 545 elif self.var.lower() == 'velRadial_V'.lower():
547 546 wr_type = 'V'
548 547 elif self.var.lower() == 'Sigmav_W'.lower():
549 548 wr_type = 'S'
550 549 elif self.var.lower() == 'dataPP_POWER'.lower():
551 550 wr_type = 'Pow'
552 551 elif self.var.lower() == 'dataPP_DOP'.lower():
553 552 wr_type = 'Dop'
554 553
555 554
556 555 #Z_SOPHy_El10.0_20200505_14:02:15.h5
557 556 #Z_SOPHy_Az40.0_20200505_14:02:15.h5
558 557 if self.dataOut.flagMode == 1: #'AZI' #PPI
559 558 ang_type = 'El'
560 559 len_aux = int(self.dataOut.data_ele.shape[0]/4)
561 560 mean = numpy.mean(self.dataOut.data_ele[len_aux:-len_aux])
562 561 ang_ = round(mean,1)
563 562 elif self.dataOut.flagMode == 0: #'ELE' #RHI
564 563 ang_type = 'Az'
565 564 len_aux = int(self.dataOut.data_azi.shape[0]/4)
566 565 mean = numpy.mean(self.dataOut.data_azi[len_aux:-len_aux])
567 566 ang_ = round(mean,1)
568 567
569 568 file = '%s%s%s%2.1f%s%2.2d%2.2d%2.2d%s%2.2d%2.2d%2.2d%s' % (wr_type,
570 569 '_SOPHy_',
571 570 ang_type,
572 571 ang_,
573 572 '_',
574 573 timeTuple.tm_year,
575 574 timeTuple.tm_mon,
576 575 timeTuple.tm_mday,
577 576 '_',
578 577 timeTuple.tm_hour,
579 578 timeTuple.tm_min,
580 579 timeTuple.tm_sec,
581 580 ext )
582 581
583 582 else:
584 583 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
585 584 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
586 585 timeTuple.tm_year,
587 586 timeTuple.tm_yday,
588 587 setFile,
589 588 ext )
590 589
591 590 self.filename = os.path.join( path, subfolder, file )
592 591
593 592 #Setting HDF5 File
594 593 #print("filename",self.filename)
595 594 self.fp = h5py.File(self.filename, 'w')
596 595 #write metadata
597 596 self.writeMetadata(self.fp)
598 597 #Write data
599 598 self.writeData(self.fp)
600 599
601 600 def getLabel(self, name, x=None):
602 601
603 602 if x is None:
604 603 if 'Data' in self.description:
605 604 data = self.description['Data']
606 605 if 'Metadata' in self.description:
607 606 data.update(self.description['Metadata'])
608 607 else:
609 608 data = self.description
610 609 if name in data:
611 610 if isinstance(data[name], str):
612 611 return data[name]
613 612 elif isinstance(data[name], list):
614 613 return None
615 614 elif isinstance(data[name], dict):
616 615 for key, value in data[name].items():
617 616 return key
618 617 return name
619 618 else:
620 619 if 'Metadata' in self.description:
621 620 meta = self.description['Metadata']
622 621 else:
623 622 meta = self.description
624 623 if name in meta:
625 624 if isinstance(meta[name], list):
626 625 return meta[name][x]
627 626 elif isinstance(meta[name], dict):
628 627 for key, value in meta[name].items():
629 628 return value[x]
630 629 if 'cspc' in name:
631 630 return 'pair{:02d}'.format(x)
632 631 else:
633 632 return 'channel{:02d}'.format(x)
634 633
635 634 def writeMetadata(self, fp):
636 635
637 636 if self.description:
638 637 if 'Metadata' in self.description:
639 638 grp = fp.create_group('Metadata')
640 639 else:
641 640 grp = fp
642 641 else:
643 642 grp = fp.create_group('Metadata')
644 643
645 644 for i in range(len(self.metadataList)):
646 645 if not hasattr(self.dataOut, self.metadataList[i]):
647 646 log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
648 647 continue
649 648 value = getattr(self.dataOut, self.metadataList[i])
650 649 if isinstance(value, bool):
651 650 if value is True:
652 651 value = 1
653 652 else:
654 653 value = 0
655 654 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
656 655 return
657 656
658 657 def writeData(self, fp):
659 658
660 659 if self.description:
661 660 if 'Data' in self.description:
662 661 grp = fp.create_group('Data')
663 662 else:
664 663 grp = fp
665 664 else:
666 665 grp = fp.create_group('Data')
667 666
668 667 dtsets = []
669 668 data = []
670 669
671 670 for dsInfo in self.dsList:
672 671 if dsInfo['nDim'] == 0:
673 672 ds = grp.create_dataset(
674 673 self.getLabel(dsInfo['variable']),
675 674 (self.blocksPerFile, ),
676 675 chunks=True,
677 676 dtype=numpy.float64)
678 677 dtsets.append(ds)
679 678 data.append((dsInfo['variable'], -1))
680 679 else:
681 680 label = self.getLabel(dsInfo['variable'])
682 681 if label is not None:
683 682 sgrp = grp.create_group(label)
684 683 else:
685 684 sgrp = grp
686 685 for i in range(dsInfo['dsNumber']):
687 686 ds = sgrp.create_dataset(
688 687 self.getLabel(dsInfo['variable'], i),
689 688 (self.blocksPerFile, ) + dsInfo['shape'][1:],
690 689 chunks=True,
691 690 dtype=dsInfo['dtype'])
692 691 dtsets.append(ds)
693 692 data.append((dsInfo['variable'], i))
694 693 fp.flush()
695 694
696 695 log.log('Creating file: {}'.format(fp.filename), self.name)
697 696
698 697 self.ds = dtsets
699 698 self.data = data
700 699 self.firsttime = True
701 700 self.blockIndex = 0
702 701 return
703 702
704 703 def putData(self):
705 704
706 705 if (self.blockIndex == self.blocksPerFile) or self.timeFlag():# or self.generalFlag_vRF():
707 706 self.closeFile()
708 707 self.setNextFile()
709 708
710 709 for i, ds in enumerate(self.ds):
711 710 attr, ch = self.data[i]
712 711 if ch == -1:
713 712 ds[self.blockIndex] = getattr(self.dataOut, attr)
714 713 else:
715 714 ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
716 715
717 716 self.fp.flush()
718 717 self.blockIndex += 1
719 718 log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
720 719
721 720 return
722 721
723 722 def closeFile(self):
724 723
725 724 if self.blockIndex != self.blocksPerFile:
726 725 for ds in self.ds:
727 726 ds.resize(self.blockIndex, axis=0)
728 727
729 728 if self.fp:
730 729 self.fp.flush()
731 730 self.fp.close()
732 731
733 732 def close(self):
734 733
735 734 self.closeFile()
Show file before | Show file after | Hide comments
@@ -1,152 +1,195
1 1 # SOPHY PROC script
2 2 import os, sys, json, argparse
3 3 import datetime
4 4 import time
5 5
6 6 PATH = '/DATA_RM/DATA'
7 7 # PATH = '/Users/jespinoza/workspace/data/'
8 8 #PATH = '/home/roberto/DATA/data_WR_RHI/RHI'
9 9 PATH = '/home/soporte/Downloads/data_WR_RHI'
10 10
11 11
12 12 PARAM = {
13 'P': {'name': 'dataPP_POWER', 'zmin': 35, 'zmax': 60, 'colormap': 'jet', 'label': 'Power', 'cb_label': 'dB'},
14 'V': {'name': 'dataPP_DOP', 'zmin': -20, 'zmax': 20, 'colormap': 'seismic', 'label': 'Velocity', 'cb_label': 'm/s'},
15 'RH': {'name': 'RhoHV_R', 'zmin': 0, 'zmax': 1, 'colormap': 'jet', 'label': 'CoeficienteCorrelacion', 'cb_label': '*'},
16 'FD': {'name': 'PhiD_P', 'zmin': -180, 'zmax': 180, 'colormap': 'RdBu_r', 'label': 'Fase Diferencial', 'cb_label': 'ΒΊ'},
17 'ZD': {'name': 'Zdb_D', 'zmin': -20, 'zmax': 80, 'colormap': 'viridis', 'label': 'ReflectividadDiferencial', 'cb_label': 'dBz'},
18 'Z': {'name': 'Zdb', 'zmin': -20, 'zmax': 60, 'colormap': 'viridis', 'label': 'Reflectividad', 'cb_label': 'dBz'},
19 'W': {'name': 'Sigmav_W', 'zmin': -20, 'zmax': 60, 'colormap': 'viridis', 'label': 'AnchoEspectral', 'cb_label': 'hz'}
13 'P': {'name': 'dataPP_POWER','zmin': 35, 'zmax': 60, 'colormap': 'jet', 'label': 'Power', 'wrname':'Pow', 'cb_label': 'dB', 'ch':1},
14 'V': {'name': 'dataPP_DOP', 'zmin': -20, 'zmax': 20, 'colormap': 'seismic','label': 'Velocity', 'wrname':'Dop', 'cb_label': 'm/s','ch':1},
15 'RH': {'name': 'RhoHV_R', 'zmin': 0, 'zmax': 1, 'colormap': 'jet', 'label': 'Coef.Correlacion', 'wrname':'R', 'cb_label': '*', 'ch':0},
16 'FD': {'name': 'PhiD_P', 'zmin': -180,'zmax': 180,'colormap': 'RdBu_r', 'label': 'Fase Diferencial', 'wrname':'P' , 'cb_label': 'ΒΊ', 'ch':0},
17 'ZD': {'name': 'Zdb_D', 'zmin': -20, 'zmax': 80, 'colormap': 'viridis','label': 'Reflect.Diferencial','wrname':'D' , 'cb_label': 'dBz','ch':0},
18 'Z': {'name': 'Zdb', 'zmin': -20, 'zmax': 60, 'colormap': 'viridis','label': 'Reflectividad', 'wrname':'Z', 'cb_label': 'dBz','ch':1},
19 'W': {'name': 'Sigmav_W', 'zmin': -20, 'zmax': 60, 'colormap': 'viridis','label': 'AnchoEspectral', 'wrname':'S', 'cb_label': 'hz', 'ch':1}
20 20 }
21 21
22
23 #---------------------SIGNAL CHAIN ------------------------------------
24 # Definido por el usuario puede ser modificado solo se necesita definir. Ejemplo
25 '''
26 desc_wr= {
27 'Data': {
28 'dataPP_POW': 'Power',
29 'utctime': 'Time',
30 'azimuth': 'az',
31 'elevation':'el'
32 },
33 'Metadata': {
34 'heightList': 'range',
35 'channelList': 'Channels'
36 }
37 }
38 '''
39
22 40 def main(args):
23 41
24 42 experiment = args.experiment
25 43 fp = open(os.path.join(PATH, experiment, 'experiment.conf'))
26 44 conf = json.loads(fp.read())
27 45
28 46 ipp_km = conf['usrp_tx']['ipp']
29 47 ipp = ipp_km * 2 /300000
30 48 samp_rate = conf['usrp_rx']['sample_rate']
31 49 axis = ['0' if x=='elevation' else '1' for x in conf['pedestal']['speed']] # AZIMUTH 1 ELEVACION 0
32 50 speed_axis = conf['pedestal']['speed']
33 51 steeps = conf['pedestal']['table']
34 52 time_offset = args.time_offset
35 53 parameters = args.parameters
36 54 #start_date = experiment.split('@')[1].split('T')[0].replace('-', '/')
37 55 start_date = '2022/04/22'
38 56 end_date = start_date
39 57 #start_time = experiment.split('@')[1].split('T')[1]
40 58 start_time ='17:42:55'
41 59 end_time = '23:59:59'
42 60 max_index = int(samp_rate*ipp*1e6 * args.range / 60) + int(samp_rate*ipp*1e6 * 1.2 / 60)
43 61 N = int(1/(speed_axis[0]*ipp)) # 1 GRADO DE RESOLUCION
44 62 path = os.path.join(PATH, experiment, 'rawdata')
45 63 path_ped = os.path.join(PATH, experiment, 'position')
46 64 path_plots = os.path.join(PATH, experiment, 'plots')
47 65 path_save = os.path.join(PATH, experiment, 'param')
48 66
49 67 dBmin = 35
50 68 dBmax = 60
51 69 Vmin = -20
52 70 Vmax = 20
53 71
54 72
55 73 from schainpy.controller import Project
56 74
57 75 project = Project()
58 76 project.setup(id='1', name='Sophy', description='sophy proc')
59 77
60 78 reader = project.addReadUnit(datatype='DigitalRFReader',
61 79 path=path,
62 80 startDate=start_date,
63 81 endDate=end_date,
64 82 startTime=start_time,
65 83 endTime=end_time,
66 84 delay=0,
67 85 online=0,
68 86 walk=1,
69 87 ippKm = ipp_km,
70 88 getByBlock = 1,
71 89 nProfileBlocks = N,
72 90 )
73 91
74 92 voltage = project.addProcUnit(datatype='VoltageProc', inputId=reader.getId())
75 93 op = voltage.addOperation(name='setH0')
76 94 op.addParameter(name='h0', value='-1.2')
77 95
78 96 if args.range > 0:
79 97 op = voltage.addOperation(name='selectHeights')
80 98 op.addParameter(name='minIndex', value='0', format='int')
81 99 op.addParameter(name='maxIndex', value=max_index, format='int')
82 100
83 101 op = voltage.addOperation(name='PulsePair_vRF', optype='other')
84 102 op.addParameter(name='n', value=int(N), format='int')
85 103
86 104 proc = project.addProcUnit(datatype='ParametersProc', inputId=voltage.getId())
87 105 #-----------------------new--------- variables polarimetricas---------------
88 106 opObj10 = proc.addOperation(name="WeatherRadar")
89 107 opObj10.addParameter(name='variableList',value='Reflectividad,ReflectividadDiferencial,CoeficienteCorrelacion,FaseDiferencial,VelocidadRadial,AnchoEspectral')
90 108
91 109
92 110
93 111 op = proc.addOperation(name='PedestalInformation')
94 112 op.addParameter(name='path', value=path_ped, format='str')
95 113 op.addParameter(name='interval', value='0.04', format='float')
96 114 op.addParameter(name='time_offset', value=time_offset)
97 115 #op.addParameter(name='axis', value=','.join(axis)) #Preguntar en pedestal si todos los elementos
98 116 #de aqui son iguales, si lo fueran, que considere
99 117 #el primero dato como el modo (PPI o RHI) y que ya
100 118 #no pregunte por el modo porque este no cambia
101 119
102 120 for param in parameters:
103 121 op = proc.addOperation(name='Block360_vRF4')
104 122 #op.addParameter(name='axis', value=','.join(axis))
105 123 op.addParameter(name='runNextOp', value=True)
106 124 op.addParameter(name='attr_data', value=PARAM[param]['name'])
107 125
108 126 path_fig = '/AUTO{}km'.format(args.range)
109 127 op = proc.addOperation(name='WeatherParamsPlot')
110 128 op.addParameter(name='save', value=path_plots+path_fig, format='str')
111 129 op.addParameter(name='save_period', value=-1)
112 130 op.addParameter(name='show', value=args.show)
113 131 op.addParameter(name='channels', value='(1,)')
114 132 op.addParameter(name='zmin', value=PARAM[param]['zmin'])
115 133 op.addParameter(name='zmax', value=PARAM[param]['zmax'])
116 134 op.addParameter(name='attr_data', value=PARAM[param]['name'], format='str')
117 135 op.addParameter(name='labels', value=[PARAM[param]['label']])
118 136 op.addParameter(name='save_code', value=param)
119 137 op.addParameter(name='cb_label', value=PARAM[param]['cb_label'])
120 138 op.addParameter(name='colormap', value=PARAM[param]['colormap'])
121 139
122 140 if args.save:
141 desc_wr= {
142 'Data': {
143 PARAM[param]['name']: PARAM[param]['wrname'],#PARAM[param]['name']: {PARAM[param]['wrname']:['P0','P1']},
144 'utctime': 'Time'
145 },
146 'Metadata': {
147 'heightList': 'range',
148 'channelList': 'Channels',
149 'data_azi': 'azimuth',
150 'data_ele': 'elevation'
151 }
152 }
123 153 opObj10 = proc.addOperation(name='HDFWriter')
124 154 opObj10.addParameter(name='path',value=path_save, format='str')
125 155 opObj10.addParameter(name='Reset',value=True)
126 156 opObj10.addParameter(name='setType',value='weather')
127 157 opObj10.addParameter(name='blocksPerFile',value='1',format='int')
128 opObj10.addParameter(name='metadataList',value='heightList,data_azi,data_ele')
158 #opObj10.addParameter(name='channel',value=PARAM[param]['ch'],format='int')
159 opObj10.addParameter(name='metadataList',value='heightList,channelList,Typename,Datatype,Scantype,Latitude,Longitud,Heading,Waveform,PRF,CreatedBy,ContactInformation,data_azi,data_ele')
160 opObj10.addParameter(name='Typename', value=PARAM[param]['label'])
161 opObj10.addParameter(name='Datatype', value='RadialSet')
162 opObj10.addParameter(name='Scantype', value='PPI')
163 opObj10.addParameter(name='Latitude', value='-11.96')
164 opObj10.addParameter(name='Longitud', value='-76.54')
165 opObj10.addParameter(name='Heading', value='293')
166 opObj10.addParameter(name='Height', value='293')
167 opObj10.addParameter(name='Waveform', value='OFM')
168 opObj10.addParameter(name='PRF', value='2500')
169 opObj10.addParameter(name='CreatedBy', value='WeatherRadarJROTeam')
170 opObj10.addParameter(name='ContactInformation', value='dscipion@igp.gob.pe')
129 171 opObj10.addParameter(name='dataList',value=','.join([PARAM[param]['name'],'utctime']))
172 opObj10.addParameter(name='description',value=json.dumps(desc_wr))
130 173
131 174 project.start()
132 175
133 176
134 177 if __name__ == '__main__':
135 178
136 179 parser = argparse.ArgumentParser(description='Script to process SOPHy data.')
137 180 parser.add_argument('experiment',
138 181 help='Experiment name')
139 182 parser.add_argument('--parameters', nargs='*', default=['P'],
140 183 help='Variables to process: P, Z, V')
141 184 parser.add_argument('--time_offset', default=0,
142 185 help='Fix time offset')
143 186 parser.add_argument('--range', default=0, type=int,
144 187 help='Max range to plot')
145 188 parser.add_argument('--save', action='store_true',
146 189 help='Create output files')
147 190 parser.add_argument('--show', action='store_true',
148 191 help='Show matplotlib plot.')
149 192
150 193 args = parser.parse_args()
151 194 print (args)
152 195 main(args)
General Comments 0
You need to be logged in to leave comments. Login now