##// END OF EJS Templates
schain
RSS
updated with changes in v3.0-devel
rflores -
r1737:23e53bebbd06
parent child
Show More
Side by Side Unified
Context file:
r1737:23e53bebbd06 Loading diff...:
Show file before | Show file after | Hide comments
@@ -73,7 +73,9 class ConfBase():
73 73 '''
74 74 '''
75 75
76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
76 if format is not None:
77 self.parameters[name] = eval(format)(value)
78 elif isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
77 79 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
78 80 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
79 81 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
@@ -287,7 +289,7 class ReadUnitConf(ProcUnitConf):
287 289 self.parameters = {}
288 290
289 291 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
290 startTime='', endTime='', server=None, **kwargs):
292 startTime='', endTime='', server=None, topic='', **kwargs):
291 293
292 294 if datatype == None and name == None:
293 295 raise ValueError('datatype or name should be defined')
@@ -315,6 +317,8 class ReadUnitConf(ProcUnitConf):
315 317 self.addParameter(name='endDate', value=endDate)
316 318 self.addParameter(name='startTime', value=startTime)
317 319 self.addParameter(name='endTime', value=endTime)
320 self.addParameter(name='server', value=server)
321 self.addParameter(name='topic', value=topic)
318 322
319 323 for key, value in kwargs.items():
320 324 self.addParameter(name=key, value=value)
@@ -553,6 +557,10 class Project(Process):
553 557 for key in keys:
554 558 conf = self.configurations[key]
555 559 conf.createObjects()
560 if 'Reader' in str(conf):
561 reader = conf.object
562 else:
563 conf.object.reader = reader
556 564 if conf.inputId is not None:
557 565 if isinstance(conf.inputId, list):
558 566 conf.object.setInput([self.configurations[x].object for x in conf.inputId])
Show file before | Show file after | Hide comments
@@ -125,6 +125,7 class Beam:
125 125 class GenericData(object):
126 126
127 127 flagNoData = True
128 blockReader = False
128 129
129 130 def copy(self, inputObj=None):
130 131
@@ -198,7 +199,11 class JROData(GenericData):
198 199
199 200 def __str__(self):
200 201
201 return '{} - {}'.format(self.type, self.datatime())
202 try:
203 dt = self.datatime
204 except:
205 dt = 'None'
206 return '{} - {}'.format(self.type, dt)
202 207
203 208 def getNoise(self):
204 209
@@ -467,8 +472,9 class Spectra(JROData):
467 472 self.ippFactor = 1
468 473 self.beacon_heiIndexList = []
469 474 self.noise_estimation = None
475 self.spc_noise = None
470 476 self.metadata_list = ['type', 'heightList', 'timeZone', 'pairsList', 'channelList', 'nCohInt',
471 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp','nIncohInt', 'nFFTPoints', 'nProfiles']
477 'code', 'nCode', 'nBaud', 'ippSeconds', 'ipp', 'nIncohInt', 'nFFTPoints', 'nProfiles', 'flagDecodeData']
472 478
473 479 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
474 480 """
@@ -491,7 +497,10 class Spectra(JROData):
491 497
492 498 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
493 499
494 if self.noise_estimation is not None:
500 if self.spc_noise is not None:
501 # this was estimated by getNoise Operation defined in jroproc_parameters.py
502 return self.spc_noise
503 elif self.noise_estimation is not None:
495 504 # this was estimated by getNoise Operation defined in jroproc_spectra.py
496 505 return self.noise_estimation
497 506 else:
@@ -868,6 +877,7 class Parameters(Spectra):
868 877 nAvg = None
869 878 noise_estimation = None
870 879 GauSPC = None # Fit gaussian SPC
880 spc_noise = None
871 881
872 882 def __init__(self):
873 883 '''
@@ -877,6 +887,7 class Parameters(Spectra):
877 887 self.systemHeaderObj = SystemHeader()
878 888 self.type = "Parameters"
879 889 self.timeZone = 0
890 self.ippFactor = 1
880 891
881 892 def getTimeRange1(self, interval):
882 893
Show file before | Show file after | Hide comments
@@ -304,6 +304,7 class Plot(Operation):
304 304 ax.firsttime = True
305 305 ax.index = 0
306 306 ax.press = None
307 ax.cbar = None
307 308 self.axes.append(ax)
308 309 if self.showprofile:
309 310 cax = self.__add_axes(ax, size=size, pad=pad)
@@ -414,7 +415,7 class Plot(Operation):
414 415 self.pf_axes[n].grid(b=True, axis='x')
415 416 [tick.set_visible(False)
416 417 for tick in self.pf_axes[n].get_yticklabels()]
417 if self.colorbar:
418 if self.colorbar and ax.cbar == None:
418 419 ax.cbar = plt.colorbar(
419 420 ax.plt, ax=ax, fraction=0.05, pad=0.02, aspect=10)
420 421 ax.cbar.ax.tick_params(labelsize=8)
Show file before | Show file after | Hide comments
@@ -354,7 +354,7 class GenericRTIPlot(Plot):
354 354 else:
355 355 if self.zlimits is not None:
356 356 self.zmin, self.zmax = self.zlimits[n]
357 ax.collections.remove(ax.collections[0])
357 ax.plt.remove()
358 358 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
359 359 vmin=self.zmin,
360 360 vmax=self.zmax,
@@ -430,7 +430,7 class PolarMapPlot(Plot):
430 430 else:
431 431 if self.zlimits is not None:
432 432 self.zmin, self.zmax = self.zlimits[n]
433 ax.collections.remove(ax.collections[0])
433 ax.plt.remove()
434 434 ax.plt = ax.pcolormesh( # r, theta, numpy.ma.array(data, mask=numpy.isnan(data)),
435 435 x, y, numpy.ma.array(data, mask=numpy.isnan(data)),
436 436 vmin=self.zmin,
Show file before | Show file after | Hide comments
@@ -762,7 +762,7 class RTIPlot(Plot):
762 762 else:
763 763 if self.zlimits is not None:
764 764 self.zmin, self.zmax = self.zlimits[n]
765 ax.collections.remove(ax.collections[0])
765 ax.plt.remove()
766 766 ax.plt = ax.pcolormesh(x, y, z[n].T,
767 767 vmin=self.zmin,
768 768 vmax=self.zmax,
@@ -869,7 +869,7 class SpectrogramPlot(Plot):
869 869 cmap=plt.get_cmap(self.colormap)
870 870 )
871 871 else:
872 ax.collections.remove(ax.collections[0])
872 ax.plt.remove()
873 873 ax.plt = ax.pcolormesh(x, y, z[n].T,
874 874 vmin=self.zmin,
875 875 vmax=self.zmax,
Show file before | Show file after | Hide comments
@@ -90,7 +90,7 class RTIDPPlot(RTIPlot):
90 90 else:
91 91 #if self.zlimits is not None:
92 92 #self.zmin, self.zmax = self.zlimits[n]
93 ax.collections.remove(ax.collections[0])
93 ax.plt.remove()
94 94 ax.plt = ax.pcolormesh(x, y, z[n].T,
95 95 vmin=self.zmin,
96 96 vmax=self.zmax,
@@ -178,7 +178,7 class RTILPPlot(RTIPlot):
178 178 else:
179 179 if self.zlimits is not None:
180 180 self.zmin, self.zmax = self.zlimits[n]
181 ax.collections.remove(ax.collections[0])
181 ax.plt.remove()
182 182 ax.plt = ax.pcolormesh(x, y, z[n].T,
183 183 vmin=self.zmin,
184 184 vmax=self.zmax,
@@ -191,7 +191,7 class DenRTIPlot(RTIPlot):
191 191 Written by R. Flores
192 192 '''
193 193 '''
194 Plot for Den
194 RTI Plot for Electron Densities
195 195 '''
196 196
197 197 CODE = 'denrti'
@@ -250,21 +250,21 class DenRTIPlot(RTIPlot):
250 250 if numpy.log10(self.zmin)<0:
251 251 self.zmin=1
252 252 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
253 vmin=self.zmin,
254 vmax=self.zmax,
253 #vmin=self.zmin,
254 #vmax=self.zmax,
255 255 cmap=self.cmaps[n],
256 norm=colors.LogNorm()
256 norm=colors.LogNorm(vmin=self.zmin,vmax=self.zmax)
257 257 )
258 258
259 259 else:
260 260 if self.zlimits is not None:
261 261 self.zmin, self.zmax = self.zlimits[n]
262 ax.collections.remove(ax.collections[0])
262 ax.plt.remove()
263 263 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
264 vmin=self.zmin,
265 vmax=self.zmax,
264 #vmin=self.zmin,
265 #vmax=self.zmax,
266 266 cmap=self.cmaps[n],
267 norm=colors.LogNorm()
267 norm=colors.LogNorm(vmin=self.zmin,vmax=self.zmax)
268 268 )
269 269
270 270
@@ -346,7 +346,7 class ETempRTIPlot(RTIPlot):
346 346 else:
347 347 if self.zlimits is not None:
348 348 self.zmin, self.zmax = self.zlimits[n]
349 ax.collections.remove(ax.collections[0])
349 ax.plt.remove()
350 350 ax.plt = ax.pcolormesh(x, y, z[n].T * self.factors[n],
351 351 vmin=self.zmin,
352 352 vmax=self.zmax,
@@ -473,8 +473,8 class TempsDPPlot(Plot):
473 473 errTi = data['Ti_error']
474 474
475 475 if ax.firsttime:
476 ax.errorbar(Te, y, xerr=errTe, fmt='r^',elinewidth=1.0,color='b',linewidth=2.0, label='Te')
477 ax.errorbar(Ti, y, fmt='k^', xerr=errTi,elinewidth=1.0,color='b',linewidth=2.0, label='Ti')
476 ax.errorbar(Te, y, xerr=errTe, fmt='r^',elinewidth=1.0,color='r',linewidth=2.0, label='Te')
477 ax.errorbar(Ti, y, fmt='k^', xerr=errTi,elinewidth=1.0,color='k',linewidth=2.0, label='Ti')
478 478 plt.legend(loc='lower right')
479 479 self.ystep_given = 50
480 480 ax.yaxis.set_minor_locator(MultipleLocator(15))
@@ -482,8 +482,8 class TempsDPPlot(Plot):
482 482
483 483 else:
484 484 self.clear_figures()
485 ax.errorbar(Te, y, xerr=errTe, fmt='r^',elinewidth=1.0,color='b',linewidth=2.0, label='Te')
486 ax.errorbar(Ti, y, fmt='k^', xerr=errTi,elinewidth=1.0,color='b',linewidth=2.0, label='Ti')
485 ax.errorbar(Te, y, xerr=errTe, fmt='r^',elinewidth=1.0,color='r',linewidth=2.0, label='Te')
486 ax.errorbar(Ti, y, fmt='k^', xerr=errTi,elinewidth=1.0,color='k',linewidth=2.0, label='Ti')
487 487 plt.legend(loc='lower right')
488 488 ax.yaxis.set_minor_locator(MultipleLocator(15))
489 489
@@ -545,8 +545,8 class TempsHPPlot(Plot):
545 545
546 546 if ax.firsttime:
547 547
548 ax.errorbar(Te, self.y, xerr=errTe, fmt='r^',elinewidth=1.0,color='b',linewidth=2.0, label='Te')
549 ax.errorbar(Ti, self.y, fmt='k^', xerr=errTi,elinewidth=1.0,color='b',linewidth=2.0, label='Ti')
548 ax.errorbar(Te, self.y, xerr=errTe, fmt='r^',elinewidth=1.0,color='r',linewidth=2.0, label='Te')
549 ax.errorbar(Ti, self.y, fmt='k^', xerr=errTi,elinewidth=1.0,color='',linewidth=2.0, label='Ti')
550 550 plt.legend(loc='lower right')
551 551 self.ystep_given = 200
552 552 ax.yaxis.set_minor_locator(MultipleLocator(15))
@@ -554,8 +554,8 class TempsHPPlot(Plot):
554 554
555 555 else:
556 556 self.clear_figures()
557 ax.errorbar(Te, self.y, xerr=errTe, fmt='r^',elinewidth=1.0,color='b',linewidth=2.0, label='Te')
558 ax.errorbar(Ti, self.y, fmt='k^', xerr=errTi,elinewidth=1.0,color='b',linewidth=2.0, label='Ti')
557 ax.errorbar(Te, self.y, xerr=errTe, fmt='r^',elinewidth=1.0,color='r',linewidth=2.0, label='Te')
558 ax.errorbar(Ti, self.y, fmt='k^', xerr=errTi,elinewidth=1.0,color='k',linewidth=2.0, label='Ti')
559 559 plt.legend(loc='lower right')
560 560 ax.yaxis.set_minor_locator(MultipleLocator(15))
561 561 ax.grid(which='minor')
@@ -624,8 +624,8 class FracsHPPlot(Plot):
624 624
625 625 if ax.firsttime:
626 626
627 ax.errorbar(ph, self.y[cut:], xerr=eph, fmt='r^',elinewidth=1.0,color='b',linewidth=2.0, label='H+')
628 ax.errorbar(phe, self.y[cut:], fmt='k^', xerr=ephe,elinewidth=1.0,color='b',linewidth=2.0, label='He+')
627 ax.errorbar(ph, self.y[cut:], xerr=eph, fmt='r^',elinewidth=1.0,color='r',linewidth=2.0, label='H+')
628 ax.errorbar(phe, self.y[cut:], fmt='k^', xerr=ephe,elinewidth=1.0,color='k',linewidth=2.0, label='He+')
629 629 plt.legend(loc='lower right')
630 630 self.xstep_given = 0.2
631 631 self.ystep_given = 200
@@ -634,8 +634,8 class FracsHPPlot(Plot):
634 634
635 635 else:
636 636 self.clear_figures()
637 ax.errorbar(ph, self.y[cut:], xerr=eph, fmt='r^',elinewidth=1.0,color='b',linewidth=2.0, label='H+')
638 ax.errorbar(phe, self.y[cut:], fmt='k^', xerr=ephe,elinewidth=1.0,color='b',linewidth=2.0, label='He+')
637 ax.errorbar(ph, self.y[cut:], xerr=eph, fmt='r^',elinewidth=1.0,color='r',linewidth=2.0, label='H+')
638 ax.errorbar(phe, self.y[cut:], fmt='k^', xerr=ephe,elinewidth=1.0,color='k',linewidth=2.0, label='He+')
639 639 plt.legend(loc='lower right')
640 640 ax.yaxis.set_minor_locator(MultipleLocator(15))
641 641 ax.grid(which='minor')
@@ -711,14 +711,14 class EDensityPlot(Plot):
711 711 #ax.errorbar(DenFar, y[:NSHTS], xerr=1, fmt='h-',elinewidth=1.0,color='g',linewidth=1.0, label='Faraday Profile',markersize=2)
712 712 ax.errorbar(DenFar, y[:NSHTS], xerr=1, fmt='h-',elinewidth=1.0,color='g',linewidth=1.0, label='Faraday',markersize=2,linestyle='-')
713 713 #ax.errorbar(DenPow, y[:NSHTS], fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='b',linewidth=1.0, label='Power Profile',markersize=2)
714 ax.errorbar(DenPow, y[:NSHTS], fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='b',linewidth=1.0, label='Power',markersize=2,linestyle='-')
714 ax.errorbar(DenPow, y[:NSHTS], fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='k',linewidth=1.0, label='Power',markersize=2,linestyle='-')
715 715
716 716 if self.CODE=='denLP':
717 717 ax.errorbar(DenPowLP[cut:], y[cut:], xerr=errDenPowLP[cut:], fmt='r^-',elinewidth=1.0,color='r',linewidth=1.0, label='LP Profile',markersize=2)
718 718
719 719 plt.legend(loc='upper left',fontsize=8.5)
720 720 #plt.legend(loc='lower left',fontsize=8.5)
721 ax.set_xscale("log", nonposx='clip')
721 ax.set_xscale("log")#, nonposx='clip')
722 722 grid_y_ticks=numpy.arange(numpy.nanmin(y),numpy.nanmax(y),50)
723 723 self.ystep_given=100
724 724 if self.CODE=='denLP':
@@ -738,13 +738,13 class EDensityPlot(Plot):
738 738 #ax.errorbar(DenFar, y[:NSHTS], xerr=1, fmt='h-',elinewidth=1.0,color='g',linewidth=1.0, label='Faraday Profile',markersize=2)
739 739 ax.errorbar(DenFar, y[:NSHTS], xerr=1, fmt='h-',elinewidth=1.0,color='g',linewidth=1.0, label='Faraday',markersize=2,linestyle='-')
740 740 #ax.errorbar(DenPow, y[:NSHTS], fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='b',linewidth=1.0, label='Power Profile',markersize=2)
741 ax.errorbar(DenPow, y[:NSHTS], fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='b',linewidth=1.0, label='Power',markersize=2,linestyle='-')
741 ax.errorbar(DenPow, y[:NSHTS], fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='k',linewidth=1.0, label='Power',markersize=2,linestyle='-')
742 742 ax.errorbar(DenPowBefore, y[:NSHTS], elinewidth=1.0,color='r',linewidth=0.5,linestyle="dashed")
743 743
744 744 if self.CODE=='denLP':
745 745 ax.errorbar(DenPowLP[cut:], y[cut:], fmt='r^-', xerr=errDenPowLP[cut:],elinewidth=1.0,color='r',linewidth=1.0, label='LP Profile',markersize=2)
746 746
747 ax.set_xscale("log", nonposx='clip')
747 ax.set_xscale("log")#, nonposx='clip')
748 748 grid_y_ticks=numpy.arange(numpy.nanmin(y),numpy.nanmax(y),50)
749 749 ax.set_yticks(grid_y_ticks,minor=True)
750 750 locmaj = LogLocator(base=10,numticks=12)
@@ -805,11 +805,11 class RelativeDenPlot(Plot):
805 805
806 806 if ax.firsttime:
807 807 self.autoxticks=False
808 ax.errorbar(DenPow, y, fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='b',linewidth=1.0, label='Power',markersize=2,linestyle='-')
808 ax.errorbar(DenPow, y, fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='k',linewidth=1.0, label='Power',markersize=2,linestyle='-')
809 809
810 810 plt.legend(loc='upper left',fontsize=8.5)
811 811 #plt.legend(loc='lower left',fontsize=8.5)
812 ax.set_xscale("log", nonposx='clip')
812 ax.set_xscale("log")#, nonposx='clip')
813 813 grid_y_ticks=numpy.arange(numpy.nanmin(y),numpy.nanmax(y),50)
814 814 self.ystep_given=100
815 815 ax.set_yticks(grid_y_ticks,minor=True)
@@ -824,10 +824,10 class RelativeDenPlot(Plot):
824 824 dataBefore = self.data[-2]
825 825 DenPowBefore = dataBefore['den_power']
826 826 self.clear_figures()
827 ax.errorbar(DenPow, y, fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='b',linewidth=1.0, label='Power',markersize=2,linestyle='-')
827 ax.errorbar(DenPow, y, fmt='k^-', xerr=errDenPow,elinewidth=1.0,color='k',linewidth=1.0, label='Power',markersize=2,linestyle='-')
828 828 ax.errorbar(DenPowBefore, y, elinewidth=1.0,color='r',linewidth=0.5,linestyle="dashed")
829 829
830 ax.set_xscale("log", nonposx='clip')
830 ax.set_xscale("log")#, nonposx='clip')
831 831 grid_y_ticks=numpy.arange(numpy.nanmin(y),numpy.nanmax(y),50)
832 832 ax.set_yticks(grid_y_ticks,minor=True)
833 833 locmaj = LogLocator(base=10,numticks=12)
Show file before | Show file after | Hide comments
@@ -87,7 +87,7 DATA_STRUCTURE = numpy.dtype([
87 87
88 88 class BLTRParamReader(Reader, ProcessingUnit):
89 89 '''
90 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
90 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR
91 91 from *.sswma files
92 92 '''
93 93
@@ -108,9 +108,9 class BLTRParamReader(Reader, ProcessingUnit):
108 108 self.filefmt = "*********%Y%m%d******"
109 109
110 110 def setup(self, **kwargs):
111
111
112 112 self.set_kwargs(**kwargs)
113
113
114 114 if self.path is None:
115 115 raise ValueError("The path is not valid")
116 116
@@ -119,13 +119,13 class BLTRParamReader(Reader, ProcessingUnit):
119 119
120 120 for nTries in range(self.nTries):
121 121 fullpath = self.searchFilesOnLine(self.path, self.startDate,
122 self.endDate, self.expLabel, self.ext, self.walk,
122 self.endDate, self.expLabel, self.ext, self.walk,
123 123 self.filefmt, self.folderfmt)
124 124 try:
125 125 fullpath = next(fullpath)
126 126 except:
127 127 fullpath = None
128
128
129 129 if fullpath:
130 130 self.fileSize = os.path.getsize(fullpath)
131 131 self.filename = fullpath
@@ -138,17 +138,17 class BLTRParamReader(Reader, ProcessingUnit):
138 138
139 139 log.warning(
140 140 'Waiting {} sec for a valid file in {}: try {} ...'.format(
141 self.delay, self.path, nTries + 1),
141 self.delay, self.path, nTries + 1),
142 142 self.name)
143 143 time.sleep(self.delay)
144 144
145 145 if not(fullpath):
146 146 raise schainpy.admin.SchainError(
147 'There isn\'t any valid file in {}'.format(self.path))
147 'There isn\'t any valid file in {}'.format(self.path))
148 148 self.readFirstHeader()
149 149 else:
150 150 log.log("Searching files in {}".format(self.path), self.name)
151 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
151 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
152 152 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
153 153 self.setNextFile()
154 154
@@ -162,8 +162,8 class BLTRParamReader(Reader, ProcessingUnit):
162 162 if os.path.exists(fullfilename):
163 163 return fullfilename, filename
164 164 return None, filename
165
166
165
166
167 167 def readFirstHeader(self):
168 168 '''
169 169 '''
@@ -174,7 +174,7 class BLTRParamReader(Reader, ProcessingUnit):
174 174 self.nrecords = self.header_file['nrec'][0]
175 175 self.counter_records = 0
176 176 self.flagIsNewFile = 0
177 self.fileIndex += 1
177 self.fileIndex += 1
178 178
179 179 def readNextBlock(self):
180 180
@@ -184,7 +184,13 class BLTRParamReader(Reader, ProcessingUnit):
184 184 if not self.setNextFile():
185 185 return 0
186 186 try:
187 pointer = self.fp.tell()
187 if self.online and self.counter_records == 0:
188 pos = int(self.fileSize / (38512))
189 self.counter_records = pos*2 - 2
190 pointer = 38512 * (pos-1) + 48
191 self.fp.seek(pointer)
192 else:
193 pointer = self.fp.tell()
188 194 self.readBlock()
189 195 except:
190 196 if self.online and self.waitDataBlock(pointer, 38512) == 1:
@@ -255,20 +261,20 class BLTRParamReader(Reader, ProcessingUnit):
255 261 self.correction = self.header_rec['dmode_rngcorr'][0]
256 262 self.imode = self.header_rec['dmode_index'][0]
257 263 self.antenna = self.header_rec['antenna_coord']
258 self.rx_gains = self.header_rec['rx_gains']
259 self.time = self.header_rec['time'][0]
264 self.rx_gains = self.header_rec['rx_gains']
265 self.time = self.header_rec['time'][0]
260 266 dt = datetime.datetime.utcfromtimestamp(self.time)
261 267 if dt.date()>self.datatime.date():
262 268 self.flagDiscontinuousBlock = 1
263 269 self.datatime = dt
264
270
265 271 def readData(self):
266 272 '''
267 Reading and filtering data block record of BLTR rawdata file,
273 Reading and filtering data block record of BLTR rawdata file,
268 274 filtering is according to status_value.
269 275
270 276 Input:
271 status_value - Array data is set to NAN for values that are not
277 status_value - Array data is set to NAN for values that are not
272 278 equal to status_value
273 279
274 280 '''
@@ -316,7 +322,7 class BLTRParamReader(Reader, ProcessingUnit):
316 322 self.dataOut.lat = self.lat
317 323 self.dataOut.lon = self.lon
318 324 self.dataOut.channelList = list(range(self.nchannels))
319 self.dataOut.kchan = self.kchan
325 self.dataOut.kchan = self.kchan
320 326 self.dataOut.delta = self.delta
321 327 self.dataOut.correction = self.correction
322 328 self.dataOut.nmodes = self.nmodes
@@ -341,7 +347,7 class BLTRParamReader(Reader, ProcessingUnit):
341 347 self.set_output()
342 348
343 349 return 1
344
350
345 351 def run(self, **kwargs):
346 352 '''
347 353 '''
@@ -352,4 +358,4 class BLTRParamReader(Reader, ProcessingUnit):
352 358
353 359 self.getData()
354 360
355 return No newline at end of file
361 return
Show file before | Show file after | Hide comments
@@ -475,6 +475,7 class Reader(object):
475 475 warnings = True
476 476 verbose = True
477 477 server = None
478 topic = None
478 479 format = None
479 480 oneDDict = None
480 481 twoDDict = None
@@ -781,6 +782,7 class JRODataReader(Reader):
781 782 firstHeaderSize = 0
782 783 basicHeaderSize = 24
783 784 __isFirstTimeOnline = 1
785 topic = ''
784 786 filefmt = "*%Y%j***"
785 787 folderfmt = "*%Y%j"
786 788 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'online', 'delay', 'walk']
@@ -1151,13 +1153,14 class JRODataReader(Reader):
1151 1153
1152 1154 if self.server is not None:
1153 1155 if 'tcp://' in self.server:
1154 address = server
1156 address = self.server
1155 1157 else:
1156 1158 address = 'ipc:///tmp/%s' % self.server
1157 1159 self.server = address
1158 1160 self.context = zmq.Context()
1159 self.receiver = self.context.socket(zmq.PULL)
1161 self.receiver = self.context.socket(zmq.SUB)
1160 1162 self.receiver.connect(self.server)
1163 self.receiver.setsockopt(zmq.SUBSCRIBE, str.encode(str(self.topic)))
1161 1164 time.sleep(0.5)
1162 1165 print('[Starting] ReceiverData from {}'.format(self.server))
1163 1166 else:
@@ -1286,7 +1289,11 class JRODataReader(Reader):
1286 1289 if self.server is None:
1287 1290 self.getData()
1288 1291 else:
1289 self.getFromServer()
1292 try:
1293 self.getFromServer()
1294 except Exception as e:
1295 log.warning('Invalid block...')
1296 self.dataOut.flagNoData = True
1290 1297
1291 1298
1292 1299 class JRODataWriter(Reader):
@@ -1470,9 +1477,6 class JRODataWriter(Reader):
1470 1477 if self.fp != None:
1471 1478 self.fp.close()
1472 1479
1473 if not os.path.exists(path):
1474 os.mkdir(path)
1475
1476 1480 timeTuple = time.localtime(self.dataOut.utctime)
1477 1481 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1478 1482
@@ -1480,7 +1484,7 class JRODataWriter(Reader):
1480 1484 setFile = self.setFile
1481 1485
1482 1486 if not(os.path.exists(fullpath)):
1483 os.mkdir(fullpath)
1487 os.makedirs(fullpath)
1484 1488 setFile = -1 # inicializo mi contador de seteo
1485 1489 else:
1486 1490 filesList = os.listdir(fullpath)
Show file before | Show file after | Hide comments
@@ -48,6 +48,7 DEF_HEADER = {
48 48
49 49 MNEMONICS = {
50 50 10: 'jro',
51 12: 'jmp',
51 52 11: 'jbr',
52 53 14: 'jmp', #Added by R. Flores
53 54 840: 'jul',
Show file before | Show file after | Hide comments
@@ -184,6 +184,13 class HDFReader(Reader, ProcessingUnit):
184 184
185 185 self.blockList = ind
186 186 self.blocksPerFile = len(ind)
187 # similar to master
188 if len(ind)==0:
189 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.blockIndex,
190 self.blocksPerFile,
191 thisDatetime))
192 self.setNextFile()
193 # similar to master
187 194 return
188 195
189 196 def __readMetadata(self):
@@ -360,14 +367,26 class HDFWriter(Operation):
360 367 Operation.__init__(self)
361 368 return
362 369
363 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, uniqueChannel=False):
370 def set_kwargs(self, **kwargs):
371
372 for key, value in kwargs.items():
373 setattr(self, key, value)
374
375 def set_kwargs_obj(self, obj, **kwargs):
376
377 for key, value in kwargs.items():
378 setattr(obj, key, value)
379
380 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
364 381 self.path = path
365 382 self.blocksPerFile = blocksPerFile
366 383 self.metadataList = metadataList
367 384 self.dataList = [s.strip() for s in dataList]
368 385 self.setType = setType
369 386 self.description = description
370 self.uniqueChannel = uniqueChannel
387 self.set_kwargs(**kwargs)
388 #print("self.uniqueChannel: ", self.uniqueChannel)
389 #self.uniqueChannel = uniqueChannel
371 390
372 391 if self.metadataList is None:
373 392 self.metadataList = self.dataOut.metadata_list
@@ -389,7 +408,7 class HDFWriter(Operation):
389 408 elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
390 409 dsDict['nDim'] = 0
391 410 else:
392 if uniqueChannel: #Creates extra dimension to avoid the creation of multiple channels
411 if self.uniqueChannel: #Creates extra dimension to avoid the creation of multiple channels
393 412 dataAux = numpy.expand_dims(dataAux, axis=0)
394 413 #setattr(self.dataOut, self.dataList[i], numpy.expand_dims(getattr(self.dataOut, self.dataList[i]), axis=0))
395 414 #dataAux = getattr(self.dataOut, self.dataList[i])
@@ -428,13 +447,14 class HDFWriter(Operation):
428 447 return False
429 448
430 449 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
431 dataList=[], setType=None, description={}, uniqueChannel= False):
450 dataList=[], setType=None, description={}, **kwargs):
432 451
433 452 self.dataOut = dataOut
453 self.set_kwargs_obj(self.dataOut, **kwargs)
434 454 if not(self.isConfig):
435 455 self.setup(path=path, blocksPerFile=blocksPerFile,
436 456 metadataList=metadataList, dataList=dataList,
437 setType=setType, description=description, uniqueChannel=uniqueChannel)
457 setType=setType, description=description, **kwargs)
438 458
439 459 self.isConfig = True
440 460 self.setNextFile()
@@ -515,15 +535,17 class HDFWriter(Operation):
515 535 return key
516 536 return name
517 537 else:
518 if 'Metadata' in self.description:
519 meta = self.description['Metadata']
538 if 'Data' in self.description:
539 data = self.description['Data']
540 if 'Metadata' in self.description:
541 data.update(self.description['Metadata'])
520 542 else:
521 meta = self.description
522 if name in meta:
523 if isinstance(meta[name], list):
524 return meta[name][x]
525 elif isinstance(meta[name], dict):
526 for key, value in meta[name].items():
543 data = self.description
544 if name in data:
545 if isinstance(data[name], list):
546 return data[name][x]
547 elif isinstance(data[name], dict):
548 for key, value in data[name].items():
527 549 return value[x]
528 550 if 'cspc' in name:
529 551 return 'pair{:02d}'.format(x)
Show file before | Show file after | Hide comments
@@ -296,7 +296,7 class VoltageReader(JRODataReader, ProcessingUnit):
296 296 self.nReadBlocks += 1
297 297 self.blockPointer = 0
298 298
299 block = self.receiver.recv()
299 topic, block = self.receiver.recv_multipart()
300 300
301 301 self.basicHeaderObj.read(block[self.blockPointer:])
302 302 self.blockPointer += self.basicHeaderObj.length
@@ -309,7 +309,11 class VoltageReader(JRODataReader, ProcessingUnit):
309 309 self.readFirstHeaderFromServer()
310 310
311 311 timestamp = self.basicHeaderObj.get_datatime()
312 print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
312 print('[Receiving] - Block {} - {} from {}'.format(self.nTotalBlocks, timestamp, topic.decode()))
313 if self.nTotalBlocks == self.processingHeaderObj.dataBlocksPerFile:
314 self.nTotalBlocks = 0
315 self.nReadBlocks = 0
316 print('Receiving the next stream...')
313 317 current_pointer_location = self.blockPointer
314 318 junk = numpy.fromstring(
315 319 block[self.blockPointer:], self.dtype, self.blocksize)
Show file before | Show file after | Hide comments
@@ -14,8 +14,6 from threading import Thread
14 14 from multiprocessing import Process, Queue
15 15 from schainpy.utils import log
16 16
17 import copy
18
19 17 QUEUE_SIZE = int(os.environ.get('QUEUE_MAX_SIZE', '10'))
20 18
21 19 class ProcessingUnit(object):
@@ -24,6 +22,7 class ProcessingUnit(object):
24 22 '''
25 23
26 24 proc_type = 'processing'
25 bypass = False
27 26
28 27 def __init__(self):
29 28
@@ -81,8 +80,9 class ProcessingUnit(object):
81 80 else:
82 81 return self.dataIn.isReady()
83 82 elif self.dataIn is None or not self.dataIn.error:
84 #print([getattr(self, at) for at in self.inputs])
85 #print("Elif 1")
83 if 'Reader' in self.name and self.bypass:
84 print('Skipping...reader')
85 return self.dataOut.isReady()
86 86 self.run(**kwargs)
87 87 elif self.dataIn.error:
88 88 #print("Elif 2")
Show file before | Show file after | Hide comments
@@ -186,6 +186,18 class SpectraProc(ProcessingUnit):
186 186 self.profIndex += nVoltProfiles
187 187 self.id_min += nVoltProfiles
188 188 self.id_max += nVoltProfiles
189 elif nVoltProfiles > nProfiles:
190 self.reader.bypass = True
191 if self.profIndex == 0:
192 self.id_min = 0
193 self.id_max = nProfiles
194
195 self.buffer = self.dataIn.data[:, self.id_min:self.id_max,:]
196 self.profIndex += nProfiles
197 self.id_min += nProfiles
198 self.id_max += nProfiles
199 if self.id_max == nVoltProfiles:
200 self.reader.bypass = False
189 201 else:
190 202 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
191 203 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
@@ -197,7 +209,7 class SpectraProc(ProcessingUnit):
197 209 if self.firstdatatime == None:
198 210 self.firstdatatime = self.dataIn.utctime
199 211
200 if self.profIndex == nProfiles:
212 if self.profIndex % nProfiles == 0:
201 213 self.__updateSpecFromVoltage()
202 214 if pairsList == None:
203 215 self.dataOut.pairsList = [pair for pair in itertools.combinations(self.dataOut.channelList, 2)]
@@ -206,7 +218,8 class SpectraProc(ProcessingUnit):
206 218 self.__getFft()
207 219 self.dataOut.flagNoData = False
208 220 self.firstdatatime = None
209 self.profIndex = 0
221 if not self.reader.bypass:
222 self.profIndex = 0
210 223 else:
211 224 raise ValueError("The type of input object '%s' is not valid".format(
212 225 self.dataIn.type))
Show file before | Show file after | Hide comments
@@ -3815,7 +3815,7 class IncohInt(Operation):
3815 3815 dataOut.flagNoData = False
3816 3816
3817 3817 dataOut.VelRange = dataOut.getVelRange(0)
3818 dataOut.FreqRange = dataOut.getFreqRange(0)/1000.
3818 dataOut.FreqRange = dataOut.getFreqRange(0)/1000. #kHz
3819 3819 #print("VelRange: ", dataOut.VelRange)
3820 3820 #exit(1)
3821 3821
Show file before | Show file after | Hide comments
@@ -324,6 +324,18 class filterByHeights(Operation):
324 324
325 325 return dataOut
326 326
327 class setOffset(Operation):
328
329 def run(self, dataOut, offset=None):
330
331 if not offset:
332 offset = 0.0
333
334 newHeiRange = dataOut.heightList - offset
335
336 dataOut.heightList = newHeiRange
337
338 return dataOut
327 339
328 340 class setH0(Operation):
329 341
@@ -9944,9 +9956,11 class SSheightProfiles(Operation):
9944 9956 profileIndex = None
9945 9957 #print(dataOut.getFreqRange(1)/1000.)
9946 9958 #exit(1)
9959 '''
9947 9960 if dataOut.flagDataAsBlock:
9948 9961 dataOut.data = numpy.average(dataOut.data,axis=1)
9949 9962 #print("jee")
9963 '''
9950 9964 dataOut.flagDataAsBlock = False
9951 9965 if not self.isConfig:
9952 9966 self.setup(dataOut, step=step , nsamples=nsamples)
General Comments 0
You need to be logged in to leave comments. Login now