##// END OF EJS Templates
merge con schain_mp
Jose Chavez -
r977:3514ee80f098 merge
parent child
Show More
@@ -0,0 +1,4
1 {
2 "python.linting.pylintEnabled": true,
3 "git.ignoreLimitWarning": true
4 } No newline at end of file
@@ -0,0 +1,88
1 import argparse
2
3 from schainpy.controller import Project, multiSchain
4
5 desc = "HF_EXAMPLE"
6
7 controllerObj = Project()
8
9 controllerObj.setup(id='191', name='test01', description=desc)
10
11 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
12 path='/home/nanosat/data/sp1_f0',
13 startDate="2017/01/26",
14 endDate="2017/01/26",
15 startTime="00:00:00",
16 endTime="23:59:59",
17 online=0,
18 #set=1426485881,
19 walk=1,
20 verbose=1
21 #timezone=-5*3600
22 )
23
24 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
25 #
26 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
27 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
28
29 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
30 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
31
32 #
33 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
34 # opObj11.addParameter(name='id', value='1000', format='int')
35 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
36 # opObj11.addParameter(name='channelList', value='0', format='intlist')
37 # opObj11.addParameter(name='zmin', value='-120', format='float')
38 # opObj11.addParameter(name='zmax', value='-70', format='float')
39 # opObj11.addParameter(name='save', value='1', format='int')
40 # opObj11.addParameter(name='figpath', value=figpath, format='str')
41
42 opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
43 # opObj11.addParameter(name='channelList', value='0', format='intList')
44
45 opObj11.addParameter(name='id', value='2000', format='int')
46 # # opObj11.addParameter(name='colormap', value='0', format='bool')
47 opObj11.addParameter(name='onlySNR', value='1', format='bool')
48 opObj11.addParameter(name='DOP', value='0', format='bool')
49 opObj11.addParameter(name='SNR', value='1', format='bool')
50 opObj11.addParameter(name='SNRthresh', value='0', format='int')
51 opObj11.addParameter(name='SNRmin', value='-10', format='int')
52 opObj11.addParameter(name='SNRmax', value='30', format='int')
53 opObj11.addParameter(name='xmin', value='0', format='int')
54 opObj11.addParameter(name='xmax', value='24', format='int')
55
56 # opObj12 = procUnitConfObj3.addOperation(name='ParametersPlot', optype='other')
57 # #opObj11.addParameter(name='channelList', value='0', format='intlist')
58 # opObj12.addParameter(name='id', value='301', format='int')
59 # opObj12.addParameter(name='xmin', value='0', format='float')
60 # opObj12.addParameter(name='xmax', value='24', format='float')
61
62 # opObj11.addParameter(name='zmin', value='-110', format='float')
63 # opObj11.addParameter(name='zmax', value='-70', format='float')
64 # opObj11.addParameter(name='save', value='0', format='int')
65 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
66 #
67 # opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
68 # opObj12.addParameter(name='zeromq', value=1, format='int')
69 # opObj12.addParameter(name='verbose', value=0, format='bool')
70
71
72 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
73 # opObj13.addParameter(name='zeromq', value=1, format='int')
74 # opObj13.addParameter(name='server', value="juanca", format='str')
75
76 # opObj12.addParameter(name='delay', value=0, format='int')
77
78
79 # print "Escribiendo el archivo XML"
80 # controllerObj.writeXml(filename)
81 # print "Leyendo el archivo XML"
82 # controllerObj.readXml(filename)
83
84
85 # timeit.timeit('controllerObj.run()', number=2)
86
87 controllerObj.start()
88
@@ -0,0 +1,1
1 You should install "digital_rf_hdf5" module if you want to read USRP data
@@ -100,3 +100,9 ENV/
100 # eclipse
100 # eclipse
101 .project
101 .project
102 .pydevproject
102 .pydevproject
103
104 # vscode
105
106 .vscode
107
108 schainpy/scripts/ No newline at end of file
@@ -1,1 +1,1
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="Voltage" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="plotter"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="other"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
1 NO CONTENT: modified file
NO CONTENT: modified file
@@ -217,7 +217,6 class ParameterConf():
217 self.id = str(new_id)
217 self.id = str(new_id)
218
218
219 def setup(self, id, name, value, format='str'):
219 def setup(self, id, name, value, format='str'):
220
221 self.id = str(id)
220 self.id = str(id)
222 self.name = name
221 self.name = name
223 if format == 'obj':
222 if format == 'obj':
@@ -757,7 +756,8 class ReadUnitConf(ProcUnitConf):
757
756
758 return self.ELEMENTNAME
757 return self.ELEMENTNAME
759
758
760 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
759 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
760 endTime="", parentId=None, queue=None, server=None, **kwargs):
761
761
762 #Compatible with old signal chain version
762 #Compatible with old signal chain version
763 if datatype==None and name==None:
763 if datatype==None and name==None:
@@ -768,14 +768,13 class ReadUnitConf(ProcUnitConf):
768 name = datatype
768 name = datatype
769 else:
769 else:
770 name = '%sReader' %(datatype)
770 name = '%sReader' %(datatype)
771
772 if datatype==None:
771 if datatype==None:
773 datatype = name.replace('Reader','')
772 datatype = name.replace('Reader','')
774
773
775 self.id = id
774 self.id = id
776 self.name = name
775 self.name = name
777 self.datatype = datatype
776 self.datatype = datatype
778
777 if path != '':
779 self.path = os.path.abspath(path)
778 self.path = os.path.abspath(path)
780 self.startDate = startDate
779 self.startDate = startDate
781 self.endDate = endDate
780 self.endDate = endDate
@@ -785,6 +784,7 class ReadUnitConf(ProcUnitConf):
785 self.inputId = '0'
784 self.inputId = '0'
786 self.parentId = parentId
785 self.parentId = parentId
787 self.queue = queue
786 self.queue = queue
787 self.server = server
788 self.addRunOperation(**kwargs)
788 self.addRunOperation(**kwargs)
789
789
790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
@@ -826,6 +826,7 class ReadUnitConf(ProcUnitConf):
826
826
827 opObj = self.addOperation(name = 'run', optype = 'self')
827 opObj = self.addOperation(name = 'run', optype = 'self')
828
828
829 if self.server is None:
829 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
830 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
830 opObj.addParameter(name='path' , value=self.path, format='str')
831 opObj.addParameter(name='path' , value=self.path, format='str')
831 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
832 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
@@ -833,9 +834,11 class ReadUnitConf(ProcUnitConf):
833 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
834 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
834 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
835 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
835 opObj.addParameter(name='queue' , value=self.queue, format='obj')
836 opObj.addParameter(name='queue' , value=self.queue, format='obj')
836
837 for key, value in kwargs.items():
837 for key, value in kwargs.items():
838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
839 else:
840 opObj.addParameter(name='server' , value=self.server, format='str')
841
839
842
840 return opObj
843 return opObj
841
844
@@ -115,7 +115,6 class BasicHeader(Header):
115 dstFlag = None
115 dstFlag = None
116 errorCount = None
116 errorCount = None
117 datatime = None
117 datatime = None
118
119 __LOCALTIME = None
118 __LOCALTIME = None
120
119
121 def __init__(self, useLocalTime=True):
120 def __init__(self, useLocalTime=True):
@@ -133,9 +132,12 class BasicHeader(Header):
133
132
134 def read(self, fp):
133 def read(self, fp):
135
134
135 self.length = 0
136 try:
136 try:
137 if hasattr(fp, 'read'):
137 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
138 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
138
139 else:
140 header = numpy.fromstring(fp, BASIC_STRUCTURE,1)
139 except Exception, e:
141 except Exception, e:
140 print "BasicHeader: "
142 print "BasicHeader: "
141 print e
143 print e
@@ -153,6 +155,7 class BasicHeader(Header):
153 if self.size < 24:
155 if self.size < 24:
154 return 0
156 return 0
155
157
158 self.length = header.nbytes
156 return 1
159 return 1
157
160
158 def write(self, fp):
161 def write(self, fp):
@@ -197,13 +200,20 class SystemHeader(Header):
197 self.pciDioBusWidth = pciDioBusWith
200 self.pciDioBusWidth = pciDioBusWith
198
201
199 def read(self, fp):
202 def read(self, fp):
200
203 self.length = 0
204 try:
201 startFp = fp.tell()
205 startFp = fp.tell()
206 except Exception, e:
207 startFp = None
208 pass
202
209
203 try:
210 try:
211 if hasattr(fp, 'read'):
204 header = numpy.fromfile(fp,SYSTEM_STRUCTURE,1)
212 header = numpy.fromfile(fp, SYSTEM_STRUCTURE,1)
213 else:
214 header = numpy.fromstring(fp, SYSTEM_STRUCTURE,1)
205 except Exception, e:
215 except Exception, e:
206 print "System Header: " + e
216 print "System Header: " + str(e)
207 return 0
217 return 0
208
218
209 self.size = header['nSize'][0]
219 self.size = header['nSize'][0]
@@ -213,6 +223,8 class SystemHeader(Header):
213 self.adcResolution = header['nADCResolution'][0]
223 self.adcResolution = header['nADCResolution'][0]
214 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
224 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
215
225
226
227 if startFp is not None:
216 endFp = self.size + startFp
228 endFp = self.size + startFp
217
229
218 if fp.tell() > endFp:
230 if fp.tell() > endFp:
@@ -223,6 +235,7 class SystemHeader(Header):
223 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
235 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
224 return 0
236 return 0
225
237
238 self.length = header.nbytes
226 return 1
239 return 1
227
240
228 def write(self, fp):
241 def write(self, fp):
@@ -299,13 +312,21 class RadarControllerHeader(Header):
299 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
312 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
300
313
301 def read(self, fp):
314 def read(self, fp):
302
315 self.length = 0
303
316 try:
304 startFp = fp.tell()
317 startFp = fp.tell()
318 except Exception, e:
319 startFp = None
320 pass
321
305 try:
322 try:
323 if hasattr(fp, 'read'):
306 header = numpy.fromfile(fp,RADAR_STRUCTURE,1)
324 header = numpy.fromfile(fp, RADAR_STRUCTURE,1)
325 else:
326 header = numpy.fromstring(fp, RADAR_STRUCTURE,1)
327 self.length += header.nbytes
307 except Exception, e:
328 except Exception, e:
308 print "RadarControllerHeader: " + e
329 print "RadarControllerHeader: " + str(e)
309 return 0
330 return 0
310
331
311 size = int(header['nSize'][0])
332 size = int(header['nSize'][0])
@@ -326,23 +347,64 class RadarControllerHeader(Header):
326 self.rangeTxA = header['sRangeTxA'][0]
347 self.rangeTxA = header['sRangeTxA'][0]
327 self.rangeTxB = header['sRangeTxB'][0]
348 self.rangeTxB = header['sRangeTxB'][0]
328
349
350 try:
351 if hasattr(fp, 'read'):
329 samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
352 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
330
353 else:
354 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
355 self.length += samplingWindow.nbytes
356 except Exception, e:
357 print "RadarControllerHeader: " + str(e)
358 return 0
331 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
359 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
332 self.firstHeight = samplingWindow['h0']
360 self.firstHeight = samplingWindow['h0']
333 self.deltaHeight = samplingWindow['dh']
361 self.deltaHeight = samplingWindow['dh']
334 self.samplesWin = samplingWindow['nsa']
362 self.samplesWin = samplingWindow['nsa']
335
363
364
365
366 try:
367 if hasattr(fp, 'read'):
336 self.Taus = numpy.fromfile(fp,'<f4',self.numTaus)
368 self.Taus = numpy.fromfile(fp, '<f4', self.numTaus)
369 else:
370 self.Taus = numpy.fromstring(fp[self.length:], '<f4', self.numTaus)
371 self.length += self.Taus.nbytes
372 except Exception, e:
373 print "RadarControllerHeader: " + str(e)
374 return 0
375
376
337
377
338 self.code_size = 0
378 self.code_size = 0
339 if self.codeType != 0:
379 if self.codeType != 0:
340 self.nCode = int(numpy.fromfile(fp,'<u4',1))
341 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
342
380
381 try:
382 if hasattr(fp, 'read'):
383 self.nCode = numpy.fromfile(fp, '<u4', 1)
384 self.length += self.nCode.nbytes
385 self.nBaud = numpy.fromfile(fp, '<u4', 1)
386 self.length += self.nBaud.nbytes
387 else:
388 self.nCode = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
389 self.length += self.nCode.nbytes
390 self.nBaud = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
391 self.length += self.nBaud.nbytes
392 except Exception, e:
393 print "RadarControllerHeader: " + str(e)
394 return 0
343 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
395 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
396
344 for ic in range(self.nCode):
397 for ic in range(self.nCode):
398 try:
399 if hasattr(fp, 'read'):
345 temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.)))
400 temp = numpy.fromfile(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
401 else:
402 temp = numpy.fromstring(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
403 self.length += temp.nbytes
404 except Exception, e:
405 print "RadarControllerHeader: " + str(e)
406 return 0
407
346 for ib in range(self.nBaud-1,-1,-1):
408 for ib in range(self.nBaud-1,-1,-1):
347 code[ic,ib] = temp[ib/32]%2
409 code[ic,ib] = temp[ib/32]%2
348 temp[ib/32] = temp[ib/32]/2
410 temp[ib/32] = temp[ib/32]/2
@@ -355,7 +417,7 class RadarControllerHeader(Header):
355 #
417 #
356 # if self.line6Function == RCfunction.FLIP:
418 # if self.line6Function == RCfunction.FLIP:
357 # self.flip2 = numpy.fromfile(fp,'<u4',1)
419 # self.flip2 = numpy.fromfile(fp,'<u4',1)
358
420 if startFp is not None:
359 endFp = size + startFp
421 endFp = size + startFp
360
422
361 if fp.tell() != endFp:
423 if fp.tell() != endFp:
@@ -370,6 +432,7 class RadarControllerHeader(Header):
370 if fp.tell() < endFp:
432 if fp.tell() < endFp:
371 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
433 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
372
434
435
373 return 1
436 return 1
374
437
375 def write(self, fp):
438 def write(self, fp):
@@ -508,15 +571,23 class ProcessingHeader(Header):
508 self.flag_cspc = False
571 self.flag_cspc = False
509 self.flag_decode = False
572 self.flag_decode = False
510 self.flag_deflip = False
573 self.flag_deflip = False
511
574 self.length = 0
512 def read(self, fp):
575 def read(self, fp):
513
576 self.length = 0
577 try:
514 startFp = fp.tell()
578 startFp = fp.tell()
579 except Exception, e:
580 startFp = None
581 pass
515
582
516 try:
583 try:
584 if hasattr(fp, 'read'):
517 header = numpy.fromfile(fp,PROCESSING_STRUCTURE,1)
585 header = numpy.fromfile(fp, PROCESSING_STRUCTURE, 1)
586 else:
587 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
588 self.length += header.nbytes
518 except Exception, e:
589 except Exception, e:
519 print "ProcessingHeader: " + e
590 print "ProcessingHeader: " + str(e)
520 return 0
591 return 0
521
592
522 size = int(header['nSize'][0])
593 size = int(header['nSize'][0])
@@ -530,14 +601,31 class ProcessingHeader(Header):
530 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
601 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
531 self.totalSpectra = int(header['nTotalSpectra'][0])
602 self.totalSpectra = int(header['nTotalSpectra'][0])
532
603
604 try:
605 if hasattr(fp, 'read'):
533 samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
606 samplingWindow = numpy.fromfile(fp, SAMPLING_STRUCTURE, self.nWindows)
607 else:
608 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
609 self.length += samplingWindow.nbytes
610 except Exception, e:
611 print "ProcessingHeader: " + str(e)
612 return 0
534
613
535 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
614 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
536 self.firstHeight = float(samplingWindow['h0'][0])
615 self.firstHeight = float(samplingWindow['h0'][0])
537 self.deltaHeight = float(samplingWindow['dh'][0])
616 self.deltaHeight = float(samplingWindow['dh'][0])
538 self.samplesWin = samplingWindow['nsa'][0]
617 self.samplesWin = samplingWindow['nsa'][0]
539
618
619
620 try:
621 if hasattr(fp, 'read'):
540 self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
622 self.spectraComb = numpy.fromfile(fp, 'u1', 2*self.totalSpectra)
623 else:
624 self.spectraComb = numpy.fromstring(fp[self.length:], 'u1', 2*self.totalSpectra)
625 self.length += self.spectraComb.nbytes
626 except Exception, e:
627 print "ProcessingHeader: " + str(e)
628 return 0
541
629
542 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
630 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
543 self.nCode = int(numpy.fromfile(fp,'<u4',1))
631 self.nCode = int(numpy.fromfile(fp,'<u4',1))
@@ -583,8 +671,10 class ProcessingHeader(Header):
583 if nPairs > 0:
671 if nPairs > 0:
584 self.flag_cspc = True
672 self.flag_cspc = True
585
673
586 endFp = size + startFp
587
674
675
676 if startFp is not None:
677 endFp = size + startFp
588 if fp.tell() > endFp:
678 if fp.tell() > endFp:
589 sys.stderr.write("Warning: Processing header size is lower than it has to be")
679 sys.stderr.write("Warning: Processing header size is lower than it has to be")
590 return 0
680 return 0
@@ -11,8 +11,8 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time, datetime
13 import time, datetime
14 #import h5py
15 import traceback
14 import traceback
15 import zmq
16
16
17 try:
17 try:
18 from gevent import sleep
18 from gevent import sleep
@@ -994,12 +994,13 class JRODataReader(JRODataIO):
994 self.__isFirstTimeOnline = 0
994 self.__isFirstTimeOnline = 0
995
995
996 def __setNewBlock(self):
996 def __setNewBlock(self):
997
997 #if self.server is None:
998 if self.fp == None:
998 if self.fp == None:
999 return 0
999 return 0
1000
1000
1001 # if self.online:
1001 # if self.online:
1002 # self.__jumpToLastBlock()
1002 # self.__jumpToLastBlock()
1003 print 'xxxx'
1003
1004
1004 if self.flagIsNewFile:
1005 if self.flagIsNewFile:
1005 self.lastUTTime = self.basicHeaderObj.utc
1006 self.lastUTTime = self.basicHeaderObj.utc
@@ -1011,19 +1012,22 class JRODataReader(JRODataIO):
1011 return 0
1012 return 0
1012 else:
1013 else:
1013 return 1
1014 return 1
1014
1015 print 'xxxx'
1016 #if self.server is None:
1015 currentSize = self.fileSize - self.fp.tell()
1017 currentSize = self.fileSize - self.fp.tell()
1016 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1018 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1017
1018 if (currentSize >= neededSize):
1019 if (currentSize >= neededSize):
1019 self.basicHeaderObj.read(self.fp)
1020 self.basicHeaderObj.read(self.fp)
1020 self.lastUTTime = self.basicHeaderObj.utc
1021 self.lastUTTime = self.basicHeaderObj.utc
1021 return 1
1022 return 1
1022
1023 # else:
1024 # self.basicHeaderObj.read(self.zHeader)
1025 # self.lastUTTime = self.basicHeaderObj.utc
1026 # return 1
1023 if self.__waitNewBlock():
1027 if self.__waitNewBlock():
1024 self.lastUTTime = self.basicHeaderObj.utc
1028 self.lastUTTime = self.basicHeaderObj.utc
1025 return 1
1029 return 1
1026
1030 #if self.server is None:
1027 if not(self.setNextFile()):
1031 if not(self.setNextFile()):
1028 return 0
1032 return 0
1029
1033
@@ -1041,9 +1045,11 class JRODataReader(JRODataIO):
1041
1045
1042 #Skip block out of startTime and endTime
1046 #Skip block out of startTime and endTime
1043 while True:
1047 while True:
1048 print 'cxxxx'
1044 if not(self.__setNewBlock()):
1049 if not(self.__setNewBlock()):
1050 print 'returning'
1045 return 0
1051 return 0
1046
1052 print 'dxxx'
1047 if not(self.readBlock()):
1053 if not(self.readBlock()):
1048 return 0
1054 return 0
1049
1055
@@ -1274,8 +1280,21 class JRODataReader(JRODataIO):
1274 skip=None,
1280 skip=None,
1275 cursor=None,
1281 cursor=None,
1276 warnings=True,
1282 warnings=True,
1277 verbose=True):
1283 verbose=True,
1278
1284 server=None):
1285 if server is not None:
1286 if 'tcp://' in server:
1287 address = server
1288 else:
1289 address = 'ipc:///tmp/%s' % server
1290 self.server = address
1291 self.context = zmq.Context()
1292 self.receiver = self.context.socket(zmq.PULL)
1293 self.receiver.connect(self.server)
1294 time.sleep(0.5)
1295 print '[Starting] ReceiverData from {}'.format(self.server)
1296 else:
1297 self.server = None
1279 if path == None:
1298 if path == None:
1280 raise ValueError, "[Reading] The path is not valid"
1299 raise ValueError, "[Reading] The path is not valid"
1281
1300
@@ -1304,7 +1323,6 class JRODataReader(JRODataIO):
1304 self.path = path
1323 self.path = path
1305 self.foldercounter = foldercounter
1324 self.foldercounter = foldercounter
1306 last_set = None
1325 last_set = None
1307
1308 else:
1326 else:
1309 print "[Reading] Searching files in offline mode ..."
1327 print "[Reading] Searching files in offline mode ..."
1310 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1328 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
@@ -1457,6 +1475,7 class JRODataReader(JRODataIO):
1457 skip=None,
1475 skip=None,
1458 cursor=None,
1476 cursor=None,
1459 warnings=True,
1477 warnings=True,
1478 server=None,
1460 verbose=True, **kwargs):
1479 verbose=True, **kwargs):
1461
1480
1462 if not(self.isConfig):
1481 if not(self.isConfig):
@@ -1481,10 +1500,13 class JRODataReader(JRODataIO):
1481 skip=skip,
1500 skip=skip,
1482 cursor=cursor,
1501 cursor=cursor,
1483 warnings=warnings,
1502 warnings=warnings,
1503 server=server,
1484 verbose=verbose)
1504 verbose=verbose)
1485 self.isConfig = True
1505 self.isConfig = True
1486
1506 if server is None:
1487 self.getData()
1507 self.getData()
1508 else:
1509 self.getFromServer()
1488
1510
1489 class JRODataWriter(JRODataIO):
1511 class JRODataWriter(JRODataIO):
1490
1512
@@ -10,6 +10,9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 from schainpy.model.data.jrodata import Voltage
12 from schainpy.model.data.jrodata import Voltage
13 import zmq
14 import tempfile
15 from StringIO import StringIO
13 # from _sha import blocksize
16 # from _sha import blocksize
14
17
15 class VoltageReader(JRODataReader, ProcessingUnit):
18 class VoltageReader(JRODataReader, ProcessingUnit):
@@ -189,6 +192,7 class VoltageReader(JRODataReader, ProcessingUnit):
189 self.blocksize = pts2read
192 self.blocksize = pts2read
190
193
191
194
195
192 def readBlock(self):
196 def readBlock(self):
193 """
197 """
194 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
@@ -212,11 +216,23 class VoltageReader(JRODataReader, ProcessingUnit):
212 Exceptions:
216 Exceptions:
213 Si un bloque leido no es un bloque valido
217 Si un bloque leido no es un bloque valido
214 """
218 """
219
220 print 'READ BLOCK'
221 # if self.server is not None:
222 # self.zBlock = self.receiver.recv()
223 # self.zHeader = self.zBlock[:24]
224 # self.zDataBlock = self.zBlock[24:]
225 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
226 # self.processingHeaderObj.profilesPerBlock = 240
227 # self.processingHeaderObj.nHeights = 248
228 # self.systemHeaderObj.nChannels
229 # else:
215 current_pointer_location = self.fp.tell()
230 current_pointer_location = self.fp.tell()
216 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
231 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217
232
218 try:
233 try:
219 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
234 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
235 print'junked'
220 except:
236 except:
221 #print "The read block (%3d) has not enough data" %self.nReadBlocks
237 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222
238
@@ -301,6 +317,97 class VoltageReader(JRODataReader, ProcessingUnit):
301
317
302 return
318 return
303
319
320 def readFirstHeaderFromServer(self):
321
322 self.getFirstHeader()
323
324 self.firstHeaderSize = self.basicHeaderObj.size
325
326 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
327 if datatype == 0:
328 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
329 elif datatype == 1:
330 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
331 elif datatype == 2:
332 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
333 elif datatype == 3:
334 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
335 elif datatype == 4:
336 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
337 elif datatype == 5:
338 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
339 else:
340 raise ValueError, 'Data type was not defined'
341
342 self.dtype = datatype_str
343 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
344 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
345 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
346 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
347 self.getBlockDimension()
348
349
350 def getFromServer(self):
351 self.flagDiscontinuousBlock = 0
352 self.profileIndex = 0
353 self.flagIsNewBlock = 1
354 self.dataOut.flagNoData = False
355 self.nTotalBlocks += 1
356 self.nReadBlocks += 1
357 self.blockPointer = 0
358
359 block = self.receiver.recv()
360
361 self.basicHeaderObj.read(block[self.blockPointer:])
362 self.blockPointer += self.basicHeaderObj.length
363 self.systemHeaderObj.read(block[self.blockPointer:])
364 self.blockPointer += self.systemHeaderObj.length
365 self.radarControllerHeaderObj.read(block[self.blockPointer:])
366 self.blockPointer += self.radarControllerHeaderObj.length
367 self.processingHeaderObj.read(block[self.blockPointer:])
368 self.blockPointer += self.processingHeaderObj.length
369 self.readFirstHeaderFromServer()
370
371 timestamp = self.basicHeaderObj.get_datatime()
372 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
373 current_pointer_location = self.blockPointer
374 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
375
376 try:
377 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
378 except:
379 #print "The read block (%3d) has not enough data" %self.nReadBlocks
380 if self.waitDataBlock(pointer_location=current_pointer_location):
381 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
382 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
383 # return 0
384
385 #Dimensions : nChannels, nProfiles, nSamples
386
387 junk = numpy.transpose(junk, (2,0,1))
388 self.datablock = junk['real'] + junk['imag'] * 1j
389 self.profileIndex = 0
390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 if self.selBlocktime != None:
392 if self.dataOut.nCohInt is not None:
393 nCohInt = self.dataOut.nCohInt
394 else:
395 nCohInt = 1
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
398 datasize = self.dataOut.data.shape[1]
399 if datasize < self.selBlocksize:
400 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
401 buffer[:,:datasize,:] = self.dataOut.data
402 self.dataOut.data = buffer
403 self.profileIndex = blockIndex
404
405 self.dataOut.flagDataAsBlock = True
406 self.flagIsNewBlock = 1
407 self.dataOut.realtime = self.online
408
409 return self.dataOut.data
410
304 def getData(self):
411 def getData(self):
305 """
412 """
306 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
413 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
@@ -337,24 +444,19 class VoltageReader(JRODataReader, ProcessingUnit):
337 self.flagDiscontinuousBlock
444 self.flagDiscontinuousBlock
338 self.flagIsNewBlock
445 self.flagIsNewBlock
339 """
446 """
340
341 if self.flagNoMoreFiles:
447 if self.flagNoMoreFiles:
342 self.dataOut.flagNoData = True
448 self.dataOut.flagNoData = True
343 print 'Process finished'
449 print 'Process finished'
344 return 0
450 return 0
345
346 self.flagDiscontinuousBlock = 0
451 self.flagDiscontinuousBlock = 0
347 self.flagIsNewBlock = 0
452 self.flagIsNewBlock = 0
348
349 if self.__hasNotDataInBuffer():
453 if self.__hasNotDataInBuffer():
350
351 if not( self.readNextBlock() ):
454 if not( self.readNextBlock() ):
352 return 0
455 return 0
353
456
354 self.getFirstHeader()
457 self.getFirstHeader()
355
458
356 self.reshapeData()
459 self.reshapeData()
357
358 if self.datablock is None:
460 if self.datablock is None:
359 self.dataOut.flagNoData = True
461 self.dataOut.flagNoData = True
360 return 0
462 return 0
@@ -513,10 +513,10 class CohInt(Operation):
513 self.__dataReady = True
513 self.__dataReady = True
514 return avgdata, avgdatatime
514 return avgdata, avgdatatime
515
515
516 def run(self, dataOut, **kwargs):
517
516
517 def run(self, dataOut, n=None, timeInterval=None, overlapping=False, byblock=False, **kwargs):
518 if not self.isConfig:
518 if not self.isConfig:
519 self.setup(**kwargs)
519 self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
520 self.isConfig = True
520 self.isConfig = True
521
521
522 if dataOut.flagDataAsBlock:
522 if dataOut.flagDataAsBlock:
@@ -94,4 +94,4 if __name__ == '__main__':
94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
95 parser.add_argument('--nProcess', default=1, type=int)
95 parser.add_argument('--nProcess', default=1, type=int)
96 args = parser.parse_args()
96 args = parser.parse_args()
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/28')
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/26')
@@ -35,14 +35,11 def fiber(cursor, skip, q, dt):
35 # opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
35 # opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
36
36
37 #
37 #
38 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
38 opObj11 = procUnitConfObj2.addOperation(name='RTIPlot', optype='other')
39 # opObj11.addParameter(name='id', value='1000', format='int')
39 opObj11.addParameter(name='id', value='1000', format='int')
40 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
40 opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
41 # opObj11.addParameter(name='channelList', value='0', format='intlist')
41 opObj11.addParameter(name='xmin', value='0', format='int')
42 # opObj11.addParameter(name='zmin', value='-120', format='float')
42 opObj11.addParameter(name='xmax', value='24', format='int')
43 # opObj11.addParameter(name='zmax', value='-70', format='float')
44 # opObj11.addParameter(name='save', value='1', format='int')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
46
43
47 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
44 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
48 # opObj11.addParameter(name='channelList', value='0', format='intList')
45 # opObj11.addParameter(name='channelList', value='0', format='intList')
@@ -67,8 +64,8 def fiber(cursor, skip, q, dt):
67 # opObj11.addParameter(name='save', value='0', format='int')
64 # opObj11.addParameter(name='save', value='0', format='int')
68 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
65 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
69 #
66 #
70 opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
67 # opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
71 opObj12.addParameter(name='zeromq', value=1, format='int')
68 # opObj12.addParameter(name='zeromq', value=1, format='int')
72 # opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
69 # opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
73
70
74
71
@@ -16,37 +16,17 if __name__ == '__main__':
16
16
17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
18 proc1.addParameter(name='realtime', value='0', format='bool')
18 proc1.addParameter(name='realtime', value='0', format='bool')
19 proc1.addParameter(name='plottypes', value='rti,coh,phase', format='str')
19 proc1.addParameter(name='plottypes', value='snr', format='str')
20 proc1.addParameter(name='throttle', value='10', format='int')
20 proc1.addParameter(name='throttle', value='5', format='int')
21 proc1.addParameter(name='interactive', value='0', format='bool')
21 proc1.addParameter(name='interactive', value='0', format='bool')
22 # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
22 # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
23 ## TODO Agregar direccion de server de publicacion a graficos como variable
23 ## TODO Agregar direccion de server de publicacion a graficos como variable
24
24 op3 = proc1.addOperation(name='PlotSNRData', optype='other')
25 op1 = proc1.addOperation(name='PlotRTIData', optype='other')
25 op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
26 op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
26 op3.addParameter(name='zmax', value='30', format='int')
27 op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
27 op3.addParameter(name='zmin', value='-10', format='int')
28 op1.addParameter(name='show', value='0', format='bool')
28 op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
29 op1.addParameter(name='colormap', value='jet', format='str')
29 op3.addParameter(name='show', value='0', format='bool')
30 #
31 op2 = proc1.addOperation(name='PlotCOHData', optype='other')
32 op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
33 op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
34 op2.addParameter(name='colormap', value='jet', format='str')
35 op2.addParameter(name='show', value='0', format='bool')
36 # # #
37 op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
38 op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
39 op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
40 op6.addParameter(name='show', value='1', format='bool')
41 # #
42 # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
43 # # proc2.addParameter(name='server', value='juanca', format='str')
44 # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
45 # #
46 # op3 = proc1.addOperation(name='PlotSNRData', optype='other')
47 # op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
48 # op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
49 # op3.addParameter(name='show', value='0', format='bool')
50 # #
30 # #
51 # op4 = proc1.addOperation(name='PlotDOPData', optype='other')
31 # op4 = proc1.addOperation(name='PlotDOPData', optype='other')
52 # op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
32 # op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
@@ -54,6 +34,28 if __name__ == '__main__':
54 # op4.addParameter(name='show', value='0', format='bool')
34 # op4.addParameter(name='show', value='0', format='bool')
55 # op4.addParameter(name='colormap', value='jet', format='str')
35 # op4.addParameter(name='colormap', value='jet', format='str')
56
36
37 # op1 = proc1.addOperation(name='PlotRTIData', optype='other')
38 # op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
39 # op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
40 # op1.addParameter(name='show', value='0', format='bool')
41 # op1.addParameter(name='colormap', value='jet', format='str')
42 # #
43 # op2 = proc1.addOperation(name='PlotCOHData', optype='other')
44 # op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
45 # op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
46 # op2.addParameter(name='colormap', value='jet', format='str')
47 # op2.addParameter(name='show', value='0', format='bool')
48 # # # #
49 # op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
50 # op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
51 # op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
52 # op6.addParameter(name='show', value='1', format='bool')
53 # #
54 # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
55 # # proc2.addParameter(name='server', value='juanca', format='str')
56 # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
57 # #
58
57
59
58
60
59 controllerObj.start()
61 controllerObj.start()
1 NO CONTENT: modified file
NO CONTENT: modified file
General Comments 0
You need to be logged in to leave comments. Login now