##// END OF EJS Templates
merge con schain_mp
Jose Chavez -
r977:3514ee80f098 merge
parent child
Show More
@@ -0,0 +1,4
1 {
2 "python.linting.pylintEnabled": true,
3 "git.ignoreLimitWarning": true
4 } No newline at end of file
@@ -0,0 +1,88
1 import argparse
2
3 from schainpy.controller import Project, multiSchain
4
5 desc = "HF_EXAMPLE"
6
7 controllerObj = Project()
8
9 controllerObj.setup(id='191', name='test01', description=desc)
10
11 readUnitConfObj = controllerObj.addReadUnit(datatype='SpectraReader',
12 path='/home/nanosat/data/sp1_f0',
13 startDate="2017/01/26",
14 endDate="2017/01/26",
15 startTime="00:00:00",
16 endTime="23:59:59",
17 online=0,
18 #set=1426485881,
19 walk=1,
20 verbose=1
21 #timezone=-5*3600
22 )
23
24 # #opObj11 = readUnitConfObj.addOperation(name='printNumberOfBlock')
25 #
26 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=readUnitConfObj.getId())
27 # procUnitConfObj2.addParameter(name='nipp', value='5', format='int')
28
29 procUnitConfObj3 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
30 opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
31
32 #
33 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
34 # opObj11.addParameter(name='id', value='1000', format='int')
35 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
36 # opObj11.addParameter(name='channelList', value='0', format='intlist')
37 # opObj11.addParameter(name='zmin', value='-120', format='float')
38 # opObj11.addParameter(name='zmax', value='-70', format='float')
39 # opObj11.addParameter(name='save', value='1', format='int')
40 # opObj11.addParameter(name='figpath', value=figpath, format='str')
41
42 opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
43 # opObj11.addParameter(name='channelList', value='0', format='intList')
44
45 opObj11.addParameter(name='id', value='2000', format='int')
46 # # opObj11.addParameter(name='colormap', value='0', format='bool')
47 opObj11.addParameter(name='onlySNR', value='1', format='bool')
48 opObj11.addParameter(name='DOP', value='0', format='bool')
49 opObj11.addParameter(name='SNR', value='1', format='bool')
50 opObj11.addParameter(name='SNRthresh', value='0', format='int')
51 opObj11.addParameter(name='SNRmin', value='-10', format='int')
52 opObj11.addParameter(name='SNRmax', value='30', format='int')
53 opObj11.addParameter(name='xmin', value='0', format='int')
54 opObj11.addParameter(name='xmax', value='24', format='int')
55
56 # opObj12 = procUnitConfObj3.addOperation(name='ParametersPlot', optype='other')
57 # #opObj11.addParameter(name='channelList', value='0', format='intlist')
58 # opObj12.addParameter(name='id', value='301', format='int')
59 # opObj12.addParameter(name='xmin', value='0', format='float')
60 # opObj12.addParameter(name='xmax', value='24', format='float')
61
62 # opObj11.addParameter(name='zmin', value='-110', format='float')
63 # opObj11.addParameter(name='zmax', value='-70', format='float')
64 # opObj11.addParameter(name='save', value='0', format='int')
65 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
66 #
67 # opObj12 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
68 # opObj12.addParameter(name='zeromq', value=1, format='int')
69 # opObj12.addParameter(name='verbose', value=0, format='bool')
70
71
72 # opObj13 = procUnitConfObj3.addOperation(name='PublishData', optype='other')
73 # opObj13.addParameter(name='zeromq', value=1, format='int')
74 # opObj13.addParameter(name='server', value="juanca", format='str')
75
76 # opObj12.addParameter(name='delay', value=0, format='int')
77
78
79 # print "Escribiendo el archivo XML"
80 # controllerObj.writeXml(filename)
81 # print "Leyendo el archivo XML"
82 # controllerObj.readXml(filename)
83
84
85 # timeit.timeit('controllerObj.run()', number=2)
86
87 controllerObj.start()
88
@@ -0,0 +1,1
1 You should install "digital_rf_hdf5" module if you want to read USRP data
@@ -100,3 +100,9 ENV/
100 100 # eclipse
101 101 .project
102 102 .pydevproject
103
104 # vscode
105
106 .vscode
107
108 schainpy/scripts/ No newline at end of file
@@ -1,1 +1,1
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="Voltage" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="plotter"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="A schain project" id="191" name="project"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="VoltageReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/schain" /><Parameter format="date" id="191113" name="startDate" value="1970/01/01" /><Parameter format="date" id="191114" name="endDate" value="2017/12/31" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="walk" value="1" /><Parameter format="int" id="191119" name="verbose" value="1" /><Parameter format="int" id="191120" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="VoltageProc" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="ProfileSelector" priority="2" type="other"><Parameter format="intlist" id="191221" name="profileRangeList" value="120,183" /></Operation><Operation id="19123" name="RTIPlot" priority="3" type="other"><Parameter format="str" id="191231" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191232" name="showprofile" value="0" /><Parameter format="int" id="191233" name="xmin" value="0" /><Parameter format="int" id="191234" name="xmax" value="24" /><Parameter format="str" id="191235" name="figpath" value="/home/nanosat/schain/figs" /><Parameter format="int" id="191236" name="wr_period" value="5" /><Parameter format="int" id="191237" name="exp_code" value="22" /></Operation></ProcUnit></Project> No newline at end of file
1 NO CONTENT: modified file
@@ -217,7 +217,6 class ParameterConf():
217 217 self.id = str(new_id)
218 218
219 219 def setup(self, id, name, value, format='str'):
220
221 220 self.id = str(id)
222 221 self.name = name
223 222 if format == 'obj':
@@ -757,7 +756,8 class ReadUnitConf(ProcUnitConf):
757 756
758 757 return self.ELEMENTNAME
759 758
760 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, queue=None, **kwargs):
759 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
760 endTime="", parentId=None, queue=None, server=None, **kwargs):
761 761
762 762 #Compatible with old signal chain version
763 763 if datatype==None and name==None:
@@ -768,14 +768,13 class ReadUnitConf(ProcUnitConf):
768 768 name = datatype
769 769 else:
770 770 name = '%sReader' %(datatype)
771
772 771 if datatype==None:
773 772 datatype = name.replace('Reader','')
774 773
775 774 self.id = id
776 775 self.name = name
777 776 self.datatype = datatype
778
777 if path != '':
779 778 self.path = os.path.abspath(path)
780 779 self.startDate = startDate
781 780 self.endDate = endDate
@@ -785,6 +784,7 class ReadUnitConf(ProcUnitConf):
785 784 self.inputId = '0'
786 785 self.parentId = parentId
787 786 self.queue = queue
787 self.server = server
788 788 self.addRunOperation(**kwargs)
789 789
790 790 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
@@ -826,6 +826,7 class ReadUnitConf(ProcUnitConf):
826 826
827 827 opObj = self.addOperation(name = 'run', optype = 'self')
828 828
829 if self.server is None:
829 830 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
830 831 opObj.addParameter(name='path' , value=self.path, format='str')
831 832 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
@@ -833,9 +834,11 class ReadUnitConf(ProcUnitConf):
833 834 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
834 835 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
835 836 opObj.addParameter(name='queue' , value=self.queue, format='obj')
836
837 837 for key, value in kwargs.items():
838 838 opObj.addParameter(name=key, value=value, format=type(value).__name__)
839 else:
840 opObj.addParameter(name='server' , value=self.server, format='str')
841
839 842
840 843 return opObj
841 844
@@ -115,7 +115,6 class BasicHeader(Header):
115 115 dstFlag = None
116 116 errorCount = None
117 117 datatime = None
118
119 118 __LOCALTIME = None
120 119
121 120 def __init__(self, useLocalTime=True):
@@ -133,9 +132,12 class BasicHeader(Header):
133 132
134 133 def read(self, fp):
135 134
135 self.length = 0
136 136 try:
137 if hasattr(fp, 'read'):
137 138 header = numpy.fromfile(fp, BASIC_STRUCTURE,1)
138
139 else:
140 header = numpy.fromstring(fp, BASIC_STRUCTURE,1)
139 141 except Exception, e:
140 142 print "BasicHeader: "
141 143 print e
@@ -153,6 +155,7 class BasicHeader(Header):
153 155 if self.size < 24:
154 156 return 0
155 157
158 self.length = header.nbytes
156 159 return 1
157 160
158 161 def write(self, fp):
@@ -197,13 +200,20 class SystemHeader(Header):
197 200 self.pciDioBusWidth = pciDioBusWith
198 201
199 202 def read(self, fp):
200
203 self.length = 0
204 try:
201 205 startFp = fp.tell()
206 except Exception, e:
207 startFp = None
208 pass
202 209
203 210 try:
211 if hasattr(fp, 'read'):
204 212 header = numpy.fromfile(fp,SYSTEM_STRUCTURE,1)
213 else:
214 header = numpy.fromstring(fp, SYSTEM_STRUCTURE,1)
205 215 except Exception, e:
206 print "System Header: " + e
216 print "System Header: " + str(e)
207 217 return 0
208 218
209 219 self.size = header['nSize'][0]
@@ -213,6 +223,8 class SystemHeader(Header):
213 223 self.adcResolution = header['nADCResolution'][0]
214 224 self.pciDioBusWidth = header['nPCDIOBusWidth'][0]
215 225
226
227 if startFp is not None:
216 228 endFp = self.size + startFp
217 229
218 230 if fp.tell() > endFp:
@@ -223,6 +235,7 class SystemHeader(Header):
223 235 sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name)
224 236 return 0
225 237
238 self.length = header.nbytes
226 239 return 1
227 240
228 241 def write(self, fp):
@@ -299,13 +312,21 class RadarControllerHeader(Header):
299 312 self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u)
300 313
301 314 def read(self, fp):
302
303
315 self.length = 0
316 try:
304 317 startFp = fp.tell()
318 except Exception, e:
319 startFp = None
320 pass
321
305 322 try:
323 if hasattr(fp, 'read'):
306 324 header = numpy.fromfile(fp,RADAR_STRUCTURE,1)
325 else:
326 header = numpy.fromstring(fp, RADAR_STRUCTURE,1)
327 self.length += header.nbytes
307 328 except Exception, e:
308 print "RadarControllerHeader: " + e
329 print "RadarControllerHeader: " + str(e)
309 330 return 0
310 331
311 332 size = int(header['nSize'][0])
@@ -326,23 +347,64 class RadarControllerHeader(Header):
326 347 self.rangeTxA = header['sRangeTxA'][0]
327 348 self.rangeTxB = header['sRangeTxB'][0]
328 349
350 try:
351 if hasattr(fp, 'read'):
329 352 samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
330
353 else:
354 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
355 self.length += samplingWindow.nbytes
356 except Exception, e:
357 print "RadarControllerHeader: " + str(e)
358 return 0
331 359 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
332 360 self.firstHeight = samplingWindow['h0']
333 361 self.deltaHeight = samplingWindow['dh']
334 362 self.samplesWin = samplingWindow['nsa']
335 363
364
365
366 try:
367 if hasattr(fp, 'read'):
336 368 self.Taus = numpy.fromfile(fp,'<f4',self.numTaus)
369 else:
370 self.Taus = numpy.fromstring(fp[self.length:], '<f4', self.numTaus)
371 self.length += self.Taus.nbytes
372 except Exception, e:
373 print "RadarControllerHeader: " + str(e)
374 return 0
375
376
337 377
338 378 self.code_size = 0
339 379 if self.codeType != 0:
340 self.nCode = int(numpy.fromfile(fp,'<u4',1))
341 self.nBaud = int(numpy.fromfile(fp,'<u4',1))
342 380
381 try:
382 if hasattr(fp, 'read'):
383 self.nCode = numpy.fromfile(fp, '<u4', 1)
384 self.length += self.nCode.nbytes
385 self.nBaud = numpy.fromfile(fp, '<u4', 1)
386 self.length += self.nBaud.nbytes
387 else:
388 self.nCode = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
389 self.length += self.nCode.nbytes
390 self.nBaud = numpy.fromstring(fp[self.length:], '<u4', 1)[0]
391 self.length += self.nBaud.nbytes
392 except Exception, e:
393 print "RadarControllerHeader: " + str(e)
394 return 0
343 395 code = numpy.empty([self.nCode,self.nBaud],dtype='i1')
396
344 397 for ic in range(self.nCode):
398 try:
399 if hasattr(fp, 'read'):
345 400 temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.)))
401 else:
402 temp = numpy.fromstring(fp,'u4', int(numpy.ceil(self.nBaud/32.)))
403 self.length += temp.nbytes
404 except Exception, e:
405 print "RadarControllerHeader: " + str(e)
406 return 0
407
346 408 for ib in range(self.nBaud-1,-1,-1):
347 409 code[ic,ib] = temp[ib/32]%2
348 410 temp[ib/32] = temp[ib/32]/2
@@ -355,7 +417,7 class RadarControllerHeader(Header):
355 417 #
356 418 # if self.line6Function == RCfunction.FLIP:
357 419 # self.flip2 = numpy.fromfile(fp,'<u4',1)
358
420 if startFp is not None:
359 421 endFp = size + startFp
360 422
361 423 if fp.tell() != endFp:
@@ -370,6 +432,7 class RadarControllerHeader(Header):
370 432 if fp.tell() < endFp:
371 433 sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name)
372 434
435
373 436 return 1
374 437
375 438 def write(self, fp):
@@ -508,15 +571,23 class ProcessingHeader(Header):
508 571 self.flag_cspc = False
509 572 self.flag_decode = False
510 573 self.flag_deflip = False
511
574 self.length = 0
512 575 def read(self, fp):
513
576 self.length = 0
577 try:
514 578 startFp = fp.tell()
579 except Exception, e:
580 startFp = None
581 pass
515 582
516 583 try:
584 if hasattr(fp, 'read'):
517 585 header = numpy.fromfile(fp,PROCESSING_STRUCTURE,1)
586 else:
587 header = numpy.fromstring(fp, PROCESSING_STRUCTURE, 1)
588 self.length += header.nbytes
518 589 except Exception, e:
519 print "ProcessingHeader: " + e
590 print "ProcessingHeader: " + str(e)
520 591 return 0
521 592
522 593 size = int(header['nSize'][0])
@@ -530,14 +601,31 class ProcessingHeader(Header):
530 601 self.nIncohInt = int(header['nIncoherentIntegrations'][0])
531 602 self.totalSpectra = int(header['nTotalSpectra'][0])
532 603
604 try:
605 if hasattr(fp, 'read'):
533 606 samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows)
607 else:
608 samplingWindow = numpy.fromstring(fp[self.length:], SAMPLING_STRUCTURE, self.nWindows)
609 self.length += samplingWindow.nbytes
610 except Exception, e:
611 print "ProcessingHeader: " + str(e)
612 return 0
534 613
535 614 self.nHeights = int(numpy.sum(samplingWindow['nsa']))
536 615 self.firstHeight = float(samplingWindow['h0'][0])
537 616 self.deltaHeight = float(samplingWindow['dh'][0])
538 617 self.samplesWin = samplingWindow['nsa'][0]
539 618
619
620 try:
621 if hasattr(fp, 'read'):
540 622 self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra)
623 else:
624 self.spectraComb = numpy.fromstring(fp[self.length:], 'u1', 2*self.totalSpectra)
625 self.length += self.spectraComb.nbytes
626 except Exception, e:
627 print "ProcessingHeader: " + str(e)
628 return 0
541 629
542 630 if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE):
543 631 self.nCode = int(numpy.fromfile(fp,'<u4',1))
@@ -583,8 +671,10 class ProcessingHeader(Header):
583 671 if nPairs > 0:
584 672 self.flag_cspc = True
585 673
586 endFp = size + startFp
587 674
675
676 if startFp is not None:
677 endFp = size + startFp
588 678 if fp.tell() > endFp:
589 679 sys.stderr.write("Warning: Processing header size is lower than it has to be")
590 680 return 0
@@ -11,8 +11,8 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 #import h5py
15 14 import traceback
15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
@@ -994,12 +994,13 class JRODataReader(JRODataIO):
994 994 self.__isFirstTimeOnline = 0
995 995
996 996 def __setNewBlock(self):
997
997 #if self.server is None:
998 998 if self.fp == None:
999 999 return 0
1000 1000
1001 1001 # if self.online:
1002 1002 # self.__jumpToLastBlock()
1003 print 'xxxx'
1003 1004
1004 1005 if self.flagIsNewFile:
1005 1006 self.lastUTTime = self.basicHeaderObj.utc
@@ -1011,19 +1012,22 class JRODataReader(JRODataIO):
1011 1012 return 0
1012 1013 else:
1013 1014 return 1
1014
1015 print 'xxxx'
1016 #if self.server is None:
1015 1017 currentSize = self.fileSize - self.fp.tell()
1016 1018 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1017
1018 1019 if (currentSize >= neededSize):
1019 1020 self.basicHeaderObj.read(self.fp)
1020 1021 self.lastUTTime = self.basicHeaderObj.utc
1021 1022 return 1
1022
1023 # else:
1024 # self.basicHeaderObj.read(self.zHeader)
1025 # self.lastUTTime = self.basicHeaderObj.utc
1026 # return 1
1023 1027 if self.__waitNewBlock():
1024 1028 self.lastUTTime = self.basicHeaderObj.utc
1025 1029 return 1
1026
1030 #if self.server is None:
1027 1031 if not(self.setNextFile()):
1028 1032 return 0
1029 1033
@@ -1041,9 +1045,11 class JRODataReader(JRODataIO):
1041 1045
1042 1046 #Skip block out of startTime and endTime
1043 1047 while True:
1048 print 'cxxxx'
1044 1049 if not(self.__setNewBlock()):
1050 print 'returning'
1045 1051 return 0
1046
1052 print 'dxxx'
1047 1053 if not(self.readBlock()):
1048 1054 return 0
1049 1055
@@ -1274,8 +1280,21 class JRODataReader(JRODataIO):
1274 1280 skip=None,
1275 1281 cursor=None,
1276 1282 warnings=True,
1277 verbose=True):
1278
1283 verbose=True,
1284 server=None):
1285 if server is not None:
1286 if 'tcp://' in server:
1287 address = server
1288 else:
1289 address = 'ipc:///tmp/%s' % server
1290 self.server = address
1291 self.context = zmq.Context()
1292 self.receiver = self.context.socket(zmq.PULL)
1293 self.receiver.connect(self.server)
1294 time.sleep(0.5)
1295 print '[Starting] ReceiverData from {}'.format(self.server)
1296 else:
1297 self.server = None
1279 1298 if path == None:
1280 1299 raise ValueError, "[Reading] The path is not valid"
1281 1300
@@ -1304,7 +1323,6 class JRODataReader(JRODataIO):
1304 1323 self.path = path
1305 1324 self.foldercounter = foldercounter
1306 1325 last_set = None
1307
1308 1326 else:
1309 1327 print "[Reading] Searching files in offline mode ..."
1310 1328 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
@@ -1457,6 +1475,7 class JRODataReader(JRODataIO):
1457 1475 skip=None,
1458 1476 cursor=None,
1459 1477 warnings=True,
1478 server=None,
1460 1479 verbose=True, **kwargs):
1461 1480
1462 1481 if not(self.isConfig):
@@ -1481,10 +1500,13 class JRODataReader(JRODataIO):
1481 1500 skip=skip,
1482 1501 cursor=cursor,
1483 1502 warnings=warnings,
1503 server=server,
1484 1504 verbose=verbose)
1485 1505 self.isConfig = True
1486
1506 if server is None:
1487 1507 self.getData()
1508 else:
1509 self.getFromServer()
1488 1510
1489 1511 class JRODataWriter(JRODataIO):
1490 1512
@@ -10,6 +10,9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 import zmq
14 import tempfile
15 from StringIO import StringIO
13 16 # from _sha import blocksize
14 17
15 18 class VoltageReader(JRODataReader, ProcessingUnit):
@@ -189,6 +192,7 class VoltageReader(JRODataReader, ProcessingUnit):
189 192 self.blocksize = pts2read
190 193
191 194
195
192 196 def readBlock(self):
193 197 """
194 198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
@@ -212,11 +216,23 class VoltageReader(JRODataReader, ProcessingUnit):
212 216 Exceptions:
213 217 Si un bloque leido no es un bloque valido
214 218 """
219
220 print 'READ BLOCK'
221 # if self.server is not None:
222 # self.zBlock = self.receiver.recv()
223 # self.zHeader = self.zBlock[:24]
224 # self.zDataBlock = self.zBlock[24:]
225 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
226 # self.processingHeaderObj.profilesPerBlock = 240
227 # self.processingHeaderObj.nHeights = 248
228 # self.systemHeaderObj.nChannels
229 # else:
215 230 current_pointer_location = self.fp.tell()
216 231 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
217 232
218 233 try:
219 234 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
235 print'junked'
220 236 except:
221 237 #print "The read block (%3d) has not enough data" %self.nReadBlocks
222 238
@@ -301,6 +317,97 class VoltageReader(JRODataReader, ProcessingUnit):
301 317
302 318 return
303 319
320 def readFirstHeaderFromServer(self):
321
322 self.getFirstHeader()
323
324 self.firstHeaderSize = self.basicHeaderObj.size
325
326 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
327 if datatype == 0:
328 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
329 elif datatype == 1:
330 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
331 elif datatype == 2:
332 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
333 elif datatype == 3:
334 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
335 elif datatype == 4:
336 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
337 elif datatype == 5:
338 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
339 else:
340 raise ValueError, 'Data type was not defined'
341
342 self.dtype = datatype_str
343 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
344 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
345 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
346 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
347 self.getBlockDimension()
348
349
350 def getFromServer(self):
351 self.flagDiscontinuousBlock = 0
352 self.profileIndex = 0
353 self.flagIsNewBlock = 1
354 self.dataOut.flagNoData = False
355 self.nTotalBlocks += 1
356 self.nReadBlocks += 1
357 self.blockPointer = 0
358
359 block = self.receiver.recv()
360
361 self.basicHeaderObj.read(block[self.blockPointer:])
362 self.blockPointer += self.basicHeaderObj.length
363 self.systemHeaderObj.read(block[self.blockPointer:])
364 self.blockPointer += self.systemHeaderObj.length
365 self.radarControllerHeaderObj.read(block[self.blockPointer:])
366 self.blockPointer += self.radarControllerHeaderObj.length
367 self.processingHeaderObj.read(block[self.blockPointer:])
368 self.blockPointer += self.processingHeaderObj.length
369 self.readFirstHeaderFromServer()
370
371 timestamp = self.basicHeaderObj.get_datatime()
372 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
373 current_pointer_location = self.blockPointer
374 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
375
376 try:
377 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
378 except:
379 #print "The read block (%3d) has not enough data" %self.nReadBlocks
380 if self.waitDataBlock(pointer_location=current_pointer_location):
381 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
382 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
383 # return 0
384
385 #Dimensions : nChannels, nProfiles, nSamples
386
387 junk = numpy.transpose(junk, (2,0,1))
388 self.datablock = junk['real'] + junk['imag'] * 1j
389 self.profileIndex = 0
390 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
391 if self.selBlocktime != None:
392 if self.dataOut.nCohInt is not None:
393 nCohInt = self.dataOut.nCohInt
394 else:
395 nCohInt = 1
396 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
397 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
398 datasize = self.dataOut.data.shape[1]
399 if datasize < self.selBlocksize:
400 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
401 buffer[:,:datasize,:] = self.dataOut.data
402 self.dataOut.data = buffer
403 self.profileIndex = blockIndex
404
405 self.dataOut.flagDataAsBlock = True
406 self.flagIsNewBlock = 1
407 self.dataOut.realtime = self.online
408
409 return self.dataOut.data
410
304 411 def getData(self):
305 412 """
306 413 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
@@ -337,24 +444,19 class VoltageReader(JRODataReader, ProcessingUnit):
337 444 self.flagDiscontinuousBlock
338 445 self.flagIsNewBlock
339 446 """
340
341 447 if self.flagNoMoreFiles:
342 448 self.dataOut.flagNoData = True
343 449 print 'Process finished'
344 450 return 0
345
346 451 self.flagDiscontinuousBlock = 0
347 452 self.flagIsNewBlock = 0
348
349 453 if self.__hasNotDataInBuffer():
350
351 454 if not( self.readNextBlock() ):
352 455 return 0
353 456
354 457 self.getFirstHeader()
355 458
356 459 self.reshapeData()
357
358 460 if self.datablock is None:
359 461 self.dataOut.flagNoData = True
360 462 return 0
@@ -513,10 +513,10 class CohInt(Operation):
513 513 self.__dataReady = True
514 514 return avgdata, avgdatatime
515 515
516 def run(self, dataOut, **kwargs):
517 516
517 def run(self, dataOut, n=None, timeInterval=None, overlapping=False, byblock=False, **kwargs):
518 518 if not self.isConfig:
519 self.setup(**kwargs)
519 self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
520 520 self.isConfig = True
521 521
522 522 if dataOut.flagDataAsBlock:
@@ -94,4 +94,4 if __name__ == '__main__':
94 94 parser = argparse.ArgumentParser(description='Set number of parallel processes')
95 95 parser.add_argument('--nProcess', default=1, type=int)
96 96 args = parser.parse_args()
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/28')
97 multiSchain(fiber, nProcess=args.nProcess, startDate='2017/01/26', endDate='2017/01/26')
@@ -35,14 +35,11 def fiber(cursor, skip, q, dt):
35 35 # opObj11 = procUnitConfObj3.addOperation(name='SpectralMoments', optype='other')
36 36
37 37 #
38 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='other')
39 # opObj11.addParameter(name='id', value='1000', format='int')
40 # opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
41 # opObj11.addParameter(name='channelList', value='0', format='intlist')
42 # opObj11.addParameter(name='zmin', value='-120', format='float')
43 # opObj11.addParameter(name='zmax', value='-70', format='float')
44 # opObj11.addParameter(name='save', value='1', format='int')
45 # opObj11.addParameter(name='figpath', value=figpath, format='str')
38 opObj11 = procUnitConfObj2.addOperation(name='RTIPlot', optype='other')
39 opObj11.addParameter(name='id', value='1000', format='int')
40 opObj11.addParameter(name='wintitle', value='HF_Jicamarca_Spc', format='str')
41 opObj11.addParameter(name='xmin', value='0', format='int')
42 opObj11.addParameter(name='xmax', value='24', format='int')
46 43
47 44 # opObj11 = procUnitConfObj3.addOperation(name='Parameters1Plot', optype='other')
48 45 # opObj11.addParameter(name='channelList', value='0', format='intList')
@@ -67,8 +64,8 def fiber(cursor, skip, q, dt):
67 64 # opObj11.addParameter(name='save', value='0', format='int')
68 65 # # opObj11.addParameter(name='figpath', value='/tmp/', format='str')
69 66 #
70 opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
71 opObj12.addParameter(name='zeromq', value=1, format='int')
67 # opObj12 = procUnitConfObj2.addOperation(name='PublishData', optype='other')
68 # opObj12.addParameter(name='zeromq', value=1, format='int')
72 69 # opObj12.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
73 70
74 71
@@ -16,37 +16,17 if __name__ == '__main__':
16 16
17 17 proc1 = controllerObj.addProcUnit(name='ReceiverData')
18 18 proc1.addParameter(name='realtime', value='0', format='bool')
19 proc1.addParameter(name='plottypes', value='rti,coh,phase', format='str')
20 proc1.addParameter(name='throttle', value='10', format='int')
19 proc1.addParameter(name='plottypes', value='snr', format='str')
20 proc1.addParameter(name='throttle', value='5', format='int')
21 21 proc1.addParameter(name='interactive', value='0', format='bool')
22 22 # proc1.addParameter(name='server', value='tcp://10.10.10.82:7000', format='str')
23 23 ## TODO Agregar direccion de server de publicacion a graficos como variable
24
25 op1 = proc1.addOperation(name='PlotRTIData', optype='other')
26 op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
27 op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
28 op1.addParameter(name='show', value='0', format='bool')
29 op1.addParameter(name='colormap', value='jet', format='str')
30 #
31 op2 = proc1.addOperation(name='PlotCOHData', optype='other')
32 op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
33 op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
34 op2.addParameter(name='colormap', value='jet', format='str')
35 op2.addParameter(name='show', value='0', format='bool')
36 # # #
37 op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
38 op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
39 op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
40 op6.addParameter(name='show', value='1', format='bool')
41 # #
42 # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
43 # # proc2.addParameter(name='server', value='juanca', format='str')
44 # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
45 # #
46 # op3 = proc1.addOperation(name='PlotSNRData', optype='other')
47 # op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
48 # op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
49 # op3.addParameter(name='show', value='0', format='bool')
24 op3 = proc1.addOperation(name='PlotSNRData', optype='other')
25 op3.addParameter(name='wintitle', value='Julia 150Km', format='str')
26 op3.addParameter(name='zmax', value='30', format='int')
27 op3.addParameter(name='zmin', value='-10', format='int')
28 op3.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
29 op3.addParameter(name='show', value='0', format='bool')
50 30 # #
51 31 # op4 = proc1.addOperation(name='PlotDOPData', optype='other')
52 32 # op4.addParameter(name='wintitle', value='Julia 150Km', format='str')
@@ -54,6 +34,28 if __name__ == '__main__':
54 34 # op4.addParameter(name='show', value='0', format='bool')
55 35 # op4.addParameter(name='colormap', value='jet', format='str')
56 36
37 # op1 = proc1.addOperation(name='PlotRTIData', optype='other')
38 # op1.addParameter(name='wintitle', value='Julia 150Km', format='str')
39 # op1.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
40 # op1.addParameter(name='show', value='0', format='bool')
41 # op1.addParameter(name='colormap', value='jet', format='str')
42 # #
43 # op2 = proc1.addOperation(name='PlotCOHData', optype='other')
44 # op2.addParameter(name='wintitle', value='Julia 150Km', format='str')
45 # op2.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
46 # op2.addParameter(name='colormap', value='jet', format='str')
47 # op2.addParameter(name='show', value='0', format='bool')
48 # # # #
49 # op6 = proc1.addOperation(name='PlotPHASEData', optype='other')
50 # op6.addParameter(name='wintitle', value='Julia 150Km', format='str')
51 # op6.addParameter(name='save', value='/home/nanosat/Pictures', format='str')
52 # op6.addParameter(name='show', value='1', format='bool')
53 # #
54 # # proc2 = controllerObj.addProcUnit(name='ReceiverData')
55 # # proc2.addParameter(name='server', value='juanca', format='str')
56 # # proc2.addParameter(name='plottypes', value='snr,dop', format='str')
57 # #
58
57 59
58 60
59 61 controllerObj.start()
1 NO CONTENT: modified file
General Comments 0
You need to be logged in to leave comments. Login now