##// END OF EJS Templates
schain
RSS
tests
Jose Chavez -
r991:584188041a2c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,15
1 Lectura
2 200samples -> 0.47m HDD 1000''
3 200samples -> 6 HDD 100ms file 100s folder
4 200samples -> 0.48ms HDD 100ms file 1000s folder
5 200samples -> 116ms HDD 5000ms file 100s folder
6 200samples -> 182 HDD 10000ms file 100s folder
7 200samples -> 143 HDD 10000ms file 100s folder SSD
8
9 Escritura
10 200samples -> 0.78m HDD 100ms file 1000s folder
11 200samples -> 0.066m HDD 100ms file 1000s folder
12 200samples -> 0.30 HDD 100ms file 100s folder
13 200samples -> 0.23 HDD 5000ms file 100s folder
14 200samples -> 0.176 HDD 10000ms file 100s folder
15
Show file before | Show file after | Hide comments
@@ -4,6 +4,13 Created on Jul 3, 2014
4
4
5 @author: roj-idl71
5 @author: roj-idl71
6 '''
6 '''
7 # SUBCHANNELS EN VEZ DE CHANNELS
8 # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
9 # ACTUALIZACION DE VERSION
10 # HEADERS
11 # MODULO DE ESCRITURA
12 # METADATA
13
7 import os
14 import os
8 import datetime
15 import datetime
9 import numpy
16 import numpy
@@ -49,6 +56,10 class DigitalRFReader(ProcessingUnit):
49 self.__nBaud = None
56 self.__nBaud = None
50 self.__code = None
57 self.__code = None
51
58
59 def close(self):
60 print 'Average of writing to digital rf format is ', self.oldAverage * 1000
61 return
62
52 def __getCurrentSecond(self):
63 def __getCurrentSecond(self):
53
64
54 return self.__thisUnixSample/self.__sample_rate
65 return self.__thisUnixSample/self.__sample_rate
@@ -367,7 +378,9 class DigitalRFReader(ProcessingUnit):
367 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
378 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
368 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
379 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
369 )
380 )
370
381 self.oldAverage = None
382 self.count = 0
383 self.executionTime = 0
371 def __reload(self):
384 def __reload(self):
372 # print
385 # print
373 # print "%s not in range [%s, %s]" %(
386 # print "%s not in range [%s, %s]" %(
@@ -399,6 +412,15 class DigitalRFReader(ProcessingUnit):
399
412
400 return False
413 return False
401
414
415 def timeit(self, toExecute):
416 t0 = time()
417 toExecute()
418 self.executionTime = time() - t0
419 if self.oldAverage is None: self.oldAverage = self.executionTime
420 self.oldAverage = (self.executionTime + self.count*self.oldAverage) / (self.count + 1.0)
421 self.count = self.count + 1.0
422 return
423
402 def __readNextBlock(self, seconds=30, volt_scale = 1):
424 def __readNextBlock(self, seconds=30, volt_scale = 1):
403 '''
425 '''
404 '''
426 '''
@@ -424,9 +446,15 class DigitalRFReader(ProcessingUnit):
424 for thisChannelName in self.__channelNameList: ##TODO VARIOS CHANNELS?
446 for thisChannelName in self.__channelNameList: ##TODO VARIOS CHANNELS?
425 for indexSubchannel in range(self.__num_subchannels):
447 for indexSubchannel in range(self.__num_subchannels):
426 try:
448 try:
449 t0 = time()
427 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
450 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
428 self.__samples_to_read,
451 self.__samples_to_read,
429 thisChannelName, sub_channel=indexSubchannel)
452 thisChannelName, sub_channel=indexSubchannel)
453 self.executionTime = time() - t0
454 if self.oldAverage is None: self.oldAverage = self.executionTime
455 self.oldAverage = (self.executionTime + self.count*self.oldAverage) / (self.count + 1.0)
456 self.count = self.count + 1.0
457
430 except IOError, e:
458 except IOError, e:
431 #read next profile
459 #read next profile
432 self.__flagDiscontinuousBlock = True
460 self.__flagDiscontinuousBlock = True
@@ -561,30 +589,32 class DigitalRFWriter(Operation):
561 '''
589 '''
562 Operation.__init__(self, **kwargs)
590 Operation.__init__(self, **kwargs)
563 self.metadata_dict = {}
591 self.metadata_dict = {}
564 self.dataOut = None
592 self.dataOut = None
565
593
566 def setHeader(self, dataOut):
594 def setHeader(self):
595
596 self.metadata_dict['frequency'] = self.dataOut.frequency
597 self.metadata_dict['timezone'] = self.dataOut.timeZone
598 self.metadata_dict['dtype'] = cPickle.dumps(self.dataOut.dtype)
599 self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
600 self.metadata_dict['heightList'] = self.dataOut.heightList
601 self.metadata_dict['channelList'] = self.dataOut.channelList
602 self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
603 self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
604 self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
605 self.metadata_dict['flagDataAsBlock'] = self.dataOut.flagDataAsBlock
606 self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
607 self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
608
567 return
609 return
568
610
569 def setup(self, dataOut, path, frequency, set=0, metadataFile='metadata', ext='.h5'):
611 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
570 '''
612 '''
571 In this method we should set all initial parameters.
613 In this method we should set all initial parameters.
572 Input:
614 Input:
573 dataOut: Input data will also be outputa data
615 dataOut: Input data will also be outputa data
574 '''
616 '''
575 self.metadata_dict['frequency'] = dataOut.frequency
617 self.setHeader()
576 self.metadata_dict['timezone'] = dataOut.timeZone
577 self.metadata_dict['dtype'] = cPickle.dumps(dataOut.dtype)
578 self.metadata_dict['nProfiles'] = dataOut.nProfiles
579 self.metadata_dict['heightList'] = dataOut.heightList
580 self.metadata_dict['channelList'] = dataOut.channelList
581 self.metadata_dict['flagDecodeData'] = dataOut.flagDecodeData
582 self.metadata_dict['flagDeflipData'] = dataOut.flagDeflipData
583 self.metadata_dict['flagShiftFFT'] = dataOut.flagShiftFFT
584 self.metadata_dict['flagDataAsBlock'] = dataOut.flagDataAsBlock
585 self.metadata_dict['useLocalTime'] = dataOut.useLocalTime
586 self.metadata_dict['nCohInt'] = dataOut.nCohInt
587
588 self.__ippSeconds = dataOut.ippSeconds
618 self.__ippSeconds = dataOut.ippSeconds
589 self.__deltaH = dataOut.getDeltaH()
619 self.__deltaH = dataOut.getDeltaH()
590 self.__sample_rate = 1e6*0.15/self.__deltaH
620 self.__sample_rate = 1e6*0.15/self.__deltaH
@@ -599,9 +629,6 class DigitalRFWriter(Operation):
599 file_cadence_millisecs = long(1.0 * self.__blocks_per_file * self.__nProfiles * self.__nSamples / self.__sample_rate) * 1000
629 file_cadence_millisecs = long(1.0 * self.__blocks_per_file * self.__nProfiles * self.__nSamples / self.__sample_rate) * 1000
600 sub_cadence_secs = file_cadence_millisecs / 500
630 sub_cadence_secs = file_cadence_millisecs / 500
601
631
602 print file_cadence_millisecs
603 print sub_cadence_secs
604
605 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
632 sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
606 sample_rate_numerator = long(sample_rate_fraction.numerator)
633 sample_rate_numerator = long(sample_rate_fraction.numerator)
607 sample_rate_denominator = long(sample_rate_fraction.denominator)
634 sample_rate_denominator = long(sample_rate_fraction.denominator)
@@ -615,15 +642,15 class DigitalRFWriter(Operation):
615 is_continuous = True
642 is_continuous = True
616 marching_periods = False
643 marching_periods = False
617
644
618 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, 100,
645 self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
619 100, start_global_index,
646 fileCadence, start_global_index,
620 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
647 sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
621 is_complex, num_subchannels, is_continuous, marching_periods)
648 is_complex, num_subchannels, is_continuous, marching_periods)
622
649
623 metadata_dir = os.path.join(path, 'metadata')
650 metadata_dir = os.path.join(path, 'metadata')
624 os.system('mkdir %s' % (metadata_dir))
651 os.system('mkdir %s' % (metadata_dir))
625
652
626 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, 100, 1, ##236, file_cadence_millisecs / 1000
653 self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, ##236, file_cadence_millisecs / 1000
627 sample_rate_numerator, sample_rate_denominator,
654 sample_rate_numerator, sample_rate_denominator,
628 metadataFile)
655 metadataFile)
629
656
@@ -645,22 +672,28 class DigitalRFWriter(Operation):
645 return
672 return
646
673
647
674
648 def writeData(self):
675 def timeit(self, toExecute):
649 for i in range(self.dataOut.systemHeaderObj.nSamples):
650 for channel in self.dataOut.channelList:
651 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
652 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
653 t0 = time()
676 t0 = time()
654 self.digitalWriteObj.rf_write(self.arr_data)
677 toExecute()
655 self.executionTime = time() - t0
678 self.executionTime = time() - t0
656 if self.oldAverage is None: self.oldAverage = self.executionTime
679 if self.oldAverage is None: self.oldAverage = self.executionTime
657 self.oldAverage = (self.executionTime + self.count*self.oldAverage) / (self.count + 1.0)
680 self.oldAverage = (self.executionTime + self.count*self.oldAverage) / (self.count + 1.0)
658 self.count = self.count + 1.0
681 self.count = self.count + 1.0
659 if self.oldAverage is None: self.oldAverage = self.executionTime
682 return
683
684
685 def writeData(self):
686 for i in range(self.dataOut.systemHeaderObj.nSamples):
687 for channel in self.dataOut.channelList:
688 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
689 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
690
691 def f(): return self.digitalWriteObj.rf_write(self.arr_data)
692 self.timeit(f)
660
693
661 return
694 return
662
695
663 def run(self, dataOut, frequency=49.92e6, path=None, **kwargs):
696 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=100, metadataCadence=1, **kwargs):
664 '''
697 '''
665 This method will be called many times so here you should put all your code
698 This method will be called many times so here you should put all your code
666 Inputs:
699 Inputs:
@@ -669,7 +702,7 class DigitalRFWriter(Operation):
669 # print dataOut.__dict__
702 # print dataOut.__dict__
670 self.dataOut = dataOut
703 self.dataOut = dataOut
671 if not self.isConfig:
704 if not self.isConfig:
672 self.setup(dataOut, path, frequency, **kwargs)
705 self.setup(dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, **kwargs)
673
706
674 self.writeData()
707 self.writeData()
675
708
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 <Project description="Testing USRP data reader" id="191" name="test01"><ReadUnit datatype="DigitalRF" id="1911" inputId="0" name="DigitalRFReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="DigitalRF" /><Parameter format="str" id="191112" name="path" value="/home/jchavez/jicamarca/mocked_data" /><Parameter format="date" id="191113" name="startDate" value="2000/07/03" /><Parameter format="date" id="191114" name="endDate" value="2017/07/03" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /><Operation id="19122" name="Scope" priority="2" type="external"><Parameter format="int" id="191221" name="id" value="121" /><Parameter format="str" id="191222" name="wintitle" value="Scope" /></Operation></ProcUnit></Project> No newline at end of file
1 <Project description="Testing USRP data reader" id="191" name="test01"><ReadUnit datatype="DigitalRF" id="1911" inputId="0" name="DigitalRFReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="DigitalRF" /><Parameter format="str" id="191112" name="path" value="/media/jchavez/DATA/mocked_data" /><Parameter format="date" id="191113" name="startDate" value="2000/07/03" /><Parameter format="date" id="191114" name="endDate" value="2017/07/03" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Voltage" id="1912" inputId="1911" name="VoltageProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -24,7 +24,7 def main():
24 #Creating a reader object with its parameters
24 #Creating a reader object with its parameters
25 #schainpy.model.io.jroIO_usrp.USRPReader.setup()
25 #schainpy.model.io.jroIO_usrp.USRPReader.setup()
26 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRF',
26 readUnitConfObj = controllerObj.addReadUnit(datatype='DigitalRF',
27 path='/home/jchavez/jicamarca/mocked_data/',
27 path='/media/jchavez/DATA/mocked_data',
28 startDate='2000/07/03',
28 startDate='2000/07/03',
29 endDate='2017/07/03',
29 endDate='2017/07/03',
30 startTime='00:00:00',
30 startTime='00:00:00',
@@ -47,11 +47,11 def main():
47 # opObj10.addParameter(name='nBaud', value='1', format='float')
47 # opObj10.addParameter(name='nBaud', value='1', format='float')
48
48
49 # opObj10 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
49 # opObj10 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
50 # opObj10.addParameter(name='n', value='1', format='float')
50 # opObj10.addParameter(name='n', value='128', format='float')
51
51
52 opObj11 = procUnitConfObj0.addOperation(name='Scope', optype='external')
52 # opObj11 = procUnitConfObj0.addOperation(name='Scope', optype='external')
53 opObj11.addParameter(name='id', value='121', format='int')
53 # opObj11.addParameter(name='id', value='121', format='int')
54 opObj11.addParameter(name='wintitle', value='Scope', format='str')
54 # opObj11.addParameter(name='wintitle', value='Scope', format='str')
55
55
56 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra',
56 # procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra',
57 # inputId=procUnitConfObj0.getId())
57 # inputId=procUnitConfObj0.getId())
General Comments 0
You need to be logged in to leave comments. Login now