From 73660288fffc2755197e3c6ec64952923f77d063 2022-09-21 20:59:53
From: joabAM <joab.apaza32@gmail.com>
Date: 2022-09-21 20:59:53
Subject: [PATCH] fix diferent blocks per file size at the end,-> resize arrays

---

diff --git a/schainpy/model/io/utilsIO.py b/schainpy/model/io/utilsIO.py
index 2c806d9..4f2d777 100644
--- a/schainpy/model/io/utilsIO.py
+++ b/schainpy/model/io/utilsIO.py
@@ -196,6 +196,7 @@ class MergeH5(object):
 
         self.fill_dataIn(data, self.ch_dataIn[ch])
 
+
         return
 
 
@@ -235,14 +236,33 @@ class MergeH5(object):
             if data[attr].ndim == 1:
                 setattr(dataIn, attr, data[attr][:])
             else:
-                setattr(dataIn, attr, data[attr][:,:])
+                setattr(dataIn, attr, numpy.squeeze(data[attr][:,:]))
         #print("shape in", dataIn.data_spc.shape, len(dataIn.data_spc))
         if dataIn.data_spc.ndim > 3:
             dataIn.data_spc = dataIn.data_spc[0]
             #print("shape in", dataIn.data_spc.shape)
-        if self.blocksPerFile == None:
-            self.blocksPerFile = len(dataIn.data_spc)   #blocks, ch, fft, hei
-            print("blocks per file: ", self.blocksPerFile)
+
+
+
+    def getBlocksPerFile(self):
+        b = numpy.zeros(self.nChannels)
+        for i in range(self.nChannels):
+            b[i] = self.ch_dataIn[i].data_spc.shape[0] #number of blocks
+
+        self.blocksPerFile = int(b.min())
+        iresh_ch = numpy.where(b > self.blocksPerFile)[0]
+        if len(iresh_ch) > 0:
+            for ich in iresh_ch:
+                for i in range(len(self.dataList)):
+                    if hasattr(self.ch_dataIn[ich], self.dataList[i]):
+                        # print("reshaping ", self.dataList[i])
+                        # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
+                        dataAux = getattr(self.ch_dataIn[ich], self.dataList[i])
+                        setattr(self.ch_dataIn[ich], self.dataList[i], None)
+                        setattr(self.ch_dataIn[ich], self.dataList[i], dataAux[0:self.blocksPerFile])
+                        # print(getattr(self.ch_dataIn[ich], self.dataList[i]).shape)
+        else:
+            return
 
 
     def getLabel(self, name, x=None):
@@ -317,7 +337,6 @@ class MergeH5(object):
         ints = [data.nIncohInt for data in self.ch_dataIn]
         self.dataOut.nIncohInt = numpy.stack(ints, axis=1)
 
-        print("nIncohInt 1: ",self.dataOut.nIncohInt.shape)
 
         if self.dataOut.nIncohInt.ndim > 3:
             aux = self.dataOut.nIncohInt
@@ -334,7 +353,6 @@ class MergeH5(object):
         if (self.dataOut.nIncohInt.shape)[0]==self.nChannels:   ## ch,blocks, hei
             self.dataOut.nIncohInt =  numpy.swapaxes(self.dataOut.nIncohInt, 0, 1) ## blocks,ch, hei
 
-        print("nIncohInt 2: ", self.dataOut.nIncohInt.shape)
         #print("utcTime: ", time.shape)
         #print("data_spc ",self.dataOut.data_spc.shape)
         pairsList = [pair for pair in itertools.combinations(self.channelList, 2)]
@@ -487,6 +505,11 @@ class MergeH5(object):
                 #print("Opening file: ",filename)
                 self.readFile(fp,ch)
                 fp.close()
+
+            if self.blocksPerFile == None:
+                self.getBlocksPerFile()
+                print("blocks per file: ", self.blocksPerFile)
+
             self.getDataOut()
             name = name[-16:]
             #print("Final name out: ", name)
diff --git a/schainpy/model/proc/jroproc_spectra.py b/schainpy/model/proc/jroproc_spectra.py
index 74f1aa6..84533d8 100644
--- a/schainpy/model/proc/jroproc_spectra.py
+++ b/schainpy/model/proc/jroproc_spectra.py
@@ -2120,8 +2120,12 @@ class IncohInt(Operation):
                                                                             dataOut.data_spc,
                                                                             dataOut.data_cspc,
                                                                             dataOut.data_dc)
+
         self.incohInt += dataOut.nIncohInt
-        self.nOutliers += dataOut.data_outlier
+
+        if isinstance(dataOut.data_outlier,numpy.ndarray) or isinstance(dataOut.data_outlier,int) or isinstance(dataOut.data_outlier, float):
+            self.nOutliers += dataOut.data_outlier
+
         if self.__dataReady:
             #print("prof: ",dataOut.max_nIncohInt,self.__profIndex)
             dataOut.data_spc = avgdata_spc