| @@ -1,9 +1,9 | |||
|
|
1 |
# schain |
|
|
|
1 | # schain | |
|
|
2 | 2 | |
|
|
3 | 3 | Command Line Interface for SIGNAL CHAIN - jro |
|
|
4 | 4 | |
|
|
5 | 5 | # Usage |
|
|
6 | 6 | |
|
|
7 | 7 | To use it: |
|
|
8 | 8 | |
|
|
9 | 9 | $ schain-cli --help |
| @@ -1,365 +1,366 | |||
|
|
1 | 1 | ''' |
|
|
2 | 2 | Created on Nov 9, 2016 |
|
|
3 | 3 | |
|
|
4 | 4 | @author: roj- LouVD |
|
|
5 | 5 | ''' |
|
|
6 | 6 | |
|
|
7 | 7 | |
|
|
8 | 8 | import os |
|
|
9 | 9 | import sys |
|
|
10 | 10 | import time |
|
|
11 | 11 | import glob |
|
|
12 | 12 | import datetime |
|
|
13 | 13 | |
|
|
14 | 14 | import numpy |
|
|
15 | 15 | |
|
|
16 | 16 | from schainpy.model.proc.jroproc_base import ProcessingUnit |
|
|
17 | 17 | from schainpy.model.data.jrodata import Parameters |
|
|
18 | 18 | from schainpy.model.io.jroIO_base import JRODataReader, isNumber |
|
|
19 | 19 | |
|
|
20 | 20 | FILE_HEADER_STRUCTURE = numpy.dtype([ |
|
|
21 | 21 | ('FMN', '<u4'), |
|
|
22 | 22 | ('nrec', '<u4'), |
|
|
23 | 23 | ('fr_offset', '<u4'), |
|
|
24 | 24 | ('id', '<u4'), |
|
|
25 | 25 | ('site', 'u1', (32,)) |
|
|
26 | 26 | ]) |
|
|
27 | 27 | |
|
|
28 | 28 | REC_HEADER_STRUCTURE = numpy.dtype([ |
|
|
29 | 29 | ('rmn', '<u4'), |
|
|
30 | 30 | ('rcounter', '<u4'), |
|
|
31 | 31 | ('nr_offset', '<u4'), |
|
|
32 | 32 | ('tr_offset', '<u4'), |
|
|
33 | 33 | ('time', '<u4'), |
|
|
34 | 34 | ('time_msec', '<u4'), |
|
|
35 | 35 | ('tag', 'u1', (32,)), |
|
|
36 | 36 | ('comments', 'u1', (32,)), |
|
|
37 | 37 | ('lat', '<f4'), |
|
|
38 | 38 | ('lon', '<f4'), |
|
|
39 | 39 | ('gps_status', '<u4'), |
|
|
40 | 40 | ('freq', '<u4'), |
|
|
41 | 41 | ('freq0', '<u4'), |
|
|
42 | 42 | ('nchan', '<u4'), |
|
|
43 | 43 | ('delta_r', '<u4'), |
|
|
44 | 44 | ('nranges', '<u4'), |
|
|
45 | 45 | ('r0', '<u4'), |
|
|
46 | 46 | ('prf', '<u4'), |
|
|
47 | 47 | ('ncoh', '<u4'), |
|
|
48 | 48 | ('npoints', '<u4'), |
|
|
49 | 49 | ('polarization', '<i4'), |
|
|
50 | 50 | ('rx_filter', '<u4'), |
|
|
51 | 51 | ('nmodes', '<u4'), |
|
|
52 | 52 | ('dmode_index', '<u4'), |
|
|
53 | 53 | ('dmode_rngcorr', '<u4'), |
|
|
54 | 54 | ('nrxs', '<u4'), |
|
|
55 | 55 | ('acf_length', '<u4'), |
|
|
56 | 56 | ('acf_lags', '<u4'), |
|
|
57 | 57 | ('sea_to_atmos', '<f4'), |
|
|
58 | 58 | ('sea_notch', '<u4'), |
|
|
59 | 59 | ('lh_sea', '<u4'), |
|
|
60 | 60 | ('hh_sea', '<u4'), |
|
|
61 | 61 | ('nbins_sea', '<u4'), |
|
|
62 | 62 | ('min_snr', '<f4'), |
|
|
63 | 63 | ('min_cc', '<f4'), |
|
|
64 | 64 | ('max_time_diff', '<f4') |
|
|
65 | 65 | ]) |
|
|
66 | 66 | |
|
|
67 | 67 | DATA_STRUCTURE = numpy.dtype([ |
|
|
68 | 68 | ('range', '<u4'), |
|
|
69 | 69 | ('status', '<u4'), |
|
|
70 | 70 | ('zonal', '<f4'), |
|
|
71 | 71 | ('meridional', '<f4'), |
|
|
72 | 72 | ('vertical', '<f4'), |
|
|
73 | 73 | ('zonal_a', '<f4'), |
|
|
74 | 74 | ('meridional_a', '<f4'), |
|
|
75 | 75 | ('corrected_fading', '<f4'), # seconds |
|
|
76 | 76 | ('uncorrected_fading', '<f4'), # seconds |
|
|
77 | 77 | ('time_diff', '<f4'), |
|
|
78 | 78 | ('major_axis', '<f4'), |
|
|
79 | 79 | ('axial_ratio', '<f4'), |
|
|
80 | 80 | ('orientation', '<f4'), |
|
|
81 | 81 | ('sea_power', '<u4'), |
|
|
82 | 82 | ('sea_algorithm', '<u4') |
|
|
83 | 83 | ]) |
|
|
84 | 84 | |
|
|
85 | 85 | |
|
|
86 | 86 | class BLTRParamReader(JRODataReader, ProcessingUnit): |
|
|
87 | 87 | ''' |
|
|
88 | 88 | Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files |
|
|
89 | 89 | ''' |
|
|
90 | 90 | |
|
|
91 | 91 | ext = '.sswma' |
|
|
92 | 92 | |
|
|
93 | 93 | def __init__(self, **kwargs): |
|
|
94 | 94 | |
|
|
95 | 95 | ProcessingUnit.__init__(self, **kwargs) |
|
|
96 | 96 | |
|
|
97 | 97 | self.dataOut = Parameters() |
|
|
98 | 98 | self.counter_records = 0 |
|
|
99 | 99 | self.flagNoMoreFiles = 0 |
|
|
100 | 100 | self.isConfig = False |
|
|
101 | 101 | self.filename = None |
|
|
102 | 102 | |
|
|
103 | 103 | def setup(self, |
|
|
104 | 104 | path=None, |
|
|
105 | 105 | startDate=None, |
|
|
106 | 106 | endDate=None, |
|
|
107 | 107 | ext=None, |
|
|
108 | 108 | startTime=datetime.time(0, 0, 0), |
|
|
109 | 109 | endTime=datetime.time(23, 59, 59), |
|
|
110 | 110 | timezone=0, |
|
|
111 |
status_value=0, |
|
|
|
111 | status_value=0, | |
|
|
112 | **kwargs): | |
|
|
112 | 113 | |
|
|
113 | 114 | self.path = path |
|
|
114 | 115 | self.startTime = startTime |
|
|
115 | 116 | self.endTime = endTime |
|
|
116 | 117 | self.status_value = status_value |
|
|
117 | 118 | |
|
|
118 | 119 | if self.path is None: |
|
|
119 | 120 | raise ValueError, "The path is not valid" |
|
|
120 | 121 | |
|
|
121 | 122 | if ext is None: |
|
|
122 | 123 | ext = self.ext |
|
|
123 | 124 | |
|
|
124 | 125 | self.search_files(self.path, startDate, endDate, ext) |
|
|
125 | 126 | self.timezone = timezone |
|
|
126 | 127 | self.fileIndex = 0 |
|
|
127 | 128 | |
|
|
128 | 129 | if not self.fileList: |
|
|
129 | 130 | raise Warning, "There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % ( |
|
|
130 | 131 | path) |
|
|
131 | 132 | |
|
|
132 | 133 | self.setNextFile() |
|
|
133 | 134 | |
|
|
134 | 135 | def search_files(self, path, startDate, endDate, ext): |
|
|
135 | 136 | ''' |
|
|
136 | 137 | Searching for BLTR rawdata file in path |
|
|
137 | 138 | Creating a list of file to proces included in [startDate,endDate] |
|
|
138 | 139 | |
|
|
139 | 140 | Input: |
|
|
140 | 141 | path - Path to find BLTR rawdata files |
|
|
141 | 142 | startDate - Select file from this date |
|
|
142 | 143 | enDate - Select file until this date |
|
|
143 | 144 | ext - Extension of the file to read |
|
|
144 | 145 | |
|
|
145 | 146 | ''' |
|
|
146 | 147 | |
|
|
147 | 148 | print 'Searching file in %s ' % (path) |
|
|
148 | 149 | foldercounter = 0 |
|
|
149 | 150 | fileList0 = glob.glob1(path, "*%s" % ext) |
|
|
150 | 151 | fileList0.sort() |
|
|
151 | 152 | |
|
|
152 | 153 | self.fileList = [] |
|
|
153 | 154 | self.dateFileList = [] |
|
|
154 | 155 | |
|
|
155 | 156 | for thisFile in fileList0: |
|
|
156 | 157 | year = thisFile[-14:-10] |
|
|
157 | 158 | if not isNumber(year): |
|
|
158 | 159 | continue |
|
|
159 | 160 | |
|
|
160 | 161 | month = thisFile[-10:-8] |
|
|
161 | 162 | if not isNumber(month): |
|
|
162 | 163 | continue |
|
|
163 | 164 | |
|
|
164 | 165 | day = thisFile[-8:-6] |
|
|
165 | 166 | if not isNumber(day): |
|
|
166 | 167 | continue |
|
|
167 | 168 | |
|
|
168 | 169 | year, month, day = int(year), int(month), int(day) |
|
|
169 | 170 | dateFile = datetime.date(year, month, day) |
|
|
170 | 171 | |
|
|
171 | 172 | if (startDate > dateFile) or (endDate < dateFile): |
|
|
172 | 173 | continue |
|
|
173 | 174 | |
|
|
174 | 175 | self.fileList.append(thisFile) |
|
|
175 | 176 | self.dateFileList.append(dateFile) |
|
|
176 | 177 | |
|
|
177 | 178 | return |
|
|
178 | 179 | |
|
|
179 | 180 | def setNextFile(self): |
|
|
180 | 181 | |
|
|
181 | 182 | file_id = self.fileIndex |
|
|
182 | 183 | |
|
|
183 | 184 | if file_id == len(self.fileList): |
|
|
184 | 185 | print '\nNo more files in the folder' |
|
|
185 | 186 | print 'Total number of file(s) read : {}'.format(self.fileIndex + 1) |
|
|
186 | 187 | self.flagNoMoreFiles = 1 |
|
|
187 | 188 | return 0 |
|
|
188 | 189 | |
|
|
189 | 190 | print '\n[Setting file] (%s) ...' % self.fileList[file_id] |
|
|
190 | 191 | filename = os.path.join(self.path, self.fileList[file_id]) |
|
|
191 | 192 | |
|
|
192 | 193 | dirname, name = os.path.split(filename) |
|
|
193 | 194 | # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya |
|
|
194 | 195 | self.siteFile = name.split('.')[0] |
|
|
195 | 196 | if self.filename is not None: |
|
|
196 | 197 | self.fp.close() |
|
|
197 | 198 | self.filename = filename |
|
|
198 | 199 | self.fp = open(self.filename, 'rb') |
|
|
199 | 200 | self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1) |
|
|
200 | 201 | self.nrecords = self.header_file['nrec'][0] |
|
|
201 | 202 | self.sizeOfFile = os.path.getsize(self.filename) |
|
|
202 | 203 | self.counter_records = 0 |
|
|
203 | 204 | self.flagIsNewFile = 0 |
|
|
204 | 205 | self.fileIndex += 1 |
|
|
205 | 206 | |
|
|
206 | 207 | return 1 |
|
|
207 | 208 | |
|
|
208 | 209 | def readNextBlock(self): |
|
|
209 | 210 | |
|
|
210 | 211 | while True: |
|
|
211 | 212 | if self.counter_records == self.nrecords: |
|
|
212 | 213 | self.flagIsNewFile = 1 |
|
|
213 | 214 | if not self.setNextFile(): |
|
|
214 | 215 | return 0 |
|
|
215 | 216 | |
|
|
216 | 217 | self.readBlock() |
|
|
217 | 218 | |
|
|
218 | 219 | if (self.datatime.time() < self.startTime) or (self.datatime.time() > self.endTime): |
|
|
219 | 220 | print "[Reading] Record No. %d/%d -> %s [Skipping]" % ( |
|
|
220 | 221 | self.counter_records, |
|
|
221 | 222 | self.nrecords, |
|
|
222 | 223 | self.datatime.ctime()) |
|
|
223 | 224 | continue |
|
|
224 | 225 | break |
|
|
225 | 226 | |
|
|
226 | 227 | print "[Reading] Record No. %d/%d -> %s" % ( |
|
|
227 | 228 | self.counter_records, |
|
|
228 | 229 | self.nrecords, |
|
|
229 | 230 | self.datatime.ctime()) |
|
|
230 | 231 | |
|
|
231 | 232 | return 1 |
|
|
232 | 233 | |
|
|
233 | 234 | def readBlock(self): |
|
|
234 | 235 | |
|
|
235 | 236 | pointer = self.fp.tell() |
|
|
236 | 237 | header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1) |
|
|
237 | 238 | self.nchannels = header_rec['nchan'][0] / 2 |
|
|
238 | 239 | self.kchan = header_rec['nrxs'][0] |
|
|
239 | 240 | self.nmodes = header_rec['nmodes'][0] |
|
|
240 | 241 | self.nranges = header_rec['nranges'][0] |
|
|
241 | 242 | self.fp.seek(pointer) |
|
|
242 | 243 | self.height = numpy.empty((self.nmodes, self.nranges)) |
|
|
243 | 244 | self.snr = numpy.empty((self.nmodes, self.nchannels, self.nranges)) |
|
|
244 | 245 | self.buffer = numpy.empty((self.nmodes, 3, self.nranges)) |
|
|
245 | 246 | |
|
|
246 | 247 | for mode in range(self.nmodes): |
|
|
247 | 248 | self.readHeader() |
|
|
248 | 249 | data = self.readData() |
|
|
249 | 250 | self.height[mode] = (data[0] - self.correction) / 1000. |
|
|
250 | 251 | self.buffer[mode] = data[1] |
|
|
251 | 252 | self.snr[mode] = data[2] |
|
|
252 | 253 | |
|
|
253 | 254 | self.counter_records = self.counter_records + self.nmodes |
|
|
254 | 255 | |
|
|
255 | 256 | return |
|
|
256 | 257 | |
|
|
257 | 258 | def readHeader(self): |
|
|
258 | 259 | ''' |
|
|
259 | 260 | RecordHeader of BLTR rawdata file |
|
|
260 | 261 | ''' |
|
|
261 | 262 | |
|
|
262 | 263 | header_structure = numpy.dtype( |
|
|
263 | 264 | REC_HEADER_STRUCTURE.descr + [ |
|
|
264 | 265 | ('antenna_coord', 'f4', (2, self.nchannels)), |
|
|
265 | 266 | ('rx_gains', 'u4', (self.nchannels,)), |
|
|
266 | 267 | ('rx_analysis', 'u4', (self.nchannels,)) |
|
|
267 | 268 | ] |
|
|
268 | 269 | ) |
|
|
269 | 270 | |
|
|
270 | 271 | self.header_rec = numpy.fromfile(self.fp, header_structure, 1) |
|
|
271 | 272 | self.lat = self.header_rec['lat'][0] |
|
|
272 | 273 | self.lon = self.header_rec['lon'][0] |
|
|
273 | 274 | self.delta = self.header_rec['delta_r'][0] |
|
|
274 | 275 | self.correction = self.header_rec['dmode_rngcorr'][0] |
|
|
275 | 276 | self.imode = self.header_rec['dmode_index'][0] |
|
|
276 | 277 | self.antenna = self.header_rec['antenna_coord'] |
|
|
277 | 278 | self.rx_gains = self.header_rec['rx_gains'] |
|
|
278 | 279 | self.time = self.header_rec['time'][0] |
|
|
279 | 280 | tseconds = self.header_rec['time'][0] |
|
|
280 | 281 | local_t1 = time.localtime(tseconds) |
|
|
281 | 282 | self.year = local_t1.tm_year |
|
|
282 | 283 | self.month = local_t1.tm_mon |
|
|
283 | 284 | self.day = local_t1.tm_mday |
|
|
284 | 285 | self.t = datetime.datetime(self.year, self.month, self.day) |
|
|
285 | 286 | self.datatime = datetime.datetime.utcfromtimestamp(self.time) |
|
|
286 | 287 | |
|
|
287 | 288 | def readData(self): |
|
|
288 | 289 | ''' |
|
|
289 | 290 | Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value. |
|
|
290 | 291 | |
|
|
291 | 292 | Input: |
|
|
292 | 293 | status_value - Array data is set to NAN for values that are not equal to status_value |
|
|
293 | 294 | |
|
|
294 | 295 | ''' |
|
|
295 | 296 | |
|
|
296 | 297 | data_structure = numpy.dtype( |
|
|
297 | 298 | DATA_STRUCTURE.descr + [ |
|
|
298 | 299 | ('rx_saturation', 'u4', (self.nchannels,)), |
|
|
299 | 300 | ('chan_offset', 'u4', (2 * self.nchannels,)), |
|
|
300 | 301 | ('rx_amp', 'u4', (self.nchannels,)), |
|
|
301 | 302 | ('rx_snr', 'f4', (self.nchannels,)), |
|
|
302 | 303 | ('cross_snr', 'f4', (self.kchan,)), |
|
|
303 | 304 | ('sea_power_relative', 'f4', (self.kchan,))] |
|
|
304 | 305 | ) |
|
|
305 | 306 | |
|
|
306 | 307 | data = numpy.fromfile(self.fp, data_structure, self.nranges) |
|
|
307 | 308 | |
|
|
308 | 309 | height = data['range'] |
|
|
309 | 310 | winds = numpy.array( |
|
|
310 | 311 | (data['zonal'], data['meridional'], data['vertical'])) |
|
|
311 | 312 | snr = data['rx_snr'].T |
|
|
312 | 313 | |
|
|
313 | 314 | winds[numpy.where(winds == -9999.)] = numpy.nan |
|
|
314 | 315 | winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan |
|
|
315 | 316 | snr[numpy.where(snr == -9999.)] = numpy.nan |
|
|
316 | 317 | snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan |
|
|
317 | 318 | snr = numpy.power(10, snr / 10) |
|
|
318 | 319 | |
|
|
319 | 320 | return height, winds, snr |
|
|
320 | 321 | |
|
|
321 | 322 | def set_output(self): |
|
|
322 | 323 | ''' |
|
|
323 | 324 | Storing data from databuffer to dataOut object |
|
|
324 | 325 | ''' |
|
|
325 | 326 | |
|
|
326 | 327 | self.dataOut.data_SNR = self.snr |
|
|
327 | 328 | self.dataOut.height = self.height |
|
|
328 | 329 | self.dataOut.data_output = self.buffer |
|
|
329 | 330 | self.dataOut.utctimeInit = self.time |
|
|
330 | 331 | self.dataOut.utctime = self.dataOut.utctimeInit |
|
|
331 | 332 | self.dataOut.useLocalTime = False |
|
|
332 | 333 | self.dataOut.paramInterval = 157 |
|
|
333 | 334 | self.dataOut.timezone = self.timezone |
|
|
334 | 335 | self.dataOut.site = self.siteFile |
|
|
335 | 336 | self.dataOut.nrecords = self.nrecords / self.nmodes |
|
|
336 | 337 | self.dataOut.sizeOfFile = self.sizeOfFile |
|
|
337 | 338 | self.dataOut.lat = self.lat |
|
|
338 | 339 | self.dataOut.lon = self.lon |
|
|
339 | 340 | self.dataOut.channelList = range(self.nchannels) |
|
|
340 | 341 | self.dataOut.kchan = self.kchan |
|
|
341 | 342 | # self.dataOut.nHeights = self.nranges |
|
|
342 | 343 | self.dataOut.delta = self.delta |
|
|
343 | 344 | self.dataOut.correction = self.correction |
|
|
344 | 345 | self.dataOut.nmodes = self.nmodes |
|
|
345 | 346 | self.dataOut.imode = self.imode |
|
|
346 | 347 | self.dataOut.antenna = self.antenna |
|
|
347 | 348 | self.dataOut.rx_gains = self.rx_gains |
|
|
348 | 349 | self.dataOut.flagNoData = False |
|
|
349 | 350 | |
|
|
350 | 351 | def getData(self): |
|
|
351 | 352 | ''' |
|
|
352 | 353 | Storing data from databuffer to dataOut object |
|
|
353 | 354 | ''' |
|
|
354 | 355 | if self.flagNoMoreFiles: |
|
|
355 | 356 | self.dataOut.flagNoData = True |
|
|
356 | 357 | print 'No file left to process' |
|
|
357 | 358 | return 0 |
|
|
358 | 359 | |
|
|
359 | 360 | if not self.readNextBlock(): |
|
|
360 | 361 | self.dataOut.flagNoData = True |
|
|
361 | 362 | return 0 |
|
|
362 | 363 | |
|
|
363 | 364 | self.set_output() |
|
|
364 | 365 | |
|
|
365 | 366 | return 1 |
| @@ -1,44 +1,45 | |||
|
|
1 | 1 | ''' |
|
|
2 | 2 | SCHAINPY - LOG |
|
|
3 | 3 | Simple helper for log standarization |
|
|
4 | 4 | Usage: |
|
|
5 | 5 | from schainpy.utils import log |
|
|
6 | 6 | log.error('A kitten died beacuse of you') |
|
|
7 | 7 | log.warning('You are doing it wrong but what the heck, I'll allow it) |
|
|
8 | 8 | log.succes('YOU ROCK!') |
|
|
9 | 9 | To create your own logger inside your class do it like this: |
|
|
10 | 10 | from schainpy.utils import log |
|
|
11 | 11 | awesomeLogger = log.makelogger("never gonna", bg="red", fg="white") |
|
|
12 | 12 | awesomeLogger('give you up') |
|
|
13 | 13 | which will look like this: |
|
|
14 | 14 | [NEVER GONNA] - give you up |
|
|
15 | 15 | with color red as background and white as foreground. |
|
|
16 | 16 | ''' |
|
|
17 | 17 | |
|
|
18 | 18 | import click |
|
|
19 | 19 | |
|
|
20 | ||
|
|
20 | 21 | def warning(message, tag='Warning'): |
|
|
21 | 22 | click.echo(click.style('[{}] {}'.format(tag, message), fg='yellow')) |
|
|
22 | 23 | pass |
|
|
23 | 24 | |
|
|
24 | 25 | |
|
|
25 | 26 | def error(message, tag='Error'): |
|
|
26 | 27 | click.echo(click.style('[{}] {}'.format(tag, message), fg='red')) |
|
|
27 | 28 | pass |
|
|
28 | 29 | |
|
|
29 | 30 | |
|
|
30 | 31 | def success(message, tag='Info'): |
|
|
31 | 32 | click.echo(click.style('[{}] {}'.format(tag, message), fg='green')) |
|
|
32 | 33 | pass |
|
|
33 | 34 | |
|
|
34 | 35 | |
|
|
35 | 36 | def log(message, tag='Info'): |
|
|
36 | 37 | click.echo('[{}] {}'.format(tag, message)) |
|
|
37 | 38 | pass |
|
|
38 | 39 | |
|
|
39 | 40 | |
|
|
40 | 41 | def makelogger(tag, bg='reset', fg='reset'): |
|
|
41 | 42 | def func(message): |
|
|
42 |
click.echo(click.style('[{}] {}'.format( |
|
|
|
43 |
|
|
|
|
43 | click.echo(click.style('[{}] {}'.format( | |
|
|
44 | tag.upper(), message), bg=bg, fg=fg)) | |
|
|
44 | 45 | return func |
|
|
1 | NO CONTENT: file was removed |
General Comments 0
You need to be logged in to leave comments.
Login now